feat: add orchestrator recovery automation (#356) (#362)

* fix: preserve PR number in recovery and dedupe validation utilities

Bugbot #2908822306: Use parsePrFromUrl utility to correctly extract
PR number from URL instead of defaulting to 0. Applied to both
recovery/actions.ts and session-manager.ts for consistency.

Bugbot #2908822310: Extract safeJsonParse and validateStatus to
shared utils/validation.ts to eliminate duplication between
recovery/validator.ts and session-manager.ts.

Clean rebuild from origin/main with only recovery-specific changes.

* fix: add PR number fallback and remove unused import

Bugbot #2908822306: Add fallback regex to extract PR number from URL
ending when full GitHub URL pattern doesn't match (e.g., non-GitHub URLs).

Bugbot #2908822310: Remove unused SessionStatus import from session-manager.ts
(now imported from utils/validation.ts).

Tests: 406 passed, typecheck clean.

* fix(core): harden recovery action selection and escalation

* fix(core): preserve recovered agent summary metadata

* fix(core): reuse canonical PR type in URL parser

Avoid shadowing the core PRInfo type in the recovery URL parser while keeping the non-GitHub trailing-number fallback covered by tests.

* fix(core): align recovery metadata and session reconstruction

Persist restored timestamps under the canonical metadata key, share
session reconstruction logic between recovery and session loading, and
log the real escalation reason when recovery aborts on retry limits.

* fix(core): keep dry-run escalation reasons accurate

Use the assessment's actual escalation reason in dry-run results so preview output matches real execution behavior for partial-session escalations.

* fix(core): dedupe recovery scanning and honor custom log path

Reuse metadata listing rules in scanner to avoid duplicated session ID filters,
and preserve user-supplied recovery logPath in recovery manager APIs.

* fix(core): align dry-run recovery behavior with real actions

Compute recovery escalation decisions before dry-run returns and route
single-session dry runs through executeAction so action-specific fields
(reason/manual-intervention) are preserved.

* fix(core): derive dry-run recovery report from action execution

Run dry-run recovery classification through executeAction so report actions
match real execution logic, including max-attempt escalation decisions.
This commit is contained in:
Harsh Batheja 2026-03-10 13:43:42 +05:30 committed by deepak
parent 28b6d8ac5c
commit c2be259c90
13 changed files with 1416 additions and 73 deletions

View File

@ -0,0 +1,284 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import { readMetadataRaw } from "../metadata.js";
import { getSessionsDir } from "../paths.js";
import { escalateSession, recoverSession } from "../recovery/actions.js";
import { runRecovery } from "../recovery/manager.js";
import { getRecoveryLogPath, scanAllSessions } from "../recovery/scanner.js";
import {
DEFAULT_RECOVERY_CONFIG,
type RecoveryAssessment,
type RecoveryContext,
} from "../recovery/types.js";
import type { OrchestratorConfig, PluginRegistry } from "../types.js";
function makeConfig(rootDir: string): OrchestratorConfig {
return {
configPath: join(rootDir, "agent-orchestrator.yaml"),
port: 3000,
readyThresholdMs: 300_000,
defaults: {
runtime: "tmux",
agent: "claude-code",
workspace: "worktree",
notifiers: ["desktop"],
},
projects: {
app: {
name: "app",
repo: "org/repo",
path: join(rootDir, "project"),
defaultBranch: "main",
sessionPrefix: "app",
},
},
notifiers: {},
notificationRouting: {
urgent: ["desktop"],
action: ["desktop"],
warning: ["desktop"],
info: ["desktop"],
},
reactions: {},
};
}
function makeRegistry(): PluginRegistry {
return {
register: vi.fn(),
get: vi.fn().mockReturnValue(null),
list: vi.fn().mockReturnValue([]),
loadBuiltins: vi.fn().mockResolvedValue(undefined),
loadFromConfig: vi.fn().mockResolvedValue(undefined),
};
}
function makeAssessment(overrides: Partial<RecoveryAssessment> = {}): RecoveryAssessment {
return {
sessionId: "app-1",
projectId: "app",
classification: "live",
action: "recover",
reason: "Session is running normally",
runtimeAlive: true,
runtimeHandle: { id: "rt-1", runtimeName: "tmux", data: {} },
workspaceExists: true,
workspacePath: "/tmp/worktree",
agentProcessRunning: true,
agentActivity: "active",
metadataValid: true,
metadataStatus: "working",
rawMetadata: {
project: "app",
branch: "feat/test",
issue: "123",
pr: "https://github.com/org/repo/pull/42",
createdAt: "2025-01-01T00:00:00.000Z",
status: "working",
summary: "Recovered summary",
},
...overrides,
};
}
function makeContext(rootDir: string, overrides: Partial<RecoveryContext> = {}): RecoveryContext {
return {
configPath: join(rootDir, "agent-orchestrator.yaml"),
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: join(rootDir, "recovery.log"),
},
dryRun: false,
...overrides,
};
}
describe("recoverSession", () => {
let rootDir: string;
afterEach(() => {
if (rootDir) {
rmSync(rootDir, { recursive: true, force: true });
}
});
it("persists restoredAt and returns a session with restoredAt", async () => {
rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const config = makeConfig(rootDir);
const registry = makeRegistry();
const assessment = makeAssessment();
const context = makeContext(rootDir);
const result = await recoverSession(assessment, config, registry, context);
const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path);
const metadata = readMetadataRaw(sessionsDir, assessment.sessionId);
expect(result.success).toBe(true);
expect(result.session?.restoredAt).toBeInstanceOf(Date);
expect(metadata?.["restoredAt"]).toBeDefined();
expect(metadata?.["recoveredAt"]).toBeUndefined();
});
it("returns the max-attempt reason when recovery escalates", async () => {
rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const config = makeConfig(rootDir);
const registry = makeRegistry();
const assessment = makeAssessment({
rawMetadata: {
...makeAssessment().rawMetadata,
recoveryCount: "3",
},
});
const context = makeContext(rootDir, {
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: join(rootDir, "recovery.log"),
maxRecoveryAttempts: 3,
},
});
const result = await recoverSession(assessment, config, registry, context);
expect(result.success).toBe(true);
expect(result.action).toBe("escalate");
expect(result.reason).toBe("Exceeded max recovery attempts (3)");
});
it("dry-run recovery reports escalate when attempts exceed limit", async () => {
rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const config = makeConfig(rootDir);
const registry = makeRegistry();
const assessment = makeAssessment({
rawMetadata: {
...makeAssessment().rawMetadata,
recoveryCount: "3",
},
});
const context = makeContext(rootDir, {
dryRun: true,
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: join(rootDir, "recovery.log"),
maxRecoveryAttempts: 3,
},
});
const result = await recoverSession(assessment, config, registry, context);
expect(result.success).toBe(true);
expect(result.action).toBe("escalate");
expect(result.requiresManualIntervention).toBe(true);
expect(result.reason).toBe("Exceeded max recovery attempts (3)");
});
});
describe("escalateSession", () => {
let rootDir: string;
afterEach(() => {
if (rootDir) {
rmSync(rootDir, { recursive: true, force: true });
}
});
it("uses the assessment reason during dry runs", async () => {
rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const config = makeConfig(rootDir);
const registry = makeRegistry();
const assessment = makeAssessment({
action: "escalate",
classification: "partial",
reason: "Workspace exists but runtime is missing",
});
const context = makeContext(rootDir, { dryRun: true });
const result = await escalateSession(assessment, config, registry, context);
expect(result.success).toBe(true);
expect(result.action).toBe("escalate");
expect(result.reason).toBe("Workspace exists but runtime is missing");
});
});
describe("recovery manager and scanner", () => {
let rootDir: string;
afterEach(() => {
if (rootDir) {
rmSync(rootDir, { recursive: true, force: true });
}
});
it("respects custom recovery logPath in manager options", async () => {
rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const config = makeConfig(rootDir);
const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path);
mkdirSync(sessionsDir, { recursive: true });
writeFileSync(
join(sessionsDir, "app-1"),
"project=app\nstatus=terminated\nworktree=/tmp/worktree\n",
"utf-8",
);
const customLogPath = join(rootDir, "custom-recovery.log");
const registry = makeRegistry();
await runRecovery({
config,
registry,
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: customLogPath,
},
});
expect(existsSync(customLogPath)).toBe(true);
expect(readFileSync(customLogPath, "utf-8")).toContain('"sessionId":"app-1"');
const defaultLogPath = getRecoveryLogPath(config.configPath);
expect(defaultLogPath).not.toBe(customLogPath);
});
it("scans sessions using metadata listing rules", () => {
rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const config = makeConfig(rootDir);
const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path);
mkdirSync(sessionsDir, { recursive: true });
writeFileSync(join(sessionsDir, "app-1"), "project=app\nstatus=working\n", "utf-8");
writeFileSync(join(sessionsDir, ".tmp"), "project=app\n", "utf-8");
writeFileSync(join(sessionsDir, "bad.session"), "project=app\n", "utf-8");
const scanned = scanAllSessions(config);
expect(scanned).toHaveLength(1);
expect(scanned[0]?.sessionId).toBe("app-1");
});
});

View File

@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { isRetryableHttpStatus, normalizeRetryConfig, readLastJsonlEntry } from "../utils.js";
import { parsePrFromUrl } from "../utils/pr.js";
describe("readLastJsonlEntry", () => {
let tmpDir: string;
@ -114,3 +115,27 @@ describe("retry utilities", () => {
});
});
});
describe("parsePrFromUrl", () => {
it("parses GitHub PR URLs", () => {
expect(parsePrFromUrl("https://github.com/foo/bar/pull/123")).toEqual({
owner: "foo",
repo: "bar",
number: 123,
url: "https://github.com/foo/bar/pull/123",
});
});
it("falls back to trailing number for non-GitHub URLs", () => {
expect(parsePrFromUrl("https://gitlab.com/foo/bar/-/merge_requests/456")).toEqual({
owner: "",
repo: "",
number: 456,
url: "https://gitlab.com/foo/bar/-/merge_requests/456",
});
});
it("returns null when the URL has no PR number", () => {
expect(parsePrFromUrl("https://example.com/foo/bar/pull/not-a-number")).toBeNull();
});
});

View File

@ -0,0 +1,229 @@
import type { OrchestratorConfig, PluginRegistry, Runtime, Workspace } from "../types.js";
import { updateMetadata, deleteMetadata } from "../metadata.js";
import { getSessionsDir } from "../paths.js";
import { validateStatus } from "../utils/validation.js";
import { sessionFromMetadata } from "../utils/session-from-metadata.js";
import type { RecoveryAssessment, RecoveryResult, RecoveryContext } from "./types.js";
export async function recoverSession(
assessment: RecoveryAssessment,
config: OrchestratorConfig,
registry: PluginRegistry,
context: RecoveryContext,
): Promise<RecoveryResult> {
const { sessionId, projectId, rawMetadata } = assessment;
const recoveryCount = rawMetadata["recoveryCount"]
? parseInt(rawMetadata["recoveryCount"], 10) + 1
: 1;
if (context.dryRun) {
if (recoveryCount > context.recoveryConfig.maxRecoveryAttempts) {
return {
success: true,
sessionId,
action: "escalate",
requiresManualIntervention: true,
reason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`,
};
}
return {
success: true,
sessionId,
action: "recover",
};
}
try {
const now = new Date().toISOString();
const preservedStatus = validateStatus(rawMetadata["status"]);
const project = config.projects[projectId];
const sessionsDir = getSessionsDir(config.configPath, project.path);
if (recoveryCount > context.recoveryConfig.maxRecoveryAttempts) {
updateMetadata(sessionsDir, sessionId, {
status: "stuck",
escalatedAt: now,
escalationReason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`,
recoveryCount: String(recoveryCount),
});
return {
success: true,
sessionId,
action: "escalate",
requiresManualIntervention: true,
reason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`,
};
}
updateMetadata(sessionsDir, sessionId, {
status: preservedStatus,
restoredAt: now,
recoveryCount: String(recoveryCount),
});
const updatedMetadata = {
...rawMetadata,
status: preservedStatus,
restoredAt: now,
recoveryCount: String(recoveryCount),
};
const session = sessionFromMetadata(sessionId, updatedMetadata, {
status: preservedStatus,
runtimeHandle: assessment.runtimeHandle,
lastActivityAt: new Date(),
restoredAt: new Date(now),
});
return {
success: true,
sessionId,
action: "recover",
session,
};
} catch (error) {
return {
success: false,
sessionId,
action: "recover",
error: error instanceof Error ? error.message : String(error),
};
}
}
export async function cleanupSession(
assessment: RecoveryAssessment,
config: OrchestratorConfig,
registry: PluginRegistry,
context: RecoveryContext,
): Promise<RecoveryResult> {
const { sessionId, projectId, rawMetadata, runtimeAlive, workspaceExists } = assessment;
if (context.dryRun) {
return {
success: true,
sessionId,
action: "cleanup",
};
}
try {
const project = config.projects[projectId];
const runtimeName = project.runtime ?? config.defaults.runtime;
const workspaceName = project.workspace ?? config.defaults.workspace;
const runtime = registry.get<Runtime>("runtime", runtimeName);
const workspace = registry.get<Workspace>("workspace", workspaceName);
if (runtimeAlive && assessment.runtimeHandle && runtime) {
try {
await runtime.destroy(assessment.runtimeHandle);
} catch {
// ignore cleanup errors
}
}
const workspacePath = rawMetadata["worktree"];
if (workspacePath && workspaceExists && workspace) {
try {
await workspace.destroy(workspacePath);
} catch {
// ignore cleanup errors
}
}
const sessionsDir = getSessionsDir(config.configPath, project.path);
updateMetadata(sessionsDir, sessionId, {
status: "terminated",
terminatedAt: new Date().toISOString(),
terminationReason: "cleanup",
});
deleteMetadata(sessionsDir, sessionId, true);
return {
success: true,
sessionId,
action: "cleanup",
};
} catch (error) {
return {
success: false,
sessionId,
action: "cleanup",
error: error instanceof Error ? error.message : String(error),
};
}
}
export async function escalateSession(
assessment: RecoveryAssessment,
config: OrchestratorConfig,
_registry: PluginRegistry,
context: RecoveryContext,
): Promise<RecoveryResult> {
const { sessionId, projectId, reason } = assessment;
if (context.dryRun) {
return {
success: true,
sessionId,
action: "escalate",
requiresManualIntervention: true,
reason,
};
}
try {
const project = config.projects[projectId];
const sessionsDir = getSessionsDir(config.configPath, project.path);
updateMetadata(sessionsDir, sessionId, {
status: "stuck",
escalatedAt: new Date().toISOString(),
escalationReason: reason,
});
return {
success: true,
sessionId,
action: "escalate",
requiresManualIntervention: true,
reason,
};
} catch (error) {
return {
success: false,
sessionId,
action: "escalate",
error: error instanceof Error ? error.message : String(error),
requiresManualIntervention: true,
};
}
}
export async function executeAction(
assessment: RecoveryAssessment,
config: OrchestratorConfig,
registry: PluginRegistry,
context: RecoveryContext,
): Promise<RecoveryResult> {
switch (assessment.action) {
case "recover":
return recoverSession(assessment, config, registry, context);
case "cleanup":
return cleanupSession(assessment, config, registry, context);
case "escalate":
return escalateSession(assessment, config, registry, context);
case "skip":
default:
return {
success: true,
sessionId: assessment.sessionId,
action: "skip",
};
}
}

View File

@ -0,0 +1,32 @@
export type {
RecoveryClassification,
RecoveryAction,
RecoveryAssessment,
RecoveryResult,
RecoveryReport,
RecoveryLogEntry,
RecoveryConfig,
RecoveryContext,
} from "./types.js";
export { DEFAULT_RECOVERY_CONFIG } from "./types.js";
export { scanAllSessions, getRecoveryLogPath, type ScannedSession } from "./scanner.js";
export { validateSession, classifySession, determineAction } from "./validator.js";
export { recoverSession, cleanupSession, escalateSession, executeAction } from "./actions.js";
export {
writeRecoveryLog,
createLogEntry,
formatRecoveryReport,
createEmptyReport,
} from "./logger.js";
export {
runRecovery,
recoverSessionById,
type RecoveryManagerOptions,
type RecoveryRunResult,
} from "./manager.js";

View File

@ -0,0 +1,78 @@
import { appendFileSync, existsSync, mkdirSync } from "node:fs";
import { dirname } from "node:path";
import type { SessionId, SessionStatus } from "../types.js";
import type { RecoveryLogEntry, RecoveryReport } from "./types.js";
export function writeRecoveryLog(logPath: string, entry: RecoveryLogEntry): void {
const dir = dirname(logPath);
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
}
const line = JSON.stringify(entry) + "\n";
appendFileSync(logPath, line, "utf-8");
}
export function createLogEntry(
sessionId: SessionId,
action: RecoveryLogEntry["action"],
options?: {
previousStatus?: SessionStatus;
reason?: string;
error?: string;
details?: Record<string, unknown>;
},
): RecoveryLogEntry {
return {
timestamp: new Date().toISOString(),
sessionId,
action,
...options,
};
}
export function formatRecoveryReport(report: RecoveryReport): string {
const lines: string[] = [
`Recovery Report - ${report.timestamp.toISOString()}`,
`Duration: ${report.durationMs}ms`,
`Sessions scanned: ${report.totalScanned}`,
"",
];
if (report.recovered.length > 0) {
lines.push(`Recovered (${report.recovered.length}): ${report.recovered.join(", ")}`);
}
if (report.cleanedUp.length > 0) {
lines.push(`Cleaned up (${report.cleanedUp.length}): ${report.cleanedUp.join(", ")}`);
}
if (report.escalated.length > 0) {
lines.push(`Escalated (${report.escalated.length}): ${report.escalated.join(", ")}`);
}
if (report.skipped.length > 0) {
lines.push(`Skipped (${report.skipped.length}): ${report.skipped.join(", ")}`);
}
if (report.errors.length > 0) {
lines.push("", "Errors:");
for (const { sessionId, error } of report.errors) {
lines.push(` ${sessionId}: ${error}`);
}
}
return lines.join("\n");
}
export function createEmptyReport(): RecoveryReport {
return {
timestamp: new Date(),
totalScanned: 0,
recovered: [],
cleanedUp: [],
escalated: [],
skipped: [],
errors: [],
durationMs: 0,
};
}

View File

@ -0,0 +1,190 @@
import type { OrchestratorConfig, PluginRegistry, Session } from "../types.js";
import { scanAllSessions, getRecoveryLogPath } from "./scanner.js";
import { validateSession } from "./validator.js";
import { executeAction } from "./actions.js";
import { writeRecoveryLog, createLogEntry, createEmptyReport } from "./logger.js";
import {
DEFAULT_RECOVERY_CONFIG,
type RecoveryContext,
type RecoveryReport,
type RecoveryResult,
type RecoveryAssessment,
type RecoveryConfig,
} from "./types.js";
export interface RecoveryManagerOptions {
config: OrchestratorConfig;
registry: PluginRegistry;
recoveryConfig?: Partial<RecoveryConfig>;
dryRun?: boolean;
projectFilter?: string;
}
export interface RecoveryRunResult {
report: RecoveryReport;
assessments: RecoveryAssessment[];
results: RecoveryResult[];
recoveredSessions: Session[];
}
export async function runRecovery(options: RecoveryManagerOptions): Promise<RecoveryRunResult> {
const { config, registry, dryRun = false, projectFilter } = options;
const startTime = Date.now();
const recoveryConfig: RecoveryConfig = {
...DEFAULT_RECOVERY_CONFIG,
...options.recoveryConfig,
logPath: options.recoveryConfig?.logPath ?? getRecoveryLogPath(config.configPath),
};
const context: RecoveryContext = {
configPath: config.configPath,
recoveryConfig,
dryRun,
};
const report = createEmptyReport();
const assessments: RecoveryAssessment[] = [];
const results: RecoveryResult[] = [];
const recoveredSessions: Session[] = [];
const scannedSessions = scanAllSessions(config, projectFilter);
report.totalScanned = scannedSessions.length;
for (const scanned of scannedSessions) {
const assessment = await validateSession(scanned, config, registry, recoveryConfig);
assessments.push(assessment);
if (dryRun) {
const dryRunResult = await executeAction(assessment, config, registry, context);
results.push(dryRunResult);
switch (dryRunResult.action) {
case "recover":
report.recovered.push(assessment.sessionId);
break;
case "cleanup":
report.cleanedUp.push(assessment.sessionId);
break;
case "escalate":
report.escalated.push(assessment.sessionId);
break;
case "skip":
default:
report.skipped.push(assessment.sessionId);
break;
}
continue;
}
const result = await executeAction(assessment, config, registry, context);
results.push(result);
if (result.success) {
switch (result.action) {
case "recover":
report.recovered.push(assessment.sessionId);
if (result.session) {
recoveredSessions.push(result.session);
}
break;
case "cleanup":
report.cleanedUp.push(assessment.sessionId);
break;
case "escalate":
report.escalated.push(assessment.sessionId);
break;
case "skip":
report.skipped.push(assessment.sessionId);
break;
}
} else {
report.errors.push({
sessionId: assessment.sessionId,
error: result.error || "Unknown error",
});
}
const logAction = mapActionToLogAction(result.action, result.success);
writeRecoveryLog(
recoveryConfig.logPath,
createLogEntry(assessment.sessionId, logAction, {
previousStatus: assessment.metadataStatus,
reason: result.reason ?? assessment.reason,
error: result.error,
}),
);
}
report.durationMs = Date.now() - startTime;
return {
report,
assessments,
results,
recoveredSessions,
};
}
function mapActionToLogAction(
action: string,
success: boolean,
): "recovered" | "cleaned_up" | "escalated" | "skipped" | "error" {
if (!success) return "error";
switch (action) {
case "recover":
return "recovered";
case "cleanup":
return "cleaned_up";
case "escalate":
return "escalated";
default:
return "skipped";
}
}
export async function recoverSessionById(
sessionId: string,
options: RecoveryManagerOptions,
): Promise<RecoveryResult | null> {
const { config, registry, dryRun = false } = options;
const recoveryConfig: RecoveryConfig = {
...DEFAULT_RECOVERY_CONFIG,
...options.recoveryConfig,
logPath: options.recoveryConfig?.logPath ?? getRecoveryLogPath(config.configPath),
};
const context: RecoveryContext = {
configPath: config.configPath,
recoveryConfig,
dryRun,
};
const allSessions = scanAllSessions(config);
const scanned = allSessions.find((s) => s.sessionId === sessionId);
if (!scanned) {
return null;
}
const assessment = await validateSession(scanned, config, registry, recoveryConfig);
const result = await executeAction(assessment, config, registry, context);
if (dryRun) {
return result;
}
const logAction = mapActionToLogAction(result.action, result.success);
writeRecoveryLog(
recoveryConfig.logPath,
createLogEntry(sessionId, logAction, {
previousStatus: assessment.metadataStatus,
reason: result.reason ?? assessment.reason,
error: result.error,
}),
);
return result;
}

View File

@ -0,0 +1,48 @@
import { existsSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import type { SessionId, OrchestratorConfig, ProjectConfig } from "../types.js";
import { listMetadata, readMetadataRaw } from "../metadata.js";
import { getSessionsDir, generateConfigHash } from "../paths.js";
export interface ScannedSession {
sessionId: SessionId;
projectId: string;
project: ProjectConfig;
sessionsDir: string;
rawMetadata: Record<string, string>;
}
export function scanAllSessions(
config: OrchestratorConfig,
projectIdFilter?: string,
): ScannedSession[] {
const results: ScannedSession[] = [];
for (const [projectKey, project] of Object.entries(config.projects)) {
if (projectIdFilter && projectKey !== projectIdFilter) continue;
const sessionsDir = getSessionsDir(config.configPath, project.path);
if (!existsSync(sessionsDir)) continue;
for (const file of listMetadata(sessionsDir)) {
const rawMetadata = readMetadataRaw(sessionsDir, file);
if (!rawMetadata) continue;
results.push({
sessionId: file,
projectId: projectKey,
project,
sessionsDir,
rawMetadata,
});
}
}
return results;
}
export function getRecoveryLogPath(configPath: string): string {
const hash = generateConfigHash(configPath);
return join(homedir(), ".agent-orchestrator", hash, "recovery.log");
}

View File

@ -0,0 +1,216 @@
/**
* Recovery Types Session classification and recovery result types.
*
* Part of the orchestrator recovery automation system (Issue #356).
* These types define how sessions are classified and what actions are taken.
*/
import type { SessionId, SessionStatus, Session, RuntimeHandle, ActivityState } from "../types.js";
/**
* Classification of a session's recoverability state.
*
* - live: Session is running normally, just needs to be re-registered
* - dead: Runtime is gone, needs cleanup
* - partial: Some components exist but session is incomplete
* - unrecoverable: Session is in a terminal state (merged, done) - skip
*/
export type RecoveryClassification = "live" | "dead" | "partial" | "unrecoverable";
/**
* Action to take for a session during recovery.
*
* - recover: Restore session to in-memory state
* - cleanup: Remove runtime/workspace, archive metadata
* - escalate: Requires manual intervention
* - skip: No action needed
*/
export type RecoveryAction = "recover" | "cleanup" | "escalate" | "skip";
/**
* Assessment of a session's state for recovery purposes.
*/
export interface RecoveryAssessment {
/** Session ID being assessed */
sessionId: SessionId;
/** Project ID this session belongs to */
projectId: string;
/** Overall classification */
classification: RecoveryClassification;
/** Recommended action */
action: RecoveryAction;
/** Human-readable reason for classification */
reason: string;
// --- Runtime state ---
/** Whether the runtime (tmux/docker) is alive */
runtimeAlive: boolean;
/** Runtime handle if available */
runtimeHandle: RuntimeHandle | null;
// --- Workspace state ---
/** Whether the workspace directory exists */
workspaceExists: boolean;
/** Workspace path if known */
workspacePath: string | null;
// --- Agent state ---
/** Whether the agent process appears to be running */
agentProcessRunning: boolean;
/** Detected agent activity state */
agentActivity: ActivityState | null;
// --- Metadata state ---
/** Whether metadata file is valid/readable */
metadataValid: boolean;
/** Current status from metadata */
metadataStatus: SessionStatus;
/** Raw metadata key-value pairs */
rawMetadata: Record<string, string>;
}
/**
* Result of attempting to recover a single session.
*/
export interface RecoveryResult {
/** Whether the operation succeeded */
success: boolean;
/** Session ID that was processed */
sessionId: SessionId;
/** Action that was taken */
action: RecoveryAction;
/** Recovered session object (only for 'recover' action) */
session?: Session;
/** Whether manual intervention is required */
requiresManualIntervention?: boolean;
reason?: string;
/** Error message if operation failed */
error?: string;
}
/**
* Summary report of a recovery operation.
*/
export interface RecoveryReport {
/** When the recovery was run */
timestamp: Date;
/** Total sessions scanned */
totalScanned: number;
/** Sessions that were recovered (live, restored to memory) */
recovered: SessionId[];
/** Sessions that were cleaned up (dead, resources removed) */
cleanedUp: SessionId[];
/** Sessions that require manual intervention */
escalated: SessionId[];
/** Sessions that were skipped (no action needed) */
skipped: SessionId[];
/** Errors encountered during recovery */
errors: Array<{ sessionId: SessionId; error: string }>;
/** Time taken for recovery in milliseconds */
durationMs: number;
}
/**
* Entry in the recovery log.
*/
export interface RecoveryLogEntry {
/** ISO timestamp */
timestamp: string;
/** Session ID */
sessionId: SessionId;
/** Action taken */
action: "recovered" | "cleaned_up" | "escalated" | "skipped" | "error";
/** Previous status (for recovered sessions) */
previousStatus?: SessionStatus;
/** Reason for action (for cleanup/escalate) */
reason?: string;
/** Error message (for error action) */
error?: string;
/** Additional details */
details?: Record<string, unknown>;
}
/**
* Configuration for recovery behavior.
*/
export interface RecoveryConfig {
/** Enable automatic recovery on orchestrator startup */
enabled: boolean;
/** Maximum time for recovery phase in milliseconds */
timeoutMs: number;
/** Number of concurrent validation tasks */
parallelValidation: number;
/** Path to recovery log file */
logPath: string;
/** Automatically cleanup dead sessions */
autoCleanup: boolean;
/** Escalate partial sessions (vs. auto-cleanup) */
escalatePartial: boolean;
/** Maximum recovery attempts before escalating */
maxRecoveryAttempts: number;
}
/**
* Default recovery configuration.
*/
export const DEFAULT_RECOVERY_CONFIG: RecoveryConfig = {
enabled: true,
timeoutMs: 30_000,
parallelValidation: 5,
logPath: "", // Will be set to ~/.agent-orchestrator/recovery.log
autoCleanup: true,
escalatePartial: true,
maxRecoveryAttempts: 3,
};
/**
* Context passed to recovery functions.
*/
export interface RecoveryContext {
/** Root config path for the orchestrator */
configPath: string;
/** Recovery configuration */
recoveryConfig: RecoveryConfig;
/** Whether this is a dry run (no actual changes) */
dryRun: boolean;
}

View File

@ -0,0 +1,181 @@
import { existsSync } from "node:fs";
import {
TERMINAL_STATUSES as TERMINAL_STATUSES_SET,
type OrchestratorConfig,
type PluginRegistry,
type Runtime,
type Agent,
type Workspace,
type RuntimeHandle,
type SessionStatus,
type ActivityState,
} from "../types.js";
import { safeJsonParse, validateStatus } from "../utils/validation.js";
import type { ScannedSession } from "./scanner.js";
import {
DEFAULT_RECOVERY_CONFIG,
type RecoveryAssessment,
type RecoveryClassification,
type RecoveryAction,
type RecoveryConfig,
} from "./types.js";
export async function validateSession(
scanned: ScannedSession,
config: OrchestratorConfig,
registry: PluginRegistry,
recoveryConfigInput?: Partial<RecoveryConfig>,
): Promise<RecoveryAssessment> {
const { sessionId, projectId, project, rawMetadata } = scanned;
const runtimeName = project.runtime ?? config.defaults.runtime;
const agentName = rawMetadata["agent"] ?? project.agent ?? config.defaults.agent;
const workspaceName = project.workspace ?? config.defaults.workspace;
const runtime = registry.get<Runtime>("runtime", runtimeName);
const agent = registry.get<Agent>("agent", agentName);
const workspace = registry.get<Workspace>("workspace", workspaceName);
const workspacePath = rawMetadata["worktree"] || null;
const runtimeHandleStr = rawMetadata["runtimeHandle"];
const runtimeHandle = runtimeHandleStr ? safeJsonParse<RuntimeHandle>(runtimeHandleStr) : null;
const metadataStatus = validateStatus(rawMetadata["status"]);
const recoveryConfig: RecoveryConfig = {
...DEFAULT_RECOVERY_CONFIG,
...(recoveryConfigInput ?? {}),
};
let runtimeAlive = false;
if (runtime && runtimeHandle) {
try {
runtimeAlive = await runtime.isAlive(runtimeHandle);
} catch {
runtimeAlive = false;
}
}
let workspaceExists = false;
if (workspacePath) {
try {
workspaceExists = existsSync(workspacePath);
} catch {
workspaceExists = false;
}
if (!workspaceExists && workspace?.exists) {
try {
workspaceExists = await workspace.exists(workspacePath);
} catch {
workspaceExists = false;
}
}
}
let agentProcessRunning = false;
const agentActivity: ActivityState | null = null;
if (agent && runtimeHandle) {
try {
agentProcessRunning = await agent.isProcessRunning(runtimeHandle);
} catch {
agentProcessRunning = false;
}
}
const metadataValid = Object.keys(rawMetadata).length > 0;
const classification = classifySession(
runtimeAlive,
workspaceExists,
agentProcessRunning,
metadataStatus,
);
const action = determineAction(classification, metadataStatus, recoveryConfig);
return {
sessionId,
projectId,
classification,
action,
reason: getReason(classification, runtimeAlive, workspaceExists, agentProcessRunning),
runtimeAlive,
runtimeHandle,
workspaceExists,
workspacePath,
agentProcessRunning,
agentActivity,
metadataValid,
metadataStatus,
rawMetadata,
};
}
function classifySession(
runtimeAlive: boolean,
workspaceExists: boolean,
agentProcessRunning: boolean,
metadataStatus: SessionStatus,
): RecoveryClassification {
if (runtimeAlive && workspaceExists && agentProcessRunning) {
return "live";
}
if (!runtimeAlive && !workspaceExists) {
if (TERMINAL_STATUSES_SET.has(metadataStatus)) {
return "unrecoverable";
}
return "dead";
}
if (runtimeAlive && !workspaceExists) {
return "partial";
}
if (!runtimeAlive && workspaceExists) {
return "dead";
}
if (runtimeAlive && workspaceExists && !agentProcessRunning) {
return "partial";
}
return "partial";
}
function determineAction(
classification: RecoveryClassification,
_metadataStatus: SessionStatus,
recoveryConfig: RecoveryConfig,
): RecoveryAction {
switch (classification) {
case "live":
return "recover";
case "dead":
return recoveryConfig.autoCleanup ? "cleanup" : "escalate";
case "partial":
return recoveryConfig.escalatePartial ? "escalate" : "cleanup";
case "unrecoverable":
return "skip";
default:
return "skip";
}
}
function getReason(
classification: RecoveryClassification,
runtimeAlive: boolean,
workspaceExists: boolean,
agentProcessRunning: boolean,
): string {
switch (classification) {
case "live":
return "Session is running normally";
case "dead":
return `Runtime ${runtimeAlive ? "alive" : "dead"}, workspace ${workspaceExists ? "exists" : "missing"}`;
case "partial":
return `Incomplete state: runtime=${runtimeAlive}, workspace=${workspaceExists}, agent=${agentProcessRunning}`;
case "unrecoverable":
return "Session is in terminal state";
default:
return "Unknown classification";
}
}
export { classifySession, determineAction };

View File

@ -28,7 +28,6 @@ import {
type SessionId,
type SessionSpawnConfig,
type OrchestratorSpawnConfig,
type SessionStatus,
type CleanupResult,
type ClaimPROptions,
type ClaimPRResult,
@ -71,6 +70,8 @@ import {
GLOBAL_PAUSE_SOURCE_KEY,
parsePauseUntil,
} from "./global-pause.js";
import { sessionFromMetadata } from "./utils/session-from-metadata.js";
import { safeJsonParse } from "./utils/validation.js";
const execFileAsync = promisify(execFile);
const OPENCODE_DISCOVERY_TIMEOUT_MS = 2_000;
@ -191,35 +192,6 @@ function getNextSessionNumber(existingSessions: string[], prefix: string): numbe
return max + 1;
}
/** Safely parse JSON, returning null on failure. */
function safeJsonParse<T>(str: string): T | null {
try {
return JSON.parse(str) as T;
} catch {
return null;
}
}
/** Valid session statuses for validation. */
const VALID_STATUSES: ReadonlySet<string> = new Set([
"spawning",
"working",
"pr_open",
"ci_failed",
"review_pending",
"changes_requested",
"approved",
"mergeable",
"merged",
"cleanup",
"needs_input",
"stuck",
"errored",
"killed",
"done",
"terminated",
]);
const PR_TRACKING_STATUSES: ReadonlySet<string> = new Set([
"pr_open",
"ci_failed",
@ -239,14 +211,6 @@ function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Validate and normalize a status string. */
function validateStatus(raw: string | undefined): SessionStatus {
// Bash scripts write "starting" — treat as "working"
if (raw === "starting") return "working";
if (raw && VALID_STATUSES.has(raw)) return raw as SessionStatus;
return "spawning";
}
/** Reconstruct a Session object from raw metadata key=value pairs. */
function metadataToSession(
sessionId: SessionId,
@ -254,42 +218,10 @@ function metadataToSession(
createdAt?: Date,
modifiedAt?: Date,
): Session {
return {
id: sessionId,
projectId: meta["project"] ?? "",
status: validateStatus(meta["status"]),
activity: null,
branch: meta["branch"] || null,
issueId: meta["issue"] || null,
pr: meta["pr"]
? (() => {
// Parse owner/repo from GitHub PR URL: https://github.com/owner/repo/pull/123
const prUrl = meta["pr"];
const ghMatch = prUrl.match(/github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/);
return {
number: ghMatch
? parseInt(ghMatch[3], 10)
: parseInt(prUrl.match(/\/(\d+)$/)?.[1] ?? "0", 10),
url: prUrl,
title: "",
owner: ghMatch?.[1] ?? "",
repo: ghMatch?.[2] ?? "",
branch: meta["branch"] ?? "",
baseBranch: "",
isDraft: false,
};
})()
: null,
workspacePath: meta["worktree"] || null,
runtimeHandle: meta["runtimeHandle"]
? safeJsonParse<RuntimeHandle>(meta["runtimeHandle"])
: null,
agentInfo: meta["summary"] ? { summary: meta["summary"], agentSessionId: null } : null,
createdAt: meta["createdAt"] ? new Date(meta["createdAt"]) : (createdAt ?? new Date()),
return sessionFromMetadata(sessionId, meta, {
createdAt,
lastActivityAt: modifiedAt ?? new Date(),
restoredAt: meta["restoredAt"] ? new Date(meta["restoredAt"]) : undefined,
metadata: meta,
};
});
}
export interface SessionManagerDeps {

View File

@ -0,0 +1,31 @@
import type { PRInfo } from "../types.js";
export type ParsedPrUrl = Pick<PRInfo, "owner" | "repo" | "number" | "url">;
const GITHUB_PR_URL_REGEX = /github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/;
const TRAILING_NUMBER_REGEX = /\/(\d+)$/;
export function parsePrFromUrl(prUrl: string): ParsedPrUrl | null {
const githubMatch = prUrl.match(GITHUB_PR_URL_REGEX);
if (githubMatch) {
const [, owner, repo, prNumber] = githubMatch;
return {
owner,
repo,
number: parseInt(prNumber, 10),
url: prUrl,
};
}
const trailingNumberMatch = prUrl.match(TRAILING_NUMBER_REGEX);
if (trailingNumberMatch) {
return {
owner: "",
repo: "",
number: parseInt(trailingNumberMatch[1], 10),
url: prUrl,
};
}
return null;
}

View File

@ -0,0 +1,55 @@
import type { RuntimeHandle, Session, SessionId, SessionStatus } from "../types.js";
import { parsePrFromUrl } from "./pr.js";
import { safeJsonParse, validateStatus } from "./validation.js";
interface SessionFromMetadataOptions {
status?: SessionStatus;
activity?: Session["activity"];
runtimeHandle?: RuntimeHandle | null;
createdAt?: Date;
lastActivityAt?: Date;
restoredAt?: Date;
}
export function sessionFromMetadata(
sessionId: SessionId,
meta: Record<string, string>,
options: SessionFromMetadataOptions = {},
): Session {
return {
id: sessionId,
projectId: meta["project"] ?? "",
status: options.status ?? validateStatus(meta["status"]),
activity: options.activity ?? null,
branch: meta["branch"] || null,
issueId: meta["issue"] || null,
pr: meta["pr"]
? (() => {
const parsed = parsePrFromUrl(meta["pr"]);
return {
number: parsed?.number ?? 0,
url: meta["pr"],
title: "",
owner: parsed?.owner ?? "",
repo: parsed?.repo ?? "",
branch: meta["branch"] ?? "",
baseBranch: "",
isDraft: false,
};
})()
: null,
workspacePath: meta["worktree"] || null,
runtimeHandle:
options.runtimeHandle !== undefined
? options.runtimeHandle
: meta["runtimeHandle"]
? safeJsonParse<RuntimeHandle>(meta["runtimeHandle"])
: null,
agentInfo: meta["summary"] ? { summary: meta["summary"], agentSessionId: null } : null,
createdAt: meta["createdAt"] ? new Date(meta["createdAt"]) : (options.createdAt ?? new Date()),
lastActivityAt: options.lastActivityAt ?? new Date(),
restoredAt:
options.restoredAt ?? (meta["restoredAt"] ? new Date(meta["restoredAt"]) : undefined),
metadata: meta,
};
}

View File

@ -0,0 +1,42 @@
/**
* Shared validation utilities.
*/
import type { SessionStatus } from "../types.js";
/** Valid session statuses for validation. */
const VALID_STATUSES: ReadonlySet<string> = new Set([
"spawning",
"working",
"pr_open",
"ci_failed",
"review_pending",
"changes_requested",
"approved",
"mergeable",
"merged",
"cleanup",
"needs_input",
"stuck",
"errored",
"killed",
"done",
"terminated",
]);
/** Safely parse JSON, returning null on failure. */
export function safeJsonParse<T>(str: string): T | null {
try {
return JSON.parse(str) as T;
} catch {
return null;
}
}
/** Validate and normalize a status string. */
export function validateStatus(raw: string | undefined): SessionStatus {
// Bash scripts write "starting" — treat as "working"
if (raw === "starting") return "working";
if (raw && VALID_STATUSES.has(raw)) return raw as SessionStatus;
return "spawning";
}