diff --git a/.changeset/fix-start-restore-and-codex-resume.md b/.changeset/fix-start-restore-and-codex-resume.md new file mode 100644 index 000000000..84e465c47 --- /dev/null +++ b/.changeset/fix-start-restore-and-codex-resume.md @@ -0,0 +1,10 @@ +"@aoagents/ao-cli": patch +"@aoagents/ao-web": patch +"@aoagents/ao-plugin-agent-codex": patch +--- + +Fix restore behavior across AO session recovery flows. + +- restore the latest dead-but-restorable orchestrator on `ao start` instead of silently spawning a new orchestrator when tmux is gone +- make worker session orchestrator navigation prefer the most recently active live orchestrator for the project +- make permissionless Codex restores preserve dangerous bypass semantics so resumed workers behave like fresh permissionless launches diff --git a/packages/cli/__tests__/commands/start.test.ts b/packages/cli/__tests__/commands/start.test.ts index 1eab54ec4..cd78d8e8c 100644 --- a/packages/cli/__tests__/commands/start.test.ts +++ b/packages/cli/__tests__/commands/start.test.ts @@ -30,12 +30,12 @@ const { mockConfigRef: { current: null as Record | null }, mockSessionManager: { list: vi.fn(), + restore: vi.fn(), kill: vi.fn(), cleanup: vi.fn(), get: vi.fn(), spawn: vi.fn(), spawnOrchestrator: vi.fn(), - restore: vi.fn(), send: vi.fn(), claimPR: vi.fn(), }, @@ -57,8 +57,10 @@ const { mockPromptSelect, mockPromptConfirm } = vi.hoisted(() => ({ mockPromptConfirm: vi.fn().mockResolvedValue(true), })); -const { mockIsAlreadyRunning, mockUnregister, mockWaitForExit } = vi.hoisted(() => ({ +const { mockAcquireStartupLock, mockIsAlreadyRunning, mockRegister, mockUnregister, mockWaitForExit } = vi.hoisted(() => ({ + mockAcquireStartupLock: vi.fn().mockResolvedValue(() => {}), mockIsAlreadyRunning: vi.fn().mockReturnValue(null), + mockRegister: vi.fn(), mockUnregister: vi.fn(), mockWaitForExit: vi.fn().mockReturnValue(true), })); @@ -142,7 +144,8 @@ vi.mock("../../src/lib/preflight.js", () => ({ })); vi.mock("../../src/lib/running-state.js", () => ({ - register: vi.fn(), + acquireStartupLock: (...args: unknown[]) => mockAcquireStartupLock(...args), + register: (...args: unknown[]) => mockRegister(...args), unregister: (...args: unknown[]) => mockUnregister(...args), isAlreadyRunning: (...args: unknown[]) => mockIsAlreadyRunning(...args), getRunning: vi.fn().mockReturnValue(null), @@ -247,9 +250,10 @@ beforeEach(async () => { mockSessionManager.list.mockReset(); mockSessionManager.list.mockResolvedValue([]); + mockSessionManager.restore.mockReset(); + mockSessionManager.restore.mockResolvedValue({ id: "app-orchestrator-restored" }); mockSessionManager.get.mockReset(); mockSessionManager.spawnOrchestrator.mockReset(); - mockSessionManager.restore.mockReset(); mockSessionManager.kill.mockReset(); mockExec.mockReset(); mockExecSilent.mockReset(); @@ -281,8 +285,12 @@ beforeEach(async () => { mockPromptSelect.mockReset(); mockPromptConfirm.mockReset(); mockPromptConfirm.mockResolvedValue(true); + mockAcquireStartupLock.mockReset(); + mockAcquireStartupLock.mockResolvedValue(() => {}); mockIsAlreadyRunning.mockReset(); mockIsAlreadyRunning.mockResolvedValue(null); + mockRegister.mockReset(); + mockRegister.mockResolvedValue(undefined); mockUnregister.mockReset(); mockWaitForExit.mockReset(); mockWaitForExit.mockResolvedValue(true); @@ -953,6 +961,93 @@ describe("start command — orchestrator session strategy display", () => { expect(mockSessionManager.spawnOrchestrator).not.toHaveBeenCalled(); }); + it("restores the latest restorable orchestrator when tmux is gone", async () => { + mockConfigRef.current = makeConfig({ "my-app": makeProject() }); + + const now = new Date(); + mockSessionManager.list.mockResolvedValue([ + { + id: "app-orchestrator-1", + projectId: "my-app", + status: "killed", + activity: "exited", + metadata: { role: "orchestrator" }, + lastActivityAt: new Date(now.getTime() - 1000), + lifecycle: { + version: 2, + session: { + kind: "orchestrator", + state: "working", + reason: "task_in_progress", + startedAt: now.toISOString(), + completedAt: null, + terminatedAt: null, + lastTransitionAt: now.toISOString(), + }, + pr: { + state: "none", + reason: "not_created", + number: null, + url: null, + lastObservedAt: null, + }, + runtime: { + state: "missing", + reason: "tmux_missing", + lastObservedAt: now.toISOString(), + handle: null, + tmuxName: "tmux-old-1", + }, + }, + }, + { + id: "app-orchestrator-2", + projectId: "my-app", + status: "killed", + activity: "exited", + metadata: { role: "orchestrator" }, + lastActivityAt: now, + lifecycle: { + version: 2, + session: { + kind: "orchestrator", + state: "working", + reason: "task_in_progress", + startedAt: now.toISOString(), + completedAt: null, + terminatedAt: null, + lastTransitionAt: now.toISOString(), + }, + pr: { + state: "none", + reason: "not_created", + number: null, + url: null, + lastObservedAt: null, + }, + runtime: { + state: "missing", + reason: "tmux_missing", + lastObservedAt: now.toISOString(), + handle: null, + tmuxName: "tmux-old-2", + }, + }, + }, + ]); + mockSessionManager.restore.mockResolvedValue({ + id: "app-orchestrator-2", + runtimeHandle: { id: "tmux-restored-2" }, + }); + + await program.parseAsync(["node", "test", "start", "--no-dashboard"]); + + const output = getLoggedOutput(); + expect(output).toContain("ao session attach app-orchestrator-2"); + expect(mockSessionManager.restore).toHaveBeenCalledWith("app-orchestrator-2"); + expect(mockSessionManager.spawnOrchestrator).not.toHaveBeenCalled(); + }); + it("navigates directly to session page when one existing orchestrator found with dashboard enabled", async () => { mockConfigRef.current = makeConfig({ "my-app": makeProject() }); @@ -1232,6 +1327,34 @@ describe("start command — orchestrator session strategy display", () => { expect(fakeDashboard.kill).toHaveBeenCalled(); }); + it("reports startup lock acquisition failures through the normal CLI error path", async () => { + mockConfigRef.current = makeConfig({ "my-app": makeProject() }); + mockAcquireStartupLock.mockRejectedValueOnce( + new Error("Could not acquire startup lock (/tmp/startup.lock)"), + ); + + await expect(program.parseAsync(["node", "test", "start"])).rejects.toThrow("process.exit(1)"); + + const errors = vi + .mocked(console.error) + .mock.calls.map((c) => c.join(" ")) + .join("\n"); + expect(errors).toContain("Could not acquire startup lock (/tmp/startup.lock)"); + expect(mockIsAlreadyRunning).not.toHaveBeenCalled(); + }); + + it("releases the startup lock before exiting on startup failures", async () => { + mockConfigRef.current = makeConfig({ "my-app": makeProject() }); + const releaseStartupLock = vi.fn(); + mockAcquireStartupLock.mockResolvedValueOnce(releaseStartupLock); + mockSessionManager.list.mockResolvedValue([]); + mockSessionManager.spawnOrchestrator.mockRejectedValue(new Error("Spawn failed")); + + await expect(program.parseAsync(["node", "test", "start"])).rejects.toThrow("process.exit(1)"); + + expect(releaseStartupLock).toHaveBeenCalledTimes(1); + }); + it("fails and cleans up dashboard when sm.restore throws on a killed orchestrator", async () => { mockConfigRef.current = makeConfig({ "my-app": makeProject() }); diff --git a/packages/cli/__tests__/lib/running-state.test.ts b/packages/cli/__tests__/lib/running-state.test.ts new file mode 100644 index 000000000..f3b2f6319 --- /dev/null +++ b/packages/cli/__tests__/lib/running-state.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { existsSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +const testHome = join(process.cwd(), ".tmp-running-state-home"); + +vi.mock("node:os", () => ({ + homedir: () => testHome, +})); + +describe("running-state", () => { + beforeEach(() => { + rmSync(testHome, { recursive: true, force: true }); + vi.restoreAllMocks(); + vi.resetModules(); + }); + + afterEach(() => { + rmSync(testHome, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it("keeps running.json when the pid probe returns EPERM", async () => { + const runningState = await import("../../src/lib/running-state.js"); + const killSpy = vi.spyOn(process, "kill").mockImplementation(() => { + const error = new Error("operation not permitted") as Error & { code?: string }; + error.code = "EPERM"; + throw error; + }); + + await runningState.register({ + pid: 424242, + configPath: "/tmp/agent-orchestrator.yaml", + port: 4321, + startedAt: new Date("2026-04-19T00:00:00.000Z").toISOString(), + projects: ["my-app"], + }); + + const state = await runningState.getRunning(); + const stateFile = join(testHome, ".agent-orchestrator", "running.json"); + + expect(state).toEqual({ + pid: 424242, + configPath: "/tmp/agent-orchestrator.yaml", + port: 4321, + startedAt: new Date("2026-04-19T00:00:00.000Z").toISOString(), + projects: ["my-app"], + }); + expect(existsSync(stateFile)).toBe(true); + expect(killSpy).toHaveBeenCalledWith(424242, 0); + }); + + it("keeps startup locks alive when the pid probe returns EPERM", async () => { + const runningState = await import("../../src/lib/running-state.js"); + const lockDir = join(testHome, ".agent-orchestrator"); + const lockFile = join(lockDir, "startup.lock"); + const killSpy = vi.spyOn(process, "kill").mockImplementation(() => { + const error = new Error("operation not permitted") as Error & { code?: string }; + error.code = "EPERM"; + throw error; + }); + + const release = await runningState.acquireStartupLock(100); + + await expect(runningState.acquireStartupLock(100)).rejects.toThrow( + `Could not acquire startup lock (${lockFile})`, + ); + expect(readFileSync(lockFile, "utf-8")).toContain(`"pid":${process.pid}`); + + release(); + expect(killSpy).toHaveBeenCalledWith(process.pid, 0); + }); +}); diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 25de264e9..fa6d5b97b 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -52,7 +52,14 @@ import { } from "../lib/web-dir.js"; import { rebuildDashboardProductionArtifacts } from "../lib/dashboard-rebuild.js"; import { preflight } from "../lib/preflight.js"; -import { register, unregister, isAlreadyRunning, getRunning, waitForExit } from "../lib/running-state.js"; +import { + register, + unregister, + isAlreadyRunning, + getRunning, + waitForExit, + acquireStartupLock, +} from "../lib/running-state.js"; import { preventIdleSleep } from "../lib/prevent-sleep.js"; import { isHumanCaller } from "../lib/caller-context.js"; import { detectEnvironment } from "../lib/detect-env.js"; @@ -1379,7 +1386,16 @@ export function registerStart(program: Command): void { interactive?: boolean; }, ) => { + let releaseStartupLock: (() => void) | undefined; + let startupLockReleased = false; + const unlockStartup = (): void => { + if (startupLockReleased || !releaseStartupLock) return; + startupLockReleased = true; + releaseStartupLock(); + }; + try { + releaseStartupLock = await acquireStartupLock(); let config: OrchestratorConfig; let projectId: string; let project: ProjectConfig; @@ -1408,6 +1424,7 @@ export function registerStart(program: Command): void { if (choice === "open") { const url = `http://localhost:${running.port}`; openUrl(url); + unlockStartup(); process.exit(0); } else if (choice === "new") { // Defer config mutation until after config is loaded below @@ -1427,6 +1444,7 @@ export function registerStart(program: Command): void { console.log(chalk.yellow("\n Stopped existing instance. Restarting...\n")); // Continue to startup below } else { + unlockStartup(); process.exit(0); } } else { @@ -1436,6 +1454,7 @@ export function registerStart(program: Command): void { console.log(`PID: ${running.pid}`); console.log(`Projects: ${running.projects.join(", ")}`); console.log(`To restart: ao stop && ao start`); + unlockStartup(); process.exit(0); } } @@ -1565,6 +1584,7 @@ export function registerStart(program: Command): void { startedAt: new Date().toISOString(), projects: [projectId], }); + unlockStartup(); // Install shutdown handlers so `ao stop` (which sends SIGTERM to // this pid) flushes lifecycle health state before exit. Handlers @@ -1590,7 +1610,10 @@ export function registerStart(program: Command): void { } else { console.error(chalk.red("\nError:"), String(err)); } + unlockStartup(); process.exit(1); + } finally { + unlockStartup(); } }, ); diff --git a/packages/cli/src/lib/running-state.ts b/packages/cli/src/lib/running-state.ts index 666a64780..6e51911c2 100644 --- a/packages/cli/src/lib/running-state.ts +++ b/packages/cli/src/lib/running-state.ts @@ -1,4 +1,13 @@ -import { readFileSync, writeFileSync, mkdirSync, unlinkSync, openSync, closeSync, constants } from "node:fs"; +import { + readFileSync, + writeFileSync, + mkdirSync, + unlinkSync, + openSync, + closeSync, + constants, + statSync, +} from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; import { setTimeout as sleep } from "node:timers/promises"; @@ -13,28 +22,83 @@ export interface RunningState { const STATE_DIR = join(homedir(), ".agent-orchestrator"); const STATE_FILE = join(STATE_DIR, "running.json"); -const LOCK_FILE = join(STATE_DIR, "running.lock"); +const STATE_LOCK_FILE = join(STATE_DIR, "running.lock"); +const STARTUP_LOCK_FILE = join(STATE_DIR, "startup.lock"); +const UNPARSEABLE_LOCK_GRACE_MS = 5_000; + +interface LockMetadata { + pid: number; + acquiredAt: string; +} + +type ProcessProbeResult = "alive" | "forbidden" | "missing"; function ensureDir(): void { mkdirSync(STATE_DIR, { recursive: true }); } -function isProcessAlive(pid: number): boolean { +function probeProcess(pid: number): ProcessProbeResult { try { process.kill(pid, 0); - return true; + return "alive"; + } catch (error: unknown) { + if ((error as { code?: string }).code === "EPERM") { + return "forbidden"; + } + return "missing"; + } +} + +function isLockOwnerAlive(pid: number): boolean { + return probeProcess(pid) !== "missing"; +} + +function isRunningProcessAlive(pid: number): boolean { + return probeProcess(pid) !== "missing"; +} + +function readLockMetadata(lockFile: string): LockMetadata | null { + try { + const raw = readFileSync(lockFile, "utf-8"); + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed.pid !== "number") return null; + return { + pid: parsed.pid, + acquiredAt: + typeof parsed.acquiredAt === "string" ? parsed.acquiredAt : new Date(0).toISOString(), + }; + } catch { + return null; + } +} + +function isStaleUnparseableLock(lockFile: string): boolean { + try { + const mtimeMs = statSync(lockFile).mtimeMs; + return Date.now() - mtimeMs > UNPARSEABLE_LOCK_GRACE_MS; } catch { return false; } } /** Try to create the lockfile atomically. Returns a release function on success, null on failure. */ -function tryAcquire(): (() => void) | null { +function tryAcquire(lockFile: string): (() => void) | null { try { - const fd = openSync(LOCK_FILE, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY); - closeSync(fd); + const fd = openSync(lockFile, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY); + const metadata: LockMetadata = { + pid: process.pid, + acquiredAt: new Date().toISOString(), + }; + try { + writeFileSync(fd, JSON.stringify(metadata), "utf-8"); + } catch { + try { unlinkSync(lockFile); } catch { /* best effort */ } + return null; + } finally { + try { closeSync(fd); } catch { /* best effort */ } + } return () => { - try { unlinkSync(LOCK_FILE); } catch { /* best effort */ } + try { unlinkSync(lockFile); } catch { /* best effort */ } }; } catch { return null; @@ -43,25 +107,33 @@ function tryAcquire(): (() => void) | null { /** * Advisory lockfile using O_EXCL for atomic creation. - * Retries with jittered backoff. After timeout, assumes the lock is stale - * (holder crashed) and force-removes it before one final atomic attempt. + * Retries with jittered backoff. Dead owners are treated as stale and cleaned + * up automatically. Live owners are never stolen; callers get a clear timeout. */ -async function acquireLock(timeoutMs = 5000): Promise<() => void> { +async function acquireLock( + lockFile: string, + timeoutMs = 5000, + resourceName = "lock", +): Promise<() => void> { ensureDir(); const start = Date.now(); let attempt = 0; while (true) { - const release = tryAcquire(); + const release = tryAcquire(lockFile); if (release) return release; + const owner = readLockMetadata(lockFile); + if ((!owner && isStaleUnparseableLock(lockFile)) + || (owner && !isLockOwnerAlive(owner.pid))) { + try { unlinkSync(lockFile); } catch { /* ignore */ } + const retryRelease = tryAcquire(lockFile); + if (retryRelease) return retryRelease; + } + if (Date.now() - start > timeoutMs) { - // Likely stale — remove and make one final atomic attempt. - try { unlinkSync(LOCK_FILE); } catch { /* ignore */ } - const finalRelease = tryAcquire(); - if (finalRelease) return finalRelease; - throw new Error("Could not acquire running.json lock"); + throw new Error(`Could not acquire ${resourceName} (${lockFile})`); } // Jittered backoff: 30-70ms base, growing with attempts (capped at 200ms) @@ -97,7 +169,7 @@ function writeState(state: RunningState | null): void { * Uses a lockfile to prevent concurrent registration. */ export async function register(entry: RunningState): Promise { - const release = await acquireLock(); + const release = await acquireLock(STATE_LOCK_FILE, 5000, "running.json lock"); try { writeState(entry); } finally { @@ -109,7 +181,7 @@ export async function register(entry: RunningState): Promise { * Unregister the running AO instance. */ export async function unregister(): Promise { - const release = await acquireLock(); + const release = await acquireLock(STATE_LOCK_FILE, 5000, "running.json lock"); try { writeState(null); } finally { @@ -122,12 +194,12 @@ export async function unregister(): Promise { * Auto-prunes stale entries (dead PIDs). */ export async function getRunning(): Promise { - const release = await acquireLock(); + const release = await acquireLock(STATE_LOCK_FILE, 5000, "running.json lock"); try { const state = readState(); if (!state) return null; - if (!isProcessAlive(state.pid)) { + if (!isRunningProcessAlive(state.pid)) { // Stale entry — process is dead, clean up writeState(null); return null; @@ -148,14 +220,22 @@ export async function isAlreadyRunning(): Promise { } /** - * Wait for a process to exit, polling isProcessAlive. + * Serialize `ao start` so concurrent startups cannot both observe an empty + * running.json and create competing orchestrator/dashboard processes. + */ +export async function acquireStartupLock(timeoutMs = 30000): Promise<() => void> { + return await acquireLock(STARTUP_LOCK_FILE, timeoutMs, "startup lock"); +} + +/** + * Wait for a process to exit, polling isRunningProcessAlive. * Returns true if the process exited, false if timeout reached. */ export async function waitForExit(pid: number, timeoutMs = 5000): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { - if (!isProcessAlive(pid)) return true; + if (!isRunningProcessAlive(pid)) return true; await sleep(100); } - return !isProcessAlive(pid); + return !isRunningProcessAlive(pid); } diff --git a/packages/plugins/agent-codex/src/index.test.ts b/packages/plugins/agent-codex/src/index.test.ts index 0b9d2664c..86c2bfabd 100644 --- a/packages/plugins/agent-codex/src/index.test.ts +++ b/packages/plugins/agent-codex/src/index.test.ts @@ -1470,7 +1470,7 @@ describe("getRestoreCommand", () => { expect(cmd).toContain("--dangerously-bypass-approvals-and-sandbox"); }); - it("demotes worker restore permissionless mode to ask-for-approval never", async () => { + it("uses dangerous bypass for worker restore permissionless mode", async () => { const content = jsonl( { type: "session_meta", cwd: "/workspace/test", model: "gpt-4o" }, { threadId: "thread-1" }, @@ -1489,11 +1489,11 @@ describe("getRestoreCommand", () => { }), ); - expect(cmd).toContain("--ask-for-approval never"); - expect(cmd).not.toContain("--dangerously-bypass-approvals-and-sandbox"); + expect(cmd).toContain("--dangerously-bypass-approvals-and-sandbox"); + expect(cmd).not.toContain("--ask-for-approval"); }); - it("includes --ask-for-approval never from project config", async () => { + it("keeps auto-edit restore policy at ask-for-approval never", async () => { const content = jsonl( { type: "session_meta", cwd: "/workspace/test", model: "gpt-4o" }, { threadId: "thread-1" }, diff --git a/packages/plugins/agent-codex/src/index.ts b/packages/plugins/agent-codex/src/index.ts index 4ac3714f9..ccea0b538 100644 --- a/packages/plugins/agent-codex/src/index.ts +++ b/packages/plugins/agent-codex/src/index.ts @@ -733,8 +733,7 @@ function createCodexAgent(): Agent { const parts: string[] = [shellEscape(binary), "resume"]; appendNoUpdateCheckFlag(parts); - const isOrchestrator = session.metadata?.["role"] === "orchestrator"; - appendApprovalFlags(parts, project.agentConfig?.permissions, isOrchestrator); + appendApprovalFlags(parts, project.agentConfig?.permissions); const effectiveModel = (project.agentConfig?.model ?? data.model) as string | undefined; appendModelFlags(parts, effectiveModel ?? undefined); diff --git a/packages/web/src/__tests__/api-routes.test.ts b/packages/web/src/__tests__/api-routes.test.ts index 07b9097c4..b5676ad97 100644 --- a/packages/web/src/__tests__/api-routes.test.ts +++ b/packages/web/src/__tests__/api-routes.test.ts @@ -336,6 +336,153 @@ describe("API Routes", () => { expect(mockSessionManager.listCached).toHaveBeenCalledWith("docs-app"); }); + it("prefers the most recently active live orchestrator for project-scoped worker navigation", async () => { + const deadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T11:00:00.000Z")); + deadLifecycle.session.state = "terminated"; + deadLifecycle.session.reason = "runtime_missing"; + deadLifecycle.session.terminatedAt = "2026-04-19T11:00:00.000Z"; + deadLifecycle.session.lastTransitionAt = "2026-04-19T11:00:00.000Z"; + deadLifecycle.runtime.state = "missing"; + deadLifecycle.runtime.reason = "process_missing"; + deadLifecycle.runtime.lastObservedAt = "2026-04-19T11:00:00.000Z"; + + const olderLive = makeSession({ + id: "my-app-orchestrator-1", + projectId: "my-app", + metadata: { role: "orchestrator" }, + lastActivityAt: new Date("2026-04-19T09:00:00.000Z"), + }); + const newerLive = makeSession({ + id: "my-app-orchestrator-2", + projectId: "my-app", + metadata: { role: "orchestrator" }, + lastActivityAt: new Date("2026-04-19T10:00:00.000Z"), + }); + const deadOlder = makeSession({ + id: "my-app-orchestrator-0", + projectId: "my-app", + metadata: { role: "orchestrator" }, + status: "killed", + activity: "exited", + lastActivityAt: new Date("2026-04-19T11:00:00.000Z"), + lifecycle: deadLifecycle, + }); + (mockSessionManager.listCached as ReturnType).mockResolvedValueOnce([ + deadOlder, + olderLive, + newerLive, + makeSession({ + id: "backend-3", + projectId: "my-app", + status: "working", + activity: "active", + }), + ]); + + const res = await sessionsGET( + makeRequest("http://localhost:3000/api/sessions?project=my-app&orchestratorOnly=true"), + ); + expect(res.status).toBe(200); + const data = await res.json(); + + expect(data.orchestratorId).toBe("my-app-orchestrator-2"); + expect(data.orchestrators.map((session: { id: string }) => session.id)).toEqual([ + "my-app-orchestrator-1", + "my-app-orchestrator-2", + ]); + expect(data.sessions).toEqual([]); + expect(mockSessionManager.listCached).toHaveBeenCalledWith("my-app"); + }); + + it("keeps dead orchestrators as the fallback project-scoped payload when none are live", async () => { + const deadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T11:00:00.000Z")); + deadLifecycle.session.state = "terminated"; + deadLifecycle.session.reason = "runtime_missing"; + deadLifecycle.session.terminatedAt = "2026-04-19T11:00:00.000Z"; + deadLifecycle.session.lastTransitionAt = "2026-04-19T11:00:00.000Z"; + deadLifecycle.runtime.state = "missing"; + deadLifecycle.runtime.reason = "process_missing"; + deadLifecycle.runtime.lastObservedAt = "2026-04-19T11:00:00.000Z"; + + (mockSessionManager.listCached as ReturnType).mockResolvedValueOnce([ + makeSession({ + id: "my-app-orchestrator-0", + projectId: "my-app", + metadata: { role: "orchestrator" }, + status: "killed", + activity: "exited", + lastActivityAt: new Date("2026-04-19T11:00:00.000Z"), + lifecycle: deadLifecycle, + }), + ]); + + const res = await sessionsGET( + makeRequest("http://localhost:3000/api/sessions?project=my-app&orchestratorOnly=true"), + ); + expect(res.status).toBe(200); + const data = await res.json(); + + expect(data.orchestratorId).toBe("my-app-orchestrator-0"); + expect(data.orchestrators).toEqual([ + { id: "my-app-orchestrator-0", projectId: "my-app", projectName: "My App" }, + ]); + expect(data.sessions).toEqual([]); + }); + + it("prefers the most recently active dead orchestrator when no live project orchestrator exists", async () => { + const olderDeadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T10:00:00.000Z")); + olderDeadLifecycle.session.state = "terminated"; + olderDeadLifecycle.session.reason = "runtime_missing"; + olderDeadLifecycle.session.terminatedAt = "2026-04-19T10:00:00.000Z"; + olderDeadLifecycle.session.lastTransitionAt = "2026-04-19T10:00:00.000Z"; + olderDeadLifecycle.runtime.state = "missing"; + olderDeadLifecycle.runtime.reason = "process_missing"; + olderDeadLifecycle.runtime.lastObservedAt = "2026-04-19T10:00:00.000Z"; + + const newerDeadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T11:00:00.000Z")); + newerDeadLifecycle.session.state = "terminated"; + newerDeadLifecycle.session.reason = "runtime_missing"; + newerDeadLifecycle.session.terminatedAt = "2026-04-19T11:00:00.000Z"; + newerDeadLifecycle.session.lastTransitionAt = "2026-04-19T11:00:00.000Z"; + newerDeadLifecycle.runtime.state = "missing"; + newerDeadLifecycle.runtime.reason = "process_missing"; + newerDeadLifecycle.runtime.lastObservedAt = "2026-04-19T11:00:00.000Z"; + + (mockSessionManager.listCached as ReturnType).mockResolvedValueOnce([ + makeSession({ + id: "my-app-orchestrator-0", + projectId: "my-app", + metadata: { role: "orchestrator" }, + status: "killed", + activity: "exited", + lastActivityAt: new Date("2026-04-19T10:00:00.000Z"), + lifecycle: olderDeadLifecycle, + }), + makeSession({ + id: "my-app-orchestrator-9", + projectId: "my-app", + metadata: { role: "orchestrator" }, + status: "killed", + activity: "exited", + lastActivityAt: new Date("2026-04-19T11:00:00.000Z"), + lifecycle: newerDeadLifecycle, + }), + ]); + + const res = await sessionsGET( + makeRequest("http://localhost:3000/api/sessions?project=my-app&orchestratorOnly=true"), + ); + expect(res.status).toBe(200); + const data = await res.json(); + + expect(data.orchestratorId).toBe("my-app-orchestrator-9"); + expect(data.orchestrators).toEqual([ + { id: "my-app-orchestrator-0", projectId: "my-app", projectName: "My App" }, + { id: "my-app-orchestrator-9", projectId: "my-app", projectName: "My App" }, + ]); + expect(data.sessions).toEqual([]); + }); + it("enriches all PRs concurrently, not sequentially", async () => { vi.useFakeTimers(); diff --git a/packages/web/src/app/api/sessions/route.ts b/packages/web/src/app/api/sessions/route.ts index 85a563671..59c73a5df 100644 --- a/packages/web/src/app/api/sessions/route.ts +++ b/packages/web/src/app/api/sessions/route.ts @@ -1,4 +1,4 @@ -import { ACTIVITY_STATE, isOrchestratorSession } from "@aoagents/ao-core"; +import { ACTIVITY_STATE, isOrchestratorSession, isTerminalSession } from "@aoagents/ao-core"; import { getServices, getSCM } from "@/lib/services"; import { sessionToDashboard, @@ -16,6 +16,55 @@ const METADATA_ENRICH_TIMEOUT_MS = 3_000; const PR_ENRICH_TIMEOUT_MS = 4_000; const PER_PR_ENRICH_TIMEOUT_MS = 1_500; +function compareOrchestratorRecency(a: { lastActivityAt?: Date | null; createdAt?: Date | null; id: string }, b: { lastActivityAt?: Date | null; createdAt?: Date | null; id: string }): number { + return ( + (b.lastActivityAt?.getTime() ?? 0) - (a.lastActivityAt?.getTime() ?? 0) || + (b.createdAt?.getTime() ?? 0) - (a.createdAt?.getTime() ?? 0) || + a.id.localeCompare(b.id) + ); +} + +function listProjectOrchestratorSessions( + sessions: Parameters[0], + projects: Parameters[1], +): Parameters[0] { + const allSessionPrefixes = Object.entries(projects).map( + ([projectId, project]) => project.sessionPrefix ?? projectId, + ); + + const projectOrchestrators = sessions + .filter((session) => + isOrchestratorSession( + session, + projects[session.projectId]?.sessionPrefix ?? session.projectId, + allSessionPrefixes, + ), + ) + .sort(compareOrchestratorRecency); + + const liveOrchestrators = projectOrchestrators.filter((session) => !isTerminalSession(session)); + return liveOrchestrators.length > 0 ? liveOrchestrators : projectOrchestrators; +} + +function selectPreferredOrchestratorId( + sessions: Parameters[0], + projects: Parameters[1], +): string | null { + return listProjectOrchestratorSessions(sessions, projects)[0]?.id ?? null; +} + +function listPreferredProjectOrchestrators( + sessions: Parameters[0], + projects: Parameters[1], +) { + const preferredOrchestrators = listProjectOrchestratorSessions(sessions, projects); + const liveOrchestrators = preferredOrchestrators.filter((session) => !isTerminalSession(session)); + return listDashboardOrchestrators( + liveOrchestrators.length > 0 ? liveOrchestrators : preferredOrchestrators, + projects, + ); +} + export async function GET(request: Request) { const correlationId = getCorrelationId(request); const startedAt = Date.now(); @@ -32,8 +81,12 @@ export async function GET(request: Request) { : undefined; const coreSessions = await sessionManager.listCached(requestedProjectId); const visibleSessions = filterProjectSessions(coreSessions, projectFilter, config.projects); - const orchestrators = listDashboardOrchestrators(visibleSessions, config.projects); - const orchestratorId = orchestrators.length === 1 ? (orchestrators[0]?.id ?? null) : null; + const orchestrators = requestedProjectId + ? listPreferredProjectOrchestrators(visibleSessions, config.projects) + : listDashboardOrchestrators(visibleSessions, config.projects); + const orchestratorId = requestedProjectId + ? selectPreferredOrchestratorId(visibleSessions, config.projects) + : (orchestrators.length === 1 ? (orchestrators[0]?.id ?? null) : null); if (orchestratorOnly) { recordApiObservation({ diff --git a/packages/web/src/components/SessionDetail.tsx b/packages/web/src/components/SessionDetail.tsx index 5bdbc255a..5ef47213b 100644 --- a/packages/web/src/components/SessionDetail.tsx +++ b/packages/web/src/components/SessionDetail.tsx @@ -596,25 +596,23 @@ export function SessionDetail({ ) : null} - {/* Kill / Restore — only for non-orchestrator worker sessions */} - {!isOrchestrator && ( - isRestorable ? ( - - ) : !terminalEnded ? ( + {/* Restore is available for any restorable session; Kill stays worker-only. */} + {isRestorable ? ( + + ) : !isOrchestrator && !terminalEnded ? ( - ) : null - )} + ) : null} {!isOrchestrator && orchestratorHref ? ( { expect(screen.queryByTestId("direct-terminal")).not.toBeInTheDocument(); }); + it("shows restore for restorable orchestrator sessions", () => { + render( + , + ); + + expect(within(screen.getByRole("banner")).getByRole("button", { name: "Restore" })).toBeInTheDocument(); + expect(within(screen.getByRole("banner")).queryByRole("button", { name: "Kill" })).not.toBeInTheDocument(); + }); + it("hides the desktop orchestrator button on orchestrator session pages", () => { render(