Merge pull request #1308 from harshitsinghbhandari/fix/ao-start-orchestrator-reuse-race
fix: restore dead orchestrators on ao start
This commit is contained in:
Harshit Singh Bhandari
GitHub
commit
e45f34e1dd
|
|
@ -0,0 +1,10 @@
|
|||
"@aoagents/ao-cli": patch
|
||||
"@aoagents/ao-web": patch
|
||||
"@aoagents/ao-plugin-agent-codex": patch
|
||||
---
|
||||
|
||||
Fix restore behavior across AO session recovery flows.
|
||||
|
||||
- restore the latest dead-but-restorable orchestrator on `ao start` instead of silently spawning a new orchestrator when tmux is gone
|
||||
- make worker session orchestrator navigation prefer the most recently active live orchestrator for the project
|
||||
- make permissionless Codex restores preserve dangerous bypass semantics so resumed workers behave like fresh permissionless launches
|
||||
|
|
@ -30,12 +30,12 @@ const {
|
|||
mockConfigRef: { current: null as Record<string, unknown> | null },
|
||||
mockSessionManager: {
|
||||
list: vi.fn(),
|
||||
restore: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
cleanup: vi.fn(),
|
||||
get: vi.fn(),
|
||||
spawn: vi.fn(),
|
||||
spawnOrchestrator: vi.fn(),
|
||||
restore: vi.fn(),
|
||||
send: vi.fn(),
|
||||
claimPR: vi.fn(),
|
||||
},
|
||||
|
|
@ -57,8 +57,10 @@ const { mockPromptSelect, mockPromptConfirm } = vi.hoisted(() => ({
|
|||
mockPromptConfirm: vi.fn().mockResolvedValue(true),
|
||||
}));
|
||||
|
||||
const { mockIsAlreadyRunning, mockUnregister, mockWaitForExit } = vi.hoisted(() => ({
|
||||
const { mockAcquireStartupLock, mockIsAlreadyRunning, mockRegister, mockUnregister, mockWaitForExit } = vi.hoisted(() => ({
|
||||
mockAcquireStartupLock: vi.fn().mockResolvedValue(() => {}),
|
||||
mockIsAlreadyRunning: vi.fn().mockReturnValue(null),
|
||||
mockRegister: vi.fn(),
|
||||
mockUnregister: vi.fn(),
|
||||
mockWaitForExit: vi.fn().mockReturnValue(true),
|
||||
}));
|
||||
|
|
@ -142,7 +144,8 @@ vi.mock("../../src/lib/preflight.js", () => ({
|
|||
}));
|
||||
|
||||
vi.mock("../../src/lib/running-state.js", () => ({
|
||||
register: vi.fn(),
|
||||
acquireStartupLock: (...args: unknown[]) => mockAcquireStartupLock(...args),
|
||||
register: (...args: unknown[]) => mockRegister(...args),
|
||||
unregister: (...args: unknown[]) => mockUnregister(...args),
|
||||
isAlreadyRunning: (...args: unknown[]) => mockIsAlreadyRunning(...args),
|
||||
getRunning: vi.fn().mockReturnValue(null),
|
||||
|
|
@ -247,9 +250,10 @@ beforeEach(async () => {
|
|||
|
||||
mockSessionManager.list.mockReset();
|
||||
mockSessionManager.list.mockResolvedValue([]);
|
||||
mockSessionManager.restore.mockReset();
|
||||
mockSessionManager.restore.mockResolvedValue({ id: "app-orchestrator-restored" });
|
||||
mockSessionManager.get.mockReset();
|
||||
mockSessionManager.spawnOrchestrator.mockReset();
|
||||
mockSessionManager.restore.mockReset();
|
||||
mockSessionManager.kill.mockReset();
|
||||
mockExec.mockReset();
|
||||
mockExecSilent.mockReset();
|
||||
|
|
@ -281,8 +285,12 @@ beforeEach(async () => {
|
|||
mockPromptSelect.mockReset();
|
||||
mockPromptConfirm.mockReset();
|
||||
mockPromptConfirm.mockResolvedValue(true);
|
||||
mockAcquireStartupLock.mockReset();
|
||||
mockAcquireStartupLock.mockResolvedValue(() => {});
|
||||
mockIsAlreadyRunning.mockReset();
|
||||
mockIsAlreadyRunning.mockResolvedValue(null);
|
||||
mockRegister.mockReset();
|
||||
mockRegister.mockResolvedValue(undefined);
|
||||
mockUnregister.mockReset();
|
||||
mockWaitForExit.mockReset();
|
||||
mockWaitForExit.mockResolvedValue(true);
|
||||
|
|
@ -953,6 +961,93 @@ describe("start command — orchestrator session strategy display", () => {
|
|||
expect(mockSessionManager.spawnOrchestrator).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("restores the latest restorable orchestrator when tmux is gone", async () => {
|
||||
mockConfigRef.current = makeConfig({ "my-app": makeProject() });
|
||||
|
||||
const now = new Date();
|
||||
mockSessionManager.list.mockResolvedValue([
|
||||
{
|
||||
id: "app-orchestrator-1",
|
||||
projectId: "my-app",
|
||||
status: "killed",
|
||||
activity: "exited",
|
||||
metadata: { role: "orchestrator" },
|
||||
lastActivityAt: new Date(now.getTime() - 1000),
|
||||
lifecycle: {
|
||||
version: 2,
|
||||
session: {
|
||||
kind: "orchestrator",
|
||||
state: "working",
|
||||
reason: "task_in_progress",
|
||||
startedAt: now.toISOString(),
|
||||
completedAt: null,
|
||||
terminatedAt: null,
|
||||
lastTransitionAt: now.toISOString(),
|
||||
},
|
||||
pr: {
|
||||
state: "none",
|
||||
reason: "not_created",
|
||||
number: null,
|
||||
url: null,
|
||||
lastObservedAt: null,
|
||||
},
|
||||
runtime: {
|
||||
state: "missing",
|
||||
reason: "tmux_missing",
|
||||
lastObservedAt: now.toISOString(),
|
||||
handle: null,
|
||||
tmuxName: "tmux-old-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "app-orchestrator-2",
|
||||
projectId: "my-app",
|
||||
status: "killed",
|
||||
activity: "exited",
|
||||
metadata: { role: "orchestrator" },
|
||||
lastActivityAt: now,
|
||||
lifecycle: {
|
||||
version: 2,
|
||||
session: {
|
||||
kind: "orchestrator",
|
||||
state: "working",
|
||||
reason: "task_in_progress",
|
||||
startedAt: now.toISOString(),
|
||||
completedAt: null,
|
||||
terminatedAt: null,
|
||||
lastTransitionAt: now.toISOString(),
|
||||
},
|
||||
pr: {
|
||||
state: "none",
|
||||
reason: "not_created",
|
||||
number: null,
|
||||
url: null,
|
||||
lastObservedAt: null,
|
||||
},
|
||||
runtime: {
|
||||
state: "missing",
|
||||
reason: "tmux_missing",
|
||||
lastObservedAt: now.toISOString(),
|
||||
handle: null,
|
||||
tmuxName: "tmux-old-2",
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
mockSessionManager.restore.mockResolvedValue({
|
||||
id: "app-orchestrator-2",
|
||||
runtimeHandle: { id: "tmux-restored-2" },
|
||||
});
|
||||
|
||||
await program.parseAsync(["node", "test", "start", "--no-dashboard"]);
|
||||
|
||||
const output = getLoggedOutput();
|
||||
expect(output).toContain("ao session attach app-orchestrator-2");
|
||||
expect(mockSessionManager.restore).toHaveBeenCalledWith("app-orchestrator-2");
|
||||
expect(mockSessionManager.spawnOrchestrator).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("navigates directly to session page when one existing orchestrator found with dashboard enabled", async () => {
|
||||
mockConfigRef.current = makeConfig({ "my-app": makeProject() });
|
||||
|
||||
|
|
@ -1232,6 +1327,34 @@ describe("start command — orchestrator session strategy display", () => {
|
|||
expect(fakeDashboard.kill).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports startup lock acquisition failures through the normal CLI error path", async () => {
|
||||
mockConfigRef.current = makeConfig({ "my-app": makeProject() });
|
||||
mockAcquireStartupLock.mockRejectedValueOnce(
|
||||
new Error("Could not acquire startup lock (/tmp/startup.lock)"),
|
||||
);
|
||||
|
||||
await expect(program.parseAsync(["node", "test", "start"])).rejects.toThrow("process.exit(1)");
|
||||
|
||||
const errors = vi
|
||||
.mocked(console.error)
|
||||
.mock.calls.map((c) => c.join(" "))
|
||||
.join("\n");
|
||||
expect(errors).toContain("Could not acquire startup lock (/tmp/startup.lock)");
|
||||
expect(mockIsAlreadyRunning).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("releases the startup lock before exiting on startup failures", async () => {
|
||||
mockConfigRef.current = makeConfig({ "my-app": makeProject() });
|
||||
const releaseStartupLock = vi.fn();
|
||||
mockAcquireStartupLock.mockResolvedValueOnce(releaseStartupLock);
|
||||
mockSessionManager.list.mockResolvedValue([]);
|
||||
mockSessionManager.spawnOrchestrator.mockRejectedValue(new Error("Spawn failed"));
|
||||
|
||||
await expect(program.parseAsync(["node", "test", "start"])).rejects.toThrow("process.exit(1)");
|
||||
|
||||
expect(releaseStartupLock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("fails and cleans up dashboard when sm.restore throws on a killed orchestrator", async () => {
|
||||
mockConfigRef.current = makeConfig({ "my-app": makeProject() });
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync, readFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const testHome = join(process.cwd(), ".tmp-running-state-home");
|
||||
|
||||
vi.mock("node:os", () => ({
|
||||
homedir: () => testHome,
|
||||
}));
|
||||
|
||||
describe("running-state", () => {
|
||||
beforeEach(() => {
|
||||
rmSync(testHome, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(testHome, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("keeps running.json when the pid probe returns EPERM", async () => {
|
||||
const runningState = await import("../../src/lib/running-state.js");
|
||||
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => {
|
||||
const error = new Error("operation not permitted") as Error & { code?: string };
|
||||
error.code = "EPERM";
|
||||
throw error;
|
||||
});
|
||||
|
||||
await runningState.register({
|
||||
pid: 424242,
|
||||
configPath: "/tmp/agent-orchestrator.yaml",
|
||||
port: 4321,
|
||||
startedAt: new Date("2026-04-19T00:00:00.000Z").toISOString(),
|
||||
projects: ["my-app"],
|
||||
});
|
||||
|
||||
const state = await runningState.getRunning();
|
||||
const stateFile = join(testHome, ".agent-orchestrator", "running.json");
|
||||
|
||||
expect(state).toEqual({
|
||||
pid: 424242,
|
||||
configPath: "/tmp/agent-orchestrator.yaml",
|
||||
port: 4321,
|
||||
startedAt: new Date("2026-04-19T00:00:00.000Z").toISOString(),
|
||||
projects: ["my-app"],
|
||||
});
|
||||
expect(existsSync(stateFile)).toBe(true);
|
||||
expect(killSpy).toHaveBeenCalledWith(424242, 0);
|
||||
});
|
||||
|
||||
it("keeps startup locks alive when the pid probe returns EPERM", async () => {
|
||||
const runningState = await import("../../src/lib/running-state.js");
|
||||
const lockDir = join(testHome, ".agent-orchestrator");
|
||||
const lockFile = join(lockDir, "startup.lock");
|
||||
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => {
|
||||
const error = new Error("operation not permitted") as Error & { code?: string };
|
||||
error.code = "EPERM";
|
||||
throw error;
|
||||
});
|
||||
|
||||
const release = await runningState.acquireStartupLock(100);
|
||||
|
||||
await expect(runningState.acquireStartupLock(100)).rejects.toThrow(
|
||||
`Could not acquire startup lock (${lockFile})`,
|
||||
);
|
||||
expect(readFileSync(lockFile, "utf-8")).toContain(`"pid":${process.pid}`);
|
||||
|
||||
release();
|
||||
expect(killSpy).toHaveBeenCalledWith(process.pid, 0);
|
||||
});
|
||||
});
|
||||
|
|
@ -52,7 +52,14 @@ import {
|
|||
} from "../lib/web-dir.js";
|
||||
import { rebuildDashboardProductionArtifacts } from "../lib/dashboard-rebuild.js";
|
||||
import { preflight } from "../lib/preflight.js";
|
||||
import { register, unregister, isAlreadyRunning, getRunning, waitForExit } from "../lib/running-state.js";
|
||||
import {
|
||||
register,
|
||||
unregister,
|
||||
isAlreadyRunning,
|
||||
getRunning,
|
||||
waitForExit,
|
||||
acquireStartupLock,
|
||||
} from "../lib/running-state.js";
|
||||
import { preventIdleSleep } from "../lib/prevent-sleep.js";
|
||||
import { isHumanCaller } from "../lib/caller-context.js";
|
||||
import { detectEnvironment } from "../lib/detect-env.js";
|
||||
|
|
@ -1379,7 +1386,16 @@ export function registerStart(program: Command): void {
|
|||
interactive?: boolean;
|
||||
},
|
||||
) => {
|
||||
let releaseStartupLock: (() => void) | undefined;
|
||||
let startupLockReleased = false;
|
||||
const unlockStartup = (): void => {
|
||||
if (startupLockReleased || !releaseStartupLock) return;
|
||||
startupLockReleased = true;
|
||||
releaseStartupLock();
|
||||
};
|
||||
|
||||
try {
|
||||
releaseStartupLock = await acquireStartupLock();
|
||||
let config: OrchestratorConfig;
|
||||
let projectId: string;
|
||||
let project: ProjectConfig;
|
||||
|
|
@ -1408,6 +1424,7 @@ export function registerStart(program: Command): void {
|
|||
if (choice === "open") {
|
||||
const url = `http://localhost:${running.port}`;
|
||||
openUrl(url);
|
||||
unlockStartup();
|
||||
process.exit(0);
|
||||
} else if (choice === "new") {
|
||||
// Defer config mutation until after config is loaded below
|
||||
|
|
@ -1427,6 +1444,7 @@ export function registerStart(program: Command): void {
|
|||
console.log(chalk.yellow("\n Stopped existing instance. Restarting...\n"));
|
||||
// Continue to startup below
|
||||
} else {
|
||||
unlockStartup();
|
||||
process.exit(0);
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1436,6 +1454,7 @@ export function registerStart(program: Command): void {
|
|||
console.log(`PID: ${running.pid}`);
|
||||
console.log(`Projects: ${running.projects.join(", ")}`);
|
||||
console.log(`To restart: ao stop && ao start`);
|
||||
unlockStartup();
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
|
@ -1565,6 +1584,7 @@ export function registerStart(program: Command): void {
|
|||
startedAt: new Date().toISOString(),
|
||||
projects: [projectId],
|
||||
});
|
||||
unlockStartup();
|
||||
|
||||
// Install shutdown handlers so `ao stop` (which sends SIGTERM to
|
||||
// this pid) flushes lifecycle health state before exit. Handlers
|
||||
|
|
@ -1590,7 +1610,10 @@ export function registerStart(program: Command): void {
|
|||
} else {
|
||||
console.error(chalk.red("\nError:"), String(err));
|
||||
}
|
||||
unlockStartup();
|
||||
process.exit(1);
|
||||
} finally {
|
||||
unlockStartup();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,13 @@
|
|||
import { readFileSync, writeFileSync, mkdirSync, unlinkSync, openSync, closeSync, constants } from "node:fs";
|
||||
import {
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
mkdirSync,
|
||||
unlinkSync,
|
||||
openSync,
|
||||
closeSync,
|
||||
constants,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { homedir } from "node:os";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
|
|
@ -13,28 +22,83 @@ export interface RunningState {
|
|||
|
||||
const STATE_DIR = join(homedir(), ".agent-orchestrator");
|
||||
const STATE_FILE = join(STATE_DIR, "running.json");
|
||||
const LOCK_FILE = join(STATE_DIR, "running.lock");
|
||||
const STATE_LOCK_FILE = join(STATE_DIR, "running.lock");
|
||||
const STARTUP_LOCK_FILE = join(STATE_DIR, "startup.lock");
|
||||
const UNPARSEABLE_LOCK_GRACE_MS = 5_000;
|
||||
|
||||
interface LockMetadata {
|
||||
pid: number;
|
||||
acquiredAt: string;
|
||||
}
|
||||
|
||||
type ProcessProbeResult = "alive" | "forbidden" | "missing";
|
||||
|
||||
function ensureDir(): void {
|
||||
mkdirSync(STATE_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
function probeProcess(pid: number): ProcessProbeResult {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
return "alive";
|
||||
} catch (error: unknown) {
|
||||
if ((error as { code?: string }).code === "EPERM") {
|
||||
return "forbidden";
|
||||
}
|
||||
return "missing";
|
||||
}
|
||||
}
|
||||
|
||||
function isLockOwnerAlive(pid: number): boolean {
|
||||
return probeProcess(pid) !== "missing";
|
||||
}
|
||||
|
||||
function isRunningProcessAlive(pid: number): boolean {
|
||||
return probeProcess(pid) !== "missing";
|
||||
}
|
||||
|
||||
function readLockMetadata(lockFile: string): LockMetadata | null {
|
||||
try {
|
||||
const raw = readFileSync(lockFile, "utf-8");
|
||||
const parsed = JSON.parse(raw) as Partial<LockMetadata>;
|
||||
if (typeof parsed.pid !== "number") return null;
|
||||
return {
|
||||
pid: parsed.pid,
|
||||
acquiredAt:
|
||||
typeof parsed.acquiredAt === "string" ? parsed.acquiredAt : new Date(0).toISOString(),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isStaleUnparseableLock(lockFile: string): boolean {
|
||||
try {
|
||||
const mtimeMs = statSync(lockFile).mtimeMs;
|
||||
return Date.now() - mtimeMs > UNPARSEABLE_LOCK_GRACE_MS;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Try to create the lockfile atomically. Returns a release function on success, null on failure. */
|
||||
function tryAcquire(): (() => void) | null {
|
||||
function tryAcquire(lockFile: string): (() => void) | null {
|
||||
try {
|
||||
const fd = openSync(LOCK_FILE, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY);
|
||||
closeSync(fd);
|
||||
const fd = openSync(lockFile, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY);
|
||||
const metadata: LockMetadata = {
|
||||
pid: process.pid,
|
||||
acquiredAt: new Date().toISOString(),
|
||||
};
|
||||
try {
|
||||
writeFileSync(fd, JSON.stringify(metadata), "utf-8");
|
||||
} catch {
|
||||
try { unlinkSync(lockFile); } catch { /* best effort */ }
|
||||
return null;
|
||||
} finally {
|
||||
try { closeSync(fd); } catch { /* best effort */ }
|
||||
}
|
||||
return () => {
|
||||
try { unlinkSync(LOCK_FILE); } catch { /* best effort */ }
|
||||
try { unlinkSync(lockFile); } catch { /* best effort */ }
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
|
|
@ -43,25 +107,33 @@ function tryAcquire(): (() => void) | null {
|
|||
|
||||
/**
|
||||
* Advisory lockfile using O_EXCL for atomic creation.
|
||||
* Retries with jittered backoff. After timeout, assumes the lock is stale
|
||||
* (holder crashed) and force-removes it before one final atomic attempt.
|
||||
* Retries with jittered backoff. Dead owners are treated as stale and cleaned
|
||||
* up automatically. Live owners are never stolen; callers get a clear timeout.
|
||||
*/
|
||||
async function acquireLock(timeoutMs = 5000): Promise<() => void> {
|
||||
async function acquireLock(
|
||||
lockFile: string,
|
||||
timeoutMs = 5000,
|
||||
resourceName = "lock",
|
||||
): Promise<() => void> {
|
||||
ensureDir();
|
||||
|
||||
const start = Date.now();
|
||||
let attempt = 0;
|
||||
|
||||
while (true) {
|
||||
const release = tryAcquire();
|
||||
const release = tryAcquire(lockFile);
|
||||
if (release) return release;
|
||||
|
||||
const owner = readLockMetadata(lockFile);
|
||||
if ((!owner && isStaleUnparseableLock(lockFile))
|
||||
|| (owner && !isLockOwnerAlive(owner.pid))) {
|
||||
try { unlinkSync(lockFile); } catch { /* ignore */ }
|
||||
const retryRelease = tryAcquire(lockFile);
|
||||
if (retryRelease) return retryRelease;
|
||||
}
|
||||
|
||||
if (Date.now() - start > timeoutMs) {
|
||||
// Likely stale — remove and make one final atomic attempt.
|
||||
try { unlinkSync(LOCK_FILE); } catch { /* ignore */ }
|
||||
const finalRelease = tryAcquire();
|
||||
if (finalRelease) return finalRelease;
|
||||
throw new Error("Could not acquire running.json lock");
|
||||
throw new Error(`Could not acquire ${resourceName} (${lockFile})`);
|
||||
}
|
||||
|
||||
// Jittered backoff: 30-70ms base, growing with attempts (capped at 200ms)
|
||||
|
|
@ -97,7 +169,7 @@ function writeState(state: RunningState | null): void {
|
|||
* Uses a lockfile to prevent concurrent registration.
|
||||
*/
|
||||
export async function register(entry: RunningState): Promise<void> {
|
||||
const release = await acquireLock();
|
||||
const release = await acquireLock(STATE_LOCK_FILE, 5000, "running.json lock");
|
||||
try {
|
||||
writeState(entry);
|
||||
} finally {
|
||||
|
|
@ -109,7 +181,7 @@ export async function register(entry: RunningState): Promise<void> {
|
|||
* Unregister the running AO instance.
|
||||
*/
|
||||
export async function unregister(): Promise<void> {
|
||||
const release = await acquireLock();
|
||||
const release = await acquireLock(STATE_LOCK_FILE, 5000, "running.json lock");
|
||||
try {
|
||||
writeState(null);
|
||||
} finally {
|
||||
|
|
@ -122,12 +194,12 @@ export async function unregister(): Promise<void> {
|
|||
* Auto-prunes stale entries (dead PIDs).
|
||||
*/
|
||||
export async function getRunning(): Promise<RunningState | null> {
|
||||
const release = await acquireLock();
|
||||
const release = await acquireLock(STATE_LOCK_FILE, 5000, "running.json lock");
|
||||
try {
|
||||
const state = readState();
|
||||
if (!state) return null;
|
||||
|
||||
if (!isProcessAlive(state.pid)) {
|
||||
if (!isRunningProcessAlive(state.pid)) {
|
||||
// Stale entry — process is dead, clean up
|
||||
writeState(null);
|
||||
return null;
|
||||
|
|
@ -148,14 +220,22 @@ export async function isAlreadyRunning(): Promise<RunningState | null> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Wait for a process to exit, polling isProcessAlive.
|
||||
* Serialize `ao start` so concurrent startups cannot both observe an empty
|
||||
* running.json and create competing orchestrator/dashboard processes.
|
||||
*/
|
||||
export async function acquireStartupLock(timeoutMs = 30000): Promise<() => void> {
|
||||
return await acquireLock(STARTUP_LOCK_FILE, timeoutMs, "startup lock");
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a process to exit, polling isRunningProcessAlive.
|
||||
* Returns true if the process exited, false if timeout reached.
|
||||
*/
|
||||
export async function waitForExit(pid: number, timeoutMs = 5000): Promise<boolean> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (!isProcessAlive(pid)) return true;
|
||||
if (!isRunningProcessAlive(pid)) return true;
|
||||
await sleep(100);
|
||||
}
|
||||
return !isProcessAlive(pid);
|
||||
return !isRunningProcessAlive(pid);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1470,7 +1470,7 @@ describe("getRestoreCommand", () => {
|
|||
expect(cmd).toContain("--dangerously-bypass-approvals-and-sandbox");
|
||||
});
|
||||
|
||||
it("demotes worker restore permissionless mode to ask-for-approval never", async () => {
|
||||
it("uses dangerous bypass for worker restore permissionless mode", async () => {
|
||||
const content = jsonl(
|
||||
{ type: "session_meta", cwd: "/workspace/test", model: "gpt-4o" },
|
||||
{ threadId: "thread-1" },
|
||||
|
|
@ -1489,11 +1489,11 @@ describe("getRestoreCommand", () => {
|
|||
}),
|
||||
);
|
||||
|
||||
expect(cmd).toContain("--ask-for-approval never");
|
||||
expect(cmd).not.toContain("--dangerously-bypass-approvals-and-sandbox");
|
||||
expect(cmd).toContain("--dangerously-bypass-approvals-and-sandbox");
|
||||
expect(cmd).not.toContain("--ask-for-approval");
|
||||
});
|
||||
|
||||
it("includes --ask-for-approval never from project config", async () => {
|
||||
it("keeps auto-edit restore policy at ask-for-approval never", async () => {
|
||||
const content = jsonl(
|
||||
{ type: "session_meta", cwd: "/workspace/test", model: "gpt-4o" },
|
||||
{ threadId: "thread-1" },
|
||||
|
|
|
|||
|
|
@ -733,8 +733,7 @@ function createCodexAgent(): Agent {
|
|||
const parts: string[] = [shellEscape(binary), "resume"];
|
||||
appendNoUpdateCheckFlag(parts);
|
||||
|
||||
const isOrchestrator = session.metadata?.["role"] === "orchestrator";
|
||||
appendApprovalFlags(parts, project.agentConfig?.permissions, isOrchestrator);
|
||||
appendApprovalFlags(parts, project.agentConfig?.permissions);
|
||||
const effectiveModel = (project.agentConfig?.model ?? data.model) as string | undefined;
|
||||
appendModelFlags(parts, effectiveModel ?? undefined);
|
||||
|
||||
|
|
|
|||
|
|
@ -336,6 +336,153 @@ describe("API Routes", () => {
|
|||
expect(mockSessionManager.listCached).toHaveBeenCalledWith("docs-app");
|
||||
});
|
||||
|
||||
it("prefers the most recently active live orchestrator for project-scoped worker navigation", async () => {
|
||||
const deadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T11:00:00.000Z"));
|
||||
deadLifecycle.session.state = "terminated";
|
||||
deadLifecycle.session.reason = "runtime_missing";
|
||||
deadLifecycle.session.terminatedAt = "2026-04-19T11:00:00.000Z";
|
||||
deadLifecycle.session.lastTransitionAt = "2026-04-19T11:00:00.000Z";
|
||||
deadLifecycle.runtime.state = "missing";
|
||||
deadLifecycle.runtime.reason = "process_missing";
|
||||
deadLifecycle.runtime.lastObservedAt = "2026-04-19T11:00:00.000Z";
|
||||
|
||||
const olderLive = makeSession({
|
||||
id: "my-app-orchestrator-1",
|
||||
projectId: "my-app",
|
||||
metadata: { role: "orchestrator" },
|
||||
lastActivityAt: new Date("2026-04-19T09:00:00.000Z"),
|
||||
});
|
||||
const newerLive = makeSession({
|
||||
id: "my-app-orchestrator-2",
|
||||
projectId: "my-app",
|
||||
metadata: { role: "orchestrator" },
|
||||
lastActivityAt: new Date("2026-04-19T10:00:00.000Z"),
|
||||
});
|
||||
const deadOlder = makeSession({
|
||||
id: "my-app-orchestrator-0",
|
||||
projectId: "my-app",
|
||||
metadata: { role: "orchestrator" },
|
||||
status: "killed",
|
||||
activity: "exited",
|
||||
lastActivityAt: new Date("2026-04-19T11:00:00.000Z"),
|
||||
lifecycle: deadLifecycle,
|
||||
});
|
||||
(mockSessionManager.listCached as ReturnType<typeof vi.fn>).mockResolvedValueOnce([
|
||||
deadOlder,
|
||||
olderLive,
|
||||
newerLive,
|
||||
makeSession({
|
||||
id: "backend-3",
|
||||
projectId: "my-app",
|
||||
status: "working",
|
||||
activity: "active",
|
||||
}),
|
||||
]);
|
||||
|
||||
const res = await sessionsGET(
|
||||
makeRequest("http://localhost:3000/api/sessions?project=my-app&orchestratorOnly=true"),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const data = await res.json();
|
||||
|
||||
expect(data.orchestratorId).toBe("my-app-orchestrator-2");
|
||||
expect(data.orchestrators.map((session: { id: string }) => session.id)).toEqual([
|
||||
"my-app-orchestrator-1",
|
||||
"my-app-orchestrator-2",
|
||||
]);
|
||||
expect(data.sessions).toEqual([]);
|
||||
expect(mockSessionManager.listCached).toHaveBeenCalledWith("my-app");
|
||||
});
|
||||
|
||||
it("keeps dead orchestrators as the fallback project-scoped payload when none are live", async () => {
|
||||
const deadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T11:00:00.000Z"));
|
||||
deadLifecycle.session.state = "terminated";
|
||||
deadLifecycle.session.reason = "runtime_missing";
|
||||
deadLifecycle.session.terminatedAt = "2026-04-19T11:00:00.000Z";
|
||||
deadLifecycle.session.lastTransitionAt = "2026-04-19T11:00:00.000Z";
|
||||
deadLifecycle.runtime.state = "missing";
|
||||
deadLifecycle.runtime.reason = "process_missing";
|
||||
deadLifecycle.runtime.lastObservedAt = "2026-04-19T11:00:00.000Z";
|
||||
|
||||
(mockSessionManager.listCached as ReturnType<typeof vi.fn>).mockResolvedValueOnce([
|
||||
makeSession({
|
||||
id: "my-app-orchestrator-0",
|
||||
projectId: "my-app",
|
||||
metadata: { role: "orchestrator" },
|
||||
status: "killed",
|
||||
activity: "exited",
|
||||
lastActivityAt: new Date("2026-04-19T11:00:00.000Z"),
|
||||
lifecycle: deadLifecycle,
|
||||
}),
|
||||
]);
|
||||
|
||||
const res = await sessionsGET(
|
||||
makeRequest("http://localhost:3000/api/sessions?project=my-app&orchestratorOnly=true"),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const data = await res.json();
|
||||
|
||||
expect(data.orchestratorId).toBe("my-app-orchestrator-0");
|
||||
expect(data.orchestrators).toEqual([
|
||||
{ id: "my-app-orchestrator-0", projectId: "my-app", projectName: "My App" },
|
||||
]);
|
||||
expect(data.sessions).toEqual([]);
|
||||
});
|
||||
|
||||
it("prefers the most recently active dead orchestrator when no live project orchestrator exists", async () => {
|
||||
const olderDeadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T10:00:00.000Z"));
|
||||
olderDeadLifecycle.session.state = "terminated";
|
||||
olderDeadLifecycle.session.reason = "runtime_missing";
|
||||
olderDeadLifecycle.session.terminatedAt = "2026-04-19T10:00:00.000Z";
|
||||
olderDeadLifecycle.session.lastTransitionAt = "2026-04-19T10:00:00.000Z";
|
||||
olderDeadLifecycle.runtime.state = "missing";
|
||||
olderDeadLifecycle.runtime.reason = "process_missing";
|
||||
olderDeadLifecycle.runtime.lastObservedAt = "2026-04-19T10:00:00.000Z";
|
||||
|
||||
const newerDeadLifecycle = createInitialCanonicalLifecycle("orchestrator", new Date("2026-04-19T11:00:00.000Z"));
|
||||
newerDeadLifecycle.session.state = "terminated";
|
||||
newerDeadLifecycle.session.reason = "runtime_missing";
|
||||
newerDeadLifecycle.session.terminatedAt = "2026-04-19T11:00:00.000Z";
|
||||
newerDeadLifecycle.session.lastTransitionAt = "2026-04-19T11:00:00.000Z";
|
||||
newerDeadLifecycle.runtime.state = "missing";
|
||||
newerDeadLifecycle.runtime.reason = "process_missing";
|
||||
newerDeadLifecycle.runtime.lastObservedAt = "2026-04-19T11:00:00.000Z";
|
||||
|
||||
(mockSessionManager.listCached as ReturnType<typeof vi.fn>).mockResolvedValueOnce([
|
||||
makeSession({
|
||||
id: "my-app-orchestrator-0",
|
||||
projectId: "my-app",
|
||||
metadata: { role: "orchestrator" },
|
||||
status: "killed",
|
||||
activity: "exited",
|
||||
lastActivityAt: new Date("2026-04-19T10:00:00.000Z"),
|
||||
lifecycle: olderDeadLifecycle,
|
||||
}),
|
||||
makeSession({
|
||||
id: "my-app-orchestrator-9",
|
||||
projectId: "my-app",
|
||||
metadata: { role: "orchestrator" },
|
||||
status: "killed",
|
||||
activity: "exited",
|
||||
lastActivityAt: new Date("2026-04-19T11:00:00.000Z"),
|
||||
lifecycle: newerDeadLifecycle,
|
||||
}),
|
||||
]);
|
||||
|
||||
const res = await sessionsGET(
|
||||
makeRequest("http://localhost:3000/api/sessions?project=my-app&orchestratorOnly=true"),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const data = await res.json();
|
||||
|
||||
expect(data.orchestratorId).toBe("my-app-orchestrator-9");
|
||||
expect(data.orchestrators).toEqual([
|
||||
{ id: "my-app-orchestrator-0", projectId: "my-app", projectName: "My App" },
|
||||
{ id: "my-app-orchestrator-9", projectId: "my-app", projectName: "My App" },
|
||||
]);
|
||||
expect(data.sessions).toEqual([]);
|
||||
});
|
||||
|
||||
it("enriches all PRs concurrently, not sequentially", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { ACTIVITY_STATE, isOrchestratorSession } from "@aoagents/ao-core";
|
||||
import { ACTIVITY_STATE, isOrchestratorSession, isTerminalSession } from "@aoagents/ao-core";
|
||||
import { getServices, getSCM } from "@/lib/services";
|
||||
import {
|
||||
sessionToDashboard,
|
||||
|
|
@ -16,6 +16,55 @@ const METADATA_ENRICH_TIMEOUT_MS = 3_000;
|
|||
const PR_ENRICH_TIMEOUT_MS = 4_000;
|
||||
const PER_PR_ENRICH_TIMEOUT_MS = 1_500;
|
||||
|
||||
function compareOrchestratorRecency(a: { lastActivityAt?: Date | null; createdAt?: Date | null; id: string }, b: { lastActivityAt?: Date | null; createdAt?: Date | null; id: string }): number {
|
||||
return (
|
||||
(b.lastActivityAt?.getTime() ?? 0) - (a.lastActivityAt?.getTime() ?? 0) ||
|
||||
(b.createdAt?.getTime() ?? 0) - (a.createdAt?.getTime() ?? 0) ||
|
||||
a.id.localeCompare(b.id)
|
||||
);
|
||||
}
|
||||
|
||||
function listProjectOrchestratorSessions(
|
||||
sessions: Parameters<typeof listDashboardOrchestrators>[0],
|
||||
projects: Parameters<typeof listDashboardOrchestrators>[1],
|
||||
): Parameters<typeof listDashboardOrchestrators>[0] {
|
||||
const allSessionPrefixes = Object.entries(projects).map(
|
||||
([projectId, project]) => project.sessionPrefix ?? projectId,
|
||||
);
|
||||
|
||||
const projectOrchestrators = sessions
|
||||
.filter((session) =>
|
||||
isOrchestratorSession(
|
||||
session,
|
||||
projects[session.projectId]?.sessionPrefix ?? session.projectId,
|
||||
allSessionPrefixes,
|
||||
),
|
||||
)
|
||||
.sort(compareOrchestratorRecency);
|
||||
|
||||
const liveOrchestrators = projectOrchestrators.filter((session) => !isTerminalSession(session));
|
||||
return liveOrchestrators.length > 0 ? liveOrchestrators : projectOrchestrators;
|
||||
}
|
||||
|
||||
function selectPreferredOrchestratorId(
|
||||
sessions: Parameters<typeof listDashboardOrchestrators>[0],
|
||||
projects: Parameters<typeof listDashboardOrchestrators>[1],
|
||||
): string | null {
|
||||
return listProjectOrchestratorSessions(sessions, projects)[0]?.id ?? null;
|
||||
}
|
||||
|
||||
function listPreferredProjectOrchestrators(
|
||||
sessions: Parameters<typeof listDashboardOrchestrators>[0],
|
||||
projects: Parameters<typeof listDashboardOrchestrators>[1],
|
||||
) {
|
||||
const preferredOrchestrators = listProjectOrchestratorSessions(sessions, projects);
|
||||
const liveOrchestrators = preferredOrchestrators.filter((session) => !isTerminalSession(session));
|
||||
return listDashboardOrchestrators(
|
||||
liveOrchestrators.length > 0 ? liveOrchestrators : preferredOrchestrators,
|
||||
projects,
|
||||
);
|
||||
}
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const correlationId = getCorrelationId(request);
|
||||
const startedAt = Date.now();
|
||||
|
|
@ -32,8 +81,12 @@ export async function GET(request: Request) {
|
|||
: undefined;
|
||||
const coreSessions = await sessionManager.listCached(requestedProjectId);
|
||||
const visibleSessions = filterProjectSessions(coreSessions, projectFilter, config.projects);
|
||||
const orchestrators = listDashboardOrchestrators(visibleSessions, config.projects);
|
||||
const orchestratorId = orchestrators.length === 1 ? (orchestrators[0]?.id ?? null) : null;
|
||||
const orchestrators = requestedProjectId
|
||||
? listPreferredProjectOrchestrators(visibleSessions, config.projects)
|
||||
: listDashboardOrchestrators(visibleSessions, config.projects);
|
||||
const orchestratorId = requestedProjectId
|
||||
? selectPreferredOrchestratorId(visibleSessions, config.projects)
|
||||
: (orchestrators.length === 1 ? (orchestrators[0]?.id ?? null) : null);
|
||||
|
||||
if (orchestratorOnly) {
|
||||
recordApiObservation({
|
||||
|
|
|
|||
|
|
@ -596,25 +596,23 @@ export function SessionDetail({
|
|||
</div>
|
||||
) : null}
|
||||
|
||||
{/* Kill / Restore — only for non-orchestrator worker sessions */}
|
||||
{!isOrchestrator && (
|
||||
isRestorable ? (
|
||||
<button type="button" className="dashboard-app-btn" onClick={handleRestore}>
|
||||
<svg className="h-3 w-3" fill="none" stroke="currentColor" strokeWidth="2" viewBox="0 0 24 24">
|
||||
<polyline points="1 4 1 10 7 10" />
|
||||
<path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10" />
|
||||
</svg>
|
||||
<span className="topbar-btn-label">Restore</span>
|
||||
</button>
|
||||
) : !terminalEnded ? (
|
||||
{/* Restore is available for any restorable session; Kill stays worker-only. */}
|
||||
{isRestorable ? (
|
||||
<button type="button" className="dashboard-app-btn" onClick={handleRestore}>
|
||||
<svg className="h-3 w-3" fill="none" stroke="currentColor" strokeWidth="2" viewBox="0 0 24 24">
|
||||
<polyline points="1 4 1 10 7 10" />
|
||||
<path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10" />
|
||||
</svg>
|
||||
<span className="topbar-btn-label">Restore</span>
|
||||
</button>
|
||||
) : !isOrchestrator && !terminalEnded ? (
|
||||
<button type="button" className="dashboard-app-btn dashboard-app-btn--danger" onClick={handleKill}>
|
||||
<svg className="h-3.5 w-3.5" fill="none" stroke="currentColor" strokeWidth="2" viewBox="0 0 24 24">
|
||||
<path d="M3 6h18M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2" />
|
||||
</svg>
|
||||
<span className="topbar-btn-label">Kill</span>
|
||||
</button>
|
||||
) : null
|
||||
)}
|
||||
) : null}
|
||||
|
||||
{!isOrchestrator && orchestratorHref ? (
|
||||
<a
|
||||
|
|
|
|||
|
|
@ -195,6 +195,35 @@ describe("SessionDetail desktop layout", () => {
|
|||
expect(screen.queryByTestId("direct-terminal")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows restore for restorable orchestrator sessions", () => {
|
||||
render(
|
||||
<SessionDetail
|
||||
session={makeSession({
|
||||
id: "my-app-orchestrator",
|
||||
projectId: "my-app",
|
||||
status: "terminated",
|
||||
activity: "exited",
|
||||
summary: "Project orchestrator",
|
||||
pr: null,
|
||||
})}
|
||||
isOrchestrator
|
||||
orchestratorZones={{
|
||||
merge: 1,
|
||||
respond: 0,
|
||||
review: 0,
|
||||
pending: 0,
|
||||
working: 2,
|
||||
done: 3,
|
||||
}}
|
||||
projectOrchestratorId="my-app-orchestrator"
|
||||
projects={[{ id: "my-app", name: "My App", path: "/tmp/my-app" }]}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(within(screen.getByRole("banner")).getByRole("button", { name: "Restore" })).toBeInTheDocument();
|
||||
expect(within(screen.getByRole("banner")).queryByRole("button", { name: "Kill" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides the desktop orchestrator button on orchestrator session pages", () => {
|
||||
render(
|
||||
<SessionDetail
|
||||
|
|
|
|||
Loading…
Reference in New Issue