From 43fca9409926ff7f673733b1c43d95a333ff38ee Mon Sep 17 00:00:00 2001 From: Dhruv Sharma Date: Thu, 7 May 2026 12:07:40 +0530 Subject: [PATCH] feat(core): round out session-manager activity event instrumentation (#1657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1620 wired activity events for the worker-spawn happy path (spawn_started/spawned/spawn_failed/killed). This extends the same pattern across the rest of session-manager's failure surfaces so RCA can answer questions today's logs can't: - Did the orchestrator spawn fail? (orchestrator path had no AE envelope at all; now wrapped like worker spawn) - Did the agent's startup prompt deliver, or did all 3 retries fail? - Did `runtime.destroy` / `workspace.destroy` succeed during kill, or silently leak? - Did `runtime_lost` reconciliation fire for this session? - Did `ao send` actually deliver? New event kinds (added to ActivityEventKind union): session.kill_started, session.prompt_delivery_failed, session.send_failed, session.restore_failed, session.restore_fallback, session.rollback_started, session.rollback_step_failed, session.workspace_hooks_failed, session.cleanup_error, session.orchestrator_conflict, runtime.lost_detected, runtime.lost_persist_failed, runtime.destroy_failed, workspace.destroy_failed, agent.opencode_purge_failed, tracker.issue_fetch_failed, tracker.generate_prompt_failed, metadata.corrupt_detected Coverage (per #1657 spec): - All 8 MUST emits implemented and have a regression test in session-manager-instrumentation.test.ts - 13 SHOULD emits implemented (cleanup loop, orchestrator subpaths, kill silent catches, send/restore failure paths, etc.) - 4 COULD emits implemented - Out-of-scope per-list enrichment paths intentionally NOT instrumented Invariants preserved (extends #1620's B1-B16): - B1: state mutation BEFORE event emission (runtime_lost persist happens before the AE fires; verified by test) - B2: never wrap recordActivityEvent in try/catch - B11: prompt content excluded from prompt_delivery_failed data (verified by test that asserts the prompt string is absent) - B16: failure-only — emit on final retry exhaustion only, not per attempt - B25 (new): cleanup-stack rollbacks emit per-step via the onError callback hook, not in aggregate Co-Authored-By: Claude Opus 4.7 --- .../session-manager-instrumentation.test.ts | 442 ++++++++++++++++ packages/core/src/activity-events.ts | 18 + packages/core/src/metadata.ts | 19 + packages/core/src/session-manager.ts | 487 ++++++++++++++++-- 4 files changed, 932 insertions(+), 34 deletions(-) create mode 100644 packages/core/src/__tests__/session-manager-instrumentation.test.ts diff --git a/packages/core/src/__tests__/session-manager-instrumentation.test.ts b/packages/core/src/__tests__/session-manager-instrumentation.test.ts new file mode 100644 index 000000000..076c9a64b --- /dev/null +++ b/packages/core/src/__tests__/session-manager-instrumentation.test.ts @@ -0,0 +1,442 @@ +/** + * Regression tests for session-manager activity event instrumentation + * (issue #1657 — extends PR #1620 to cover the rest of the failure paths + * inside spawn / kill / restore / send / list). + * + * One test per MUST-class emit. Pattern follows the lifecycle-manager- + * instrumentation tests: mock `recordActivityEvent`, drive the manager into + * the failure path, then assert the right kind/level/data was logged. + * + * Invariants asserted by these tests (PR #1620 B1/B2 plus #1657 B25): + * - state mutation happens BEFORE event emission + * - failure-only emits — no event on a successful send/spawn + * - cleanup-stack rollbacks emit per failed step (not in aggregate) + * - data payload omits prompt content (`session.prompt_delivery_failed`) + */ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { createSessionManager } from "../session-manager.js"; +import { writeMetadata, readMetadataRaw } from "../metadata.js"; +import { recordActivityEvent } from "../activity-events.js"; +import type { OrchestratorConfig, PluginRegistry, Agent } from "../types.js"; +import { setupTestContext, teardownTestContext, makeHandle, type TestContext } from "./test-utils.js"; + +vi.mock("../activity-events.js", () => ({ + recordActivityEvent: vi.fn(), +})); + +let ctx: TestContext; +let sessionsDir: string; +let mockRegistry: PluginRegistry; +let config: OrchestratorConfig; + +beforeEach(() => { + ctx = setupTestContext(); + ({ sessionsDir, mockRegistry, config } = ctx); + vi.mocked(recordActivityEvent).mockClear(); +}); + +afterEach(() => { + teardownTestContext(ctx); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +function findEvent(kind: string) { + return vi + .mocked(recordActivityEvent) + .mock.calls.map((c) => c[0]) + .find((e) => e.kind === kind); +} + +function findAllEvents(kind: string) { + return vi + .mocked(recordActivityEvent) + .mock.calls.map((c) => c[0]) + .filter((e) => e.kind === kind); +} + +describe("session.kill_started (MUST)", () => { + it("emits before runtime.destroy is attempted", async () => { + let destroyCalled = false; + let killStartedEmittedBeforeDestroy = false; + + vi.mocked(ctx.mockRuntime.destroy).mockImplementation(async () => { + destroyCalled = true; + killStartedEmittedBeforeDestroy = !!findEvent("session.kill_started"); + }); + + writeMetadata(sessionsDir, "app-killed", { + worktree: "/tmp/ws", + branch: "main", + status: "working", + project: "my-app", + runtimeHandle: makeHandle("rt-1"), + }); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await sm.kill("app-killed"); + + expect(destroyCalled).toBe(true); + expect(killStartedEmittedBeforeDestroy).toBe(true); + + const start = findEvent("session.kill_started"); + expect(start).toMatchObject({ + projectId: "my-app", + sessionId: "app-killed", + source: "session-manager", + kind: "session.kill_started", + }); + }); + + it("does not emit kill_started when session is already terminated (idempotent)", async () => { + writeMetadata(sessionsDir, "app-already-killed", { + worktree: "/tmp/ws", + branch: "main", + status: "killed", + project: "my-app", + lifecycle: { + version: 2, + session: { + kind: "worker", + state: "terminated", + reason: "manually_killed", + startedAt: "2025-01-01T00:00:00.000Z", + completedAt: null, + terminatedAt: "2025-01-01T00:00:00.000Z", + lastTransitionAt: "2025-01-01T00:00:00.000Z", + }, + pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null }, + runtime: { + state: "missing", + reason: "manual_kill_requested", + lastObservedAt: "2025-01-01T00:00:00.000Z", + handle: null, + tmuxName: null, + }, + }, + }); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await sm.kill("app-already-killed"); + + expect(findEvent("session.kill_started")).toBeUndefined(); + }); +}); + +describe("session.prompt_delivery_failed (MUST)", () => { + it("emits after all 3 retries exhaust, omitting prompt content", async () => { + vi.useFakeTimers(); + const promptDeliveryAgent: Agent = { + name: "prompt-delivery-agent", + processName: "pda", + promptDelivery: "post-launch", + getLaunchCommand: vi.fn().mockReturnValue("agent --start"), + getEnvironment: vi.fn().mockReturnValue({}), + detectActivity: vi.fn().mockReturnValue("active"), + getActivityState: vi.fn().mockResolvedValue({ state: "active" }), + isProcessRunning: vi.fn().mockResolvedValue(true), + getSessionInfo: vi.fn().mockResolvedValue(null), + }; + + const originalGet = mockRegistry.get; + mockRegistry.get = vi.fn().mockImplementation((slot: string, name?: string) => { + if (slot === "agent" && (!name || name === "prompt-delivery-agent")) return promptDeliveryAgent; + return (originalGet as any)(slot, name); + }); + config.projects["my-app"]!.agent = "prompt-delivery-agent"; + + vi.mocked(ctx.mockRuntime.sendMessage).mockRejectedValue(new Error("PTY closed")); + + const sm = createSessionManager({ config, registry: mockRegistry }); + const spawnPromise = sm.spawn({ projectId: "my-app", prompt: "secret task content" }); + await vi.runAllTimersAsync(); + await spawnPromise; + + const event = findEvent("session.prompt_delivery_failed"); + expect(event).toBeDefined(); + expect(event!.level).toBe("error"); + expect(event!.source).toBe("session-manager"); + expect(event!.data).toMatchObject({ attempts: 3 }); + expect(JSON.stringify(event!.data ?? {})).not.toContain("secret task content"); + }); + + it("does not emit when prompt delivery succeeds on first attempt", async () => { + vi.useFakeTimers(); + const promptDeliveryAgent: Agent = { + name: "prompt-delivery-agent", + processName: "pda", + promptDelivery: "post-launch", + getLaunchCommand: vi.fn().mockReturnValue("agent --start"), + getEnvironment: vi.fn().mockReturnValue({}), + detectActivity: vi.fn().mockReturnValue("active"), + getActivityState: vi.fn().mockResolvedValue({ state: "active" }), + isProcessRunning: vi.fn().mockResolvedValue(true), + getSessionInfo: vi.fn().mockResolvedValue(null), + }; + + const originalGet = mockRegistry.get; + mockRegistry.get = vi.fn().mockImplementation((slot: string, name?: string) => { + if (slot === "agent" && (!name || name === "prompt-delivery-agent")) return promptDeliveryAgent; + return (originalGet as any)(slot, name); + }); + config.projects["my-app"]!.agent = "prompt-delivery-agent"; + + const sm = createSessionManager({ config, registry: mockRegistry }); + const spawnPromise = sm.spawn({ projectId: "my-app", prompt: "task" }); + await vi.runAllTimersAsync(); + await spawnPromise; + + expect(findEvent("session.prompt_delivery_failed")).toBeUndefined(); + }); +}); + +describe("session.spawn_failed — orchestrator path (MUST)", () => { + it("emits when spawnOrchestrator's workspace.create throws", async () => { + vi.mocked(ctx.mockWorkspace.create).mockRejectedValue(new Error("disk full")); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await expect( + sm.spawnOrchestrator({ projectId: "my-app", systemPrompt: "be helpful" }), + ).rejects.toThrow("disk full"); + + const events = findAllEvents("session.spawn_failed"); + expect(events.length).toBeGreaterThanOrEqual(1); + const orchestratorFailure = events.find((e) => e.data && (e.data as Record)["role"] === "orchestrator"); + expect(orchestratorFailure).toBeDefined(); + expect(orchestratorFailure!.level).toBe("error"); + expect(orchestratorFailure!.projectId).toBe("my-app"); + }); + + it("emits when spawnOrchestrator's runtime.create throws", async () => { + vi.mocked(ctx.mockRuntime.create).mockRejectedValue(new Error("tmux not found")); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await expect( + sm.spawnOrchestrator({ projectId: "my-app", systemPrompt: "be helpful" }), + ).rejects.toThrow("tmux not found"); + + const events = findAllEvents("session.spawn_failed"); + const orchestratorFailure = events.find((e) => e.data && (e.data as Record)["role"] === "orchestrator"); + expect(orchestratorFailure).toBeDefined(); + expect(orchestratorFailure!.level).toBe("error"); + }); +}); + +describe("session.workspace_hooks_failed (MUST)", () => { + it("emits when setupWorkspaceHooks throws during orchestrator spawn", async () => { + const hookFailingAgent: Agent = { + ...ctx.mockAgent, + name: "hook-failing-agent", + setupWorkspaceHooks: vi.fn().mockRejectedValue(new Error("settings.json EACCES")), + }; + const originalGet = mockRegistry.get; + mockRegistry.get = vi.fn().mockImplementation((slot: string, name?: string) => { + if (slot === "agent") return hookFailingAgent; + return (originalGet as any)(slot, name); + }); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await expect( + sm.spawnOrchestrator({ projectId: "my-app", systemPrompt: "be helpful" }), + ).rejects.toThrow("settings.json EACCES"); + + const event = findEvent("session.workspace_hooks_failed"); + expect(event).toBeDefined(); + expect(event!.level).toBe("error"); + expect(event!.projectId).toBe("my-app"); + }); +}); + +describe("runtime.lost_detected (MUST)", () => { + it("emits when sm.list() persists runtime_lost for a dead runtime", async () => { + writeMetadata(sessionsDir, "app-dead", { + worktree: "/tmp/ws", + branch: "main", + status: "working", + project: "my-app", + runtimeHandle: makeHandle("rt-dead"), + }); + + // Runtime claims dead, agent process gone + vi.mocked(ctx.mockRuntime.isAlive).mockResolvedValue(false); + vi.mocked(ctx.mockAgent.isProcessRunning).mockResolvedValue(false); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await sm.list(); + + const event = findEvent("runtime.lost_detected"); + expect(event).toBeDefined(); + expect(event!.projectId).toBe("my-app"); + expect(event!.sessionId).toBe("app-dead"); + expect(event!.level).toBe("warn"); + + // B1: state mutation BEFORE event emission — verify metadata was persisted + const persisted = readMetadataRaw(sessionsDir, "app-dead"); + expect(persisted).not.toBeNull(); + const lifecycleStr = persisted!["lifecycle"]; + expect(lifecycleStr).toBeDefined(); + const lc = JSON.parse(lifecycleStr!) as { session: { state: string; reason: string } }; + expect(lc.session.state).toBe("terminated"); + expect(lc.session.reason).toBe("runtime_lost"); + }); +}); + +describe("session.send_failed (MUST)", () => { + it("emits after send retry-with-restore exhausts", async () => { + writeMetadata(sessionsDir, "app-send", { + worktree: "/tmp/ws", + branch: "main", + status: "killed", + project: "my-app", + runtimeHandle: makeHandle("rt-1"), + lifecycle: { + version: 2, + session: { + kind: "worker", + state: "terminated", + reason: "manually_killed", + startedAt: "2025-01-01T00:00:00.000Z", + completedAt: null, + terminatedAt: "2025-01-01T00:00:00.000Z", + lastTransitionAt: "2025-01-01T00:00:00.000Z", + }, + pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null }, + runtime: { + state: "missing", + reason: "process_missing", + lastObservedAt: "2025-01-01T00:00:00.000Z", + handle: null, + tmuxName: null, + }, + }, + }); + + vi.mocked(ctx.mockRuntime.sendMessage).mockRejectedValue(new Error("send broke")); + // restore() throws so retry-with-restore exhausts + vi.mocked(ctx.mockRuntime.create).mockRejectedValue(new Error("restore failed")); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await expect(sm.send("app-send", "hi")).rejects.toThrow(); + + const event = findEvent("session.send_failed"); + expect(event).toBeDefined(); + expect(event!.level).toBe("error"); + expect(event!.sessionId).toBe("app-send"); + // B11: data must not contain message content + expect(JSON.stringify(event!.data ?? {})).not.toContain("hi"); + }); +}); + +describe("session.restore_failed (MUST)", () => { + it("emits when restore's workspace restore throws", async () => { + const wsPath = join(ctx.tmpDir, "missing-ws"); + writeMetadata(sessionsDir, "app-rest", { + worktree: wsPath, + branch: "feat/x", + status: "killed", + project: "my-app", + runtimeHandle: makeHandle("rt-old"), + lifecycle: { + version: 2, + session: { + kind: "worker", + state: "terminated", + reason: "manually_killed", + startedAt: "2025-01-01T00:00:00.000Z", + completedAt: null, + terminatedAt: "2025-01-01T00:00:00.000Z", + lastTransitionAt: "2025-01-01T00:00:00.000Z", + }, + pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null }, + runtime: { + state: "missing", + reason: "process_missing", + lastObservedAt: "2025-01-01T00:00:00.000Z", + handle: null, + tmuxName: null, + }, + }, + }); + + // workspace doesn't exist + restore throws + vi.mocked(ctx.mockWorkspace.exists ?? (() => false)).mockResolvedValue?.(false); + ctx.mockWorkspace.exists = vi.fn().mockResolvedValue(false); + ctx.mockWorkspace.restore = vi.fn().mockRejectedValue(new Error("clone failed")); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await expect(sm.restore("app-rest")).rejects.toThrow(); + + const event = findEvent("session.restore_failed"); + expect(event).toBeDefined(); + expect(event!.level).toBe("error"); + expect(event!.sessionId).toBe("app-rest"); + }); + + it("emits SessionNotRestorableError when session is not restorable", async () => { + writeMetadata(sessionsDir, "app-not-rest", { + worktree: "/tmp/ws", + branch: "feat/x", + status: "working", + project: "my-app", + runtimeHandle: makeHandle("rt-1"), + lifecycle: { + version: 2, + session: { + kind: "worker", + state: "working", + reason: "task_in_progress", + startedAt: "2025-01-01T00:00:00.000Z", + completedAt: null, + terminatedAt: null, + lastTransitionAt: "2025-01-01T00:00:00.000Z", + }, + pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null }, + runtime: { + state: "alive", + reason: "process_running", + lastObservedAt: "2025-01-01T00:00:00.000Z", + handle: makeHandle("rt-1"), + tmuxName: null, + }, + }, + }); + + // active session — not restorable + vi.mocked(ctx.mockRuntime.isAlive).mockResolvedValue(true); + vi.mocked(ctx.mockAgent.isProcessRunning).mockResolvedValue(true); + + const sm = createSessionManager({ config, registry: mockRegistry }); + await expect(sm.restore("app-not-rest")).rejects.toThrow(); + + const event = findEvent("session.restore_failed"); + expect(event).toBeDefined(); + }); +}); + +describe("metadata.corrupt_detected (MUST)", () => { + it("emits when mutateMetadata side-renames a corrupt file", async () => { + // Simulate a corrupt metadata file in the sessions dir + const sessionPath = join(sessionsDir, "app-corrupt.json"); + mkdirSync(sessionsDir, { recursive: true }); + writeFileSync(sessionPath, "{ this is not json", "utf-8"); + + const { mutateMetadata } = await import("../metadata.js"); + mutateMetadata( + sessionsDir, + "app-corrupt", + () => ({ branch: "feat/x", project: "my-app" }), + { createIfMissing: true }, + ); + + const event = findEvent("metadata.corrupt_detected"); + expect(event).toBeDefined(); + expect(event!.level).toBe("warn"); + expect(event!.source).toBe("session-manager"); + expect(event!.sessionId).toBe("app-corrupt"); + const data = event!.data as Record; + expect(data["renamed"]).toBe(true); + expect(data["corruptPath"]).toMatch(/\.corrupt-\d+$/); + }); +}); diff --git a/packages/core/src/activity-events.ts b/packages/core/src/activity-events.ts index 6065c3d41..866443c8d 100644 --- a/packages/core/src/activity-events.ts +++ b/packages/core/src/activity-events.ts @@ -28,6 +28,24 @@ export type ActivityEventKind = | "session.spawned" | "session.spawn_failed" | "session.killed" + | "session.kill_started" + | "session.prompt_delivery_failed" + | "session.send_failed" + | "session.restore_failed" + | "session.restore_fallback" + | "session.rollback_started" + | "session.rollback_step_failed" + | "session.workspace_hooks_failed" + | "session.cleanup_error" + | "session.orchestrator_conflict" + | "runtime.lost_detected" + | "runtime.lost_persist_failed" + | "runtime.destroy_failed" + | "workspace.destroy_failed" + | "agent.opencode_purge_failed" + | "tracker.issue_fetch_failed" + | "tracker.generate_prompt_failed" + | "metadata.corrupt_detected" | "activity.transition" | "lifecycle.transition" | "ci.failing" diff --git a/packages/core/src/metadata.ts b/packages/core/src/metadata.ts index 95d354f77..d1cb93cec 100644 --- a/packages/core/src/metadata.ts +++ b/packages/core/src/metadata.ts @@ -35,6 +35,7 @@ import { assertValidSessionIdComponent, SESSION_ID_COMPONENT_PATTERN } from "./u import { flattenToStringRecord } from "./utils/metadata-flatten.js"; import { validateStatus } from "./utils/validation.js"; import { withFileLockSync } from "./file-lock.js"; +import { recordActivityEvent } from "./activity-events.js"; const JSON_EXTENSION = ".json"; @@ -356,8 +357,10 @@ export function mutateMetadata( // that anything was wrong — the file just becomes "not // corrupt anymore — and missing fields". const corruptPath = `${path}.corrupt-${Date.now()}`; + let renamed = false; try { renameSync(path, corruptPath); + renamed = true; // eslint-disable-next-line no-console console.warn( `[metadata] corrupt JSON at ${path}; preserved as ${corruptPath} before rewriting`, @@ -365,6 +368,22 @@ export function mutateMetadata( } catch { // best effort — proceed even if the rename fails (e.g. EACCES) } + // Surface forensically — the original `console.warn` is invisible + // to anyone not tailing the orchestrator log. AE makes corrupt + // detection queryable. + recordActivityEvent({ + sessionId, + source: "session-manager", + kind: "metadata.corrupt_detected", + level: "warn", + summary: `corrupt metadata JSON detected: ${sessionId}`, + data: { + path, + corruptPath: renamed ? corruptPath : null, + renamed, + bytes: content.length, + }, + }); } } } else if (!options.createIfMissing) { diff --git a/packages/core/src/session-manager.ts b/packages/core/src/session-manager.ts index ec03dbb57..6960c4dfe 100644 --- a/packages/core/src/session-manager.ts +++ b/packages/core/src/session-manager.ts @@ -1168,6 +1168,18 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM // Branch will be generated as feat/{issueId} (line 329-331) } else { // Other error (auth, network, etc) - fail fast + recordActivityEvent({ + projectId: spawnConfig.projectId, + source: "session-manager", + kind: "tracker.issue_fetch_failed", + level: "error", + summary: `tracker getIssue failed for ${spawnConfig.issueId}`, + data: { + issueId: spawnConfig.issueId, + tracker: plugins.tracker.name, + reason: err instanceof Error ? err.message : String(err), + }, + }); throw new Error(`Failed to fetch issue ${spawnConfig.issueId}: ${err}`, { cause: err }); } } @@ -1242,9 +1254,23 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM if (spawnConfig.issueId && plugins.tracker && resolvedIssue) { try { issueContext = await plugins.tracker.generatePrompt(spawnConfig.issueId, project); - } catch { - // Non-fatal: continue without detailed issue context - // Silently ignore errors - caller can check if issueContext is undefined + } catch (err) { + // Non-fatal: continue without detailed issue context. Surface the + // failure via AE so RCA can answer "did the agent get an enriched + // prompt or just the bare issue ID?" + recordActivityEvent({ + projectId: spawnConfig.projectId, + sessionId, + source: "session-manager", + kind: "tracker.generate_prompt_failed", + level: "warn", + summary: `tracker generatePrompt failed for ${spawnConfig.issueId}`, + data: { + issueId: spawnConfig.issueId, + tracker: plugins.tracker.name, + reason: err instanceof Error ? err.message : String(err), + }, + }); } } @@ -1437,8 +1463,73 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM // final form. Dismiss the stack so nothing below can trigger a rollback. cleanupStack.dismiss(); - // Prompt is delivered inline via the agent's launch command (positional argument). - // No post-launch polling needed — the prompt is part of process invocation. + // Send the task-specific prompt post-launch for agents that need it + // (e.g. Claude Code exits after -p, so we send the prompt after it starts + // in interactive mode). Prompt delivery failure must NOT destroy the + // session — the agent is running; user can retry with `ao send`. + let promptDelivered = false; + let promptLastError: Error | undefined; + const maxRetries = 3; + if (plugins.agent.promptDelivery === "post-launch" && agentLaunchConfig.prompt) { + const baseDelayMs = 3_000; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + // Wait for agent to start and be ready for input + // Use exponential backoff: 3s, 6s, 9s between attempts + await new Promise((resolve) => setTimeout(resolve, baseDelayMs * attempt)); + await plugins.runtime.sendMessage(handle, agentLaunchConfig.prompt); + promptDelivered = true; + break; + } catch (err) { + promptLastError = err instanceof Error ? err : new Error(String(err)); + console.error( + `[session-manager] Prompt delivery attempt ${attempt}/${maxRetries} failed: ${promptLastError.message}`, + ); + } + } + + if (!promptDelivered) { + console.error( + `[session-manager] FAILED to deliver prompt to session ${sessionId} after ${maxRetries} attempts. ` + + `User must send manually with 'ao send'. Last error: ${promptLastError?.message}`, + ); + } + + session.metadata["promptDelivered"] = String(promptDelivered); + } else if (agentLaunchConfig.prompt) { + session.metadata["promptDelivered"] = "true"; + } + + if (session.metadata["promptDelivered"]) { + updateMetadata(sessionsDir, sessionId, session.metadata); + invalidateCache(); + } + + // Emit failure-only AE after metadata has been persisted (B1) and only + // when all retries exhausted (B16). Prompt content is intentionally + // omitted from `data` because user prompts may include free-form text + // (B11 — see #1620 review). + if ( + plugins.agent.promptDelivery === "post-launch" && + agentLaunchConfig.prompt && + !promptDelivered + ) { + recordActivityEvent({ + projectId: spawnConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.prompt_delivery_failed", + level: "error", + summary: `prompt delivery failed after ${maxRetries} retries: ${sessionId}`, + data: { + attempts: maxRetries, + agent: plugins.agent.name, + lastError: promptLastError?.message, + }, + }); + } + recordActivityEvent({ projectId: spawnConfig.projectId, sessionId, @@ -1453,14 +1544,59 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM // Log cleanup failures so they don't disappear silently. The original // code used /* best effort */ swallows; the stack preserves that // behavior (cleanup errors don't propagate) but surfaces them for debug. + recordActivityEvent({ + projectId: spawnConfig.projectId, + source: "session-manager", + kind: "session.rollback_started", + level: "warn", + summary: "spawn rollback started", + data: { reason: err instanceof Error ? err.message : String(err) }, + }); await cleanupStack.runAll((cleanupErr) => { console.error("[session-manager] spawn rollback step failed:", cleanupErr); + // B25: emit per-step rollback failure so each leaked resource is queryable. + recordActivityEvent({ + projectId: spawnConfig.projectId, + source: "session-manager", + kind: "session.rollback_step_failed", + level: "error", + summary: "spawn rollback step failed", + data: { + reason: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr), + }, + }); }); throw err; } } async function spawnOrchestrator(orchestratorConfig: OrchestratorSpawnConfig): Promise { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + source: "session-manager", + kind: "session.spawn_started", + summary: "orchestrator spawn started", + data: { agent: orchestratorConfig.agent ?? undefined, role: "orchestrator" }, + }); + try { + return await _spawnOrchestratorInner(orchestratorConfig); + } catch (err) { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator spawn failed", + data: { + role: "orchestrator", + reason: err instanceof Error ? err.message : String(err), + }, + }); + throw err; + } + } + + async function _spawnOrchestratorInner(orchestratorConfig: OrchestratorSpawnConfig): Promise { const project = config.projects[orchestratorConfig.projectId]; if (!project) { throw new Error(`Unknown project: ${orchestratorConfig.projectId}`); @@ -1521,6 +1657,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM workspacePath = wsInfo.path; adoptedManagedWorkspace = adoptedInfo !== undefined && adoptedInfo !== null; } catch (err) { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator workspace.create failed", + data: { + role: "orchestrator", + stage: "workspace_create", + reason: err instanceof Error ? err.message : String(err), + }, + }); try { deleteMetadata(sessionsDir, sessionId); } catch { @@ -1566,6 +1715,21 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM await setupPathWrapperWorkspace(workspacePath); } } catch (err) { + // PR tracking and CI fetch hooks are wired here — emit a dedicated AE + // before rolling back so RCA can answer "did the orchestrator launch + // succeed but lose its hook integration?". + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.workspace_hooks_failed", + level: "error", + summary: "orchestrator workspace hooks installation failed", + data: { + agent: plugins.agent.name, + reason: err instanceof Error ? err.message : String(err), + }, + }); await cleanupWorktreeAndMetadata(); throw err; } @@ -1581,6 +1745,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM systemPromptFile = join(projectDir, `orchestrator-prompt-${sessionId}.md`); writeFileSync(systemPromptFile, orchestratorConfig.systemPrompt, "utf-8"); } catch (err) { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator systemPrompt write failed", + data: { + role: "orchestrator", + stage: "system_prompt_write", + reason: err instanceof Error ? err.message : String(err), + }, + }); await cleanupWorktreeAndMetadata(systemPromptFile); throw err; } @@ -1590,6 +1767,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM try { writeWorkspaceOpenCodeAgentsMd(workspacePath, systemPromptFile); } catch (err) { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator AGENTS.md write failed", + data: { + role: "orchestrator", + stage: "agents_md_write", + reason: err instanceof Error ? err.message : String(err), + }, + }); await cleanupWorktreeAndMetadata(systemPromptFile); throw err; } @@ -1614,6 +1804,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM }); } } catch (err) { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator opencode session resolution failed", + data: { + role: "orchestrator", + stage: "opencode_session_reuse", + reason: err instanceof Error ? err.message : String(err), + }, + }); await cleanupWorktreeAndMetadata(systemPromptFile); throw err; } @@ -1671,6 +1874,22 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM }, }); } catch (err) { + // Outer envelope catches and emits session.spawn_failed; this inner emit + // tags the runtime.create failure path specifically so RCA can answer + // "did the orchestrator runtime fail to start at all?". + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator runtime.create failed", + data: { + role: "orchestrator", + stage: "runtime_create", + reason: err instanceof Error ? err.message : String(err), + }, + }); await cleanupWorktreeAndMetadata(systemPromptFile); throw err; } @@ -1758,6 +1977,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM } invalidateCache(); } catch (err) { + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.spawn_failed", + level: "error", + summary: "orchestrator post-launch metadata write failed", + data: { + role: "orchestrator", + stage: "post_launch_metadata", + reason: err instanceof Error ? err.message : String(err), + }, + }); // Clean up runtime on post-launch failure try { await plugins.runtime.destroy(handle); @@ -1828,6 +2060,16 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM throw err; } + recordActivityEvent({ + projectId: orchestratorConfig.projectId, + sessionId, + source: "session-manager", + kind: "session.orchestrator_conflict", + level: "warn", + summary: "concurrent orchestrator reservation conflict", + data: { reason: err instanceof Error ? err.message : String(err) }, + }); + const concurrent = await waitForConcurrentOrchestrator(sessionId); if (concurrent) return concurrent; throw err; @@ -1927,21 +2169,45 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM session.lifecycle.session.state !== "terminated" && session.lifecycle.session.state !== "done" ) { + const runtimeStateBefore = session.lifecycle.runtime.state; + const runtimeReasonBefore = session.lifecycle.runtime.reason; try { const persisted = buildUpdatedLifecycle(sessionName, raw, (next) => { next.session.state = "terminated"; next.session.reason = "runtime_lost"; next.session.terminatedAt = new Date().toISOString(); next.session.lastTransitionAt = next.session.terminatedAt; - next.runtime.state = session.lifecycle!.runtime.state; - next.runtime.reason = session.lifecycle!.runtime.reason; + next.runtime.state = runtimeStateBefore; + next.runtime.reason = runtimeReasonBefore; next.runtime.lastObservedAt = new Date().toISOString(); }); + // B1: persist BEFORE emitting the event updateMetadata(sessionsDir, sessionName, lifecycleMetadataUpdates(raw, persisted)); session.lifecycle = persisted; session.status = deriveLegacyStatus(persisted); - } catch { + recordActivityEvent({ + projectId: sessionProjectId, + sessionId: sessionName, + source: "session-manager", + kind: "runtime.lost_detected", + level: "warn", + summary: `runtime lost reconciled: ${sessionName}`, + data: { + runtimeState: runtimeStateBefore, + runtimeReason: runtimeReasonBefore, + }, + }); + } catch (err) { // Persist failed — in-memory state is still correct for this request + recordActivityEvent({ + projectId: sessionProjectId, + sessionId: sessionName, + source: "session-manager", + kind: "runtime.lost_persist_failed", + level: "error", + summary: `runtime_lost persist failed: ${sessionName}`, + data: { reason: err instanceof Error ? err.message : String(err) }, + }); } } @@ -2049,6 +2315,17 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM const killReason: LifecycleKillReason = options?.reason ?? "manually_killed"; const cleanupAgent = resolveSelectionForSession(project, sessionId, raw).agentName; + // Emit kill_started up-front — this is the only signal that the kill + // intent reached the manager (the destroys below are silent on failure). + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.kill_started", + summary: `kill started: ${sessionId}`, + data: { reason: killReason }, + }); + // Destroy runtime — prefer handle.runtimeName to find the correct plugin if (raw["runtimeHandle"]) { const handle = safeJsonParse(raw["runtimeHandle"]); @@ -2061,8 +2338,20 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM if (runtimePlugin) { try { await runtimePlugin.destroy(handle); - } catch { - // Runtime might already be gone + } catch (err) { + // Runtime might already be gone — surface as AE so leaks are queryable. + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "runtime.destroy_failed", + level: "warn", + summary: `runtime.destroy failed during kill: ${sessionId}`, + data: { + runtime: handle.runtimeName ?? null, + reason: err instanceof Error ? err.message : String(err), + }, + }); } } } @@ -2076,8 +2365,21 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM if (workspacePlugin) { try { await workspacePlugin.destroy(worktree); - } catch { - // Workspace might already be gone + } catch (err) { + // Workspace might already be gone — emit AE so abandoned worktrees + // surface for cleanup tooling. + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "workspace.destroy_failed", + level: "warn", + summary: `workspace.destroy failed during kill: ${sessionId}`, + data: { + workspace: workspacePlugin.name, + reason: err instanceof Error ? err.message : String(err), + }, + }); } } } @@ -2095,8 +2397,20 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM try { await deleteOpenCodeSession(mappedOpenCodeSessionId); didPurgeOpenCodeSession = true; - } catch { - void 0; + } catch (err) { + // Dangling opencode session is a real leak — surface for RCA. + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "agent.opencode_purge_failed", + level: "warn", + summary: `opencode session purge failed: ${sessionId}`, + data: { + opencodeSessionId: mappedOpenCodeSessionId, + reason: err instanceof Error ? err.message : String(err), + }, + }); } } } @@ -2229,9 +2543,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM pushSkipped(session.projectId, session.id); } } catch (err) { + const errorMessage = err instanceof Error ? err.message : String(err); result.errors.push({ sessionId: session.id, - error: err instanceof Error ? err.message : String(err), + error: errorMessage, + }); + recordActivityEvent({ + projectId: session.projectId, + sessionId: session.id, + source: "session-manager", + kind: "session.cleanup_error", + level: "warn", + summary: `cleanup error: ${session.id}`, + data: { reason: errorMessage }, }); } } @@ -2275,9 +2599,22 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM opencodeCleanedAt: new Date().toISOString(), })); } catch (err) { + const errorMessage = err instanceof Error ? err.message : String(err); result.errors.push({ sessionId: terminatedId, - error: `Failed to delete OpenCode session ${mappedOpenCodeSessionId}: ${err instanceof Error ? err.message : String(err)}`, + error: `Failed to delete OpenCode session ${mappedOpenCodeSessionId}: ${errorMessage}`, + }); + recordActivityEvent({ + projectId: projectKey, + sessionId: terminatedId, + source: "session-manager", + kind: "agent.opencode_purge_failed", + level: "warn", + summary: `opencode session purge failed during cleanup: ${terminatedId}`, + data: { + opencodeSessionId: mappedOpenCodeSessionId, + reason: errorMessage, + }, }); continue; } @@ -2308,7 +2645,7 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM } async function send(sessionId: SessionId, message: string): Promise { - const { raw, sessionsDir, project } = requireSessionRecord(sessionId); + const { raw, sessionsDir, project, projectId } = requireSessionRecord(sessionId); const selection = resolveSelectionForSession(project, sessionId, raw); const selectedAgent = selection.agentName; @@ -2469,6 +2806,15 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM return restored; } catch (err) { const detail = err instanceof Error ? err.message : String(err); + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.restore_failed", + level: "error", + summary: `restore for delivery failed: ${sessionId}`, + data: { reason: detail, trigger: "send" }, + }); throw new Error(`Cannot send to session ${sessionId}: ${reason} (${detail})`, { cause: err, }); @@ -2564,29 +2910,52 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM return; }; - let prepared = await prepareSession(); - + // Top-level try/catch: any final send failure (initial preparation, + // retry-with-restore, etc.) emits a single `session.send_failed` event + // (B16 — failure-only). Stage tag distinguishes which branch failed. + let stage: "prepare" | "initial" | "restore_retry" = "prepare"; try { - await sendWithConfirmation(prepared); - } catch (err) { - const shouldRetryWithRestore = prepared.restoredAt === undefined && isRestorable(prepared); + let prepared = await prepareSession(); - if (!shouldRetryWithRestore) { - if (err instanceof Error) { - throw err; - } - throw new Error(String(err), { cause: err }); - } - - prepared = await prepareSession(true); try { + stage = "initial"; await sendWithConfirmation(prepared); - } catch (retryErr) { - if (retryErr instanceof Error) { - throw retryErr; + } catch (err) { + const shouldRetryWithRestore = + prepared.restoredAt === undefined && isRestorable(prepared); + + if (!shouldRetryWithRestore) { + if (err instanceof Error) { + throw err; + } + throw new Error(String(err), { cause: err }); + } + + stage = "restore_retry"; + prepared = await prepareSession(true); + try { + await sendWithConfirmation(prepared); + } catch (retryErr) { + if (retryErr instanceof Error) { + throw retryErr; + } + throw new Error(String(retryErr), { cause: retryErr }); } - throw new Error(String(retryErr), { cause: retryErr }); } + } catch (err) { + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.send_failed", + level: "error", + summary: `send failed: ${sessionId}`, + data: { + stage, + reason: err instanceof Error ? err.message : String(err), + }, + }); + throw err; } } @@ -2779,6 +3148,20 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM const reason = NON_RESTORABLE_STATUSES.has(session.status) ? `status "${session.status}" is not restorable` : `session is not in a terminal state (status: "${session.status}", activity: "${session.activity}")`; + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.restore_failed", + level: "error", + summary: `restore not allowed: ${sessionId}`, + data: { + stage: "validation", + status: session.status, + activity: session.activity, + reason, + }, + }); throw new SessionNotRestorableError(sessionId, reason); } @@ -2821,6 +3204,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM await plugins.workspace.postCreate(wsInfo, project); } } catch (err) { + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.restore_failed", + level: "error", + summary: `workspace restore failed: ${sessionId}`, + data: { + stage: "workspace_restore", + workspacePath, + reason: err instanceof Error ? err.message : String(err), + }, + }); throw new WorkspaceMissingError( workspacePath, `restore failed: ${err instanceof Error ? err.message : String(err)}`, @@ -2911,8 +3307,31 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM restoreFallbackReason: reason, }); if (requiresNativeRestore(plugins.agent.name)) { + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.restore_failed", + level: "error", + summary: `native restore required but unavailable: ${sessionId}`, + data: { + stage: "native_restore_required", + agent: plugins.agent.name, + reason, + }, + }); throw new SessionNotRestorableError(sessionId, reason); } + // Surface that AO fell back to a fresh launch instead of native restore. + recordActivityEvent({ + projectId, + sessionId, + source: "session-manager", + kind: "session.restore_fallback", + level: "warn", + summary: `using fresh launch instead of native restore: ${sessionId}`, + data: { agent: plugins.agent.name, reason }, + }); launchCommand = plugins.agent.getLaunchCommand(agentLaunchConfig); } } else {