bkinnightskytw
/
agent-orchestrator
mirror of https://github.com/ComposioHQ/agent-orchestrator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into pr-1698 

# Conflicts:
#	packages/core/src/activity-events.ts
This commit is contained in:
whoisasx 2026-05-18 19:58:39 +05:30
parent 1ab942901c c1e43f61dc
commit 4920d267f2
49 changed files with 5418 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.changeset/activity-events-webhooks-mux.md Normal file
View File

@ -0,0 +1,18 @@
---
"@aoagents/ao-core": minor
"@aoagents/ao-web": minor
---
Wire activity events into webhook ingress and the mux WebSocket terminal server (sub-issue of #1511, follows #1620).
- `api.webhook_unverified` (warn) — signature verification failed; data includes `slug`, `remoteAddr`, `candidateCount` (never the failed signature)
- `api.webhook_rejected` (warn) — payload exceeded `maxBodyBytes`; data includes counts and `maxBodyBytes` (never the body)
- `api.webhook_received` (info|warn) — accepted webhook; data includes `projectIds`, `matchedSessions`, `parseErrorCount`, `lifecycleErrorCount` (never the body)
- `api.webhook_failed` (error) — outer pipeline crash with `errorMessage`
- `ui.terminal_connected` / `ui.terminal_disconnected` — one event per mux WS connection lifecycle
- `ui.terminal_heartbeat_lost` (warn) — fires once on 3 missed pongs (was console-only)
- `ui.terminal_pty_lost` (warn) — fires when PTY exits with subscribers attached (distinguishes "PTY died" from "user closed browser")
- `ui.terminal_protocol_error` (warn) — invalid mux client message
- `ui.session_broadcast_failed` (warn) — emitted on the healthy→failing transition only (re-arms after a successful poll), so a long outage produces one event, not 20/min
`api.webhook_unverified` is the security-audit event; treat 401s on webhooks as a signal worth retaining for the full 7-day window.

6
.changeset/issue-1660-recovery-metadata-events.md Normal file
View File

@ -0,0 +1,6 @@
---
"@aoagents/ao-cli": patch
"@aoagents/ao-core": minor
---
Wire activity events for the recovery subsystem, metadata-corruption detection, and agent-report apply path. New event kinds: `recovery.session_failed`, `recovery.action_failed`, `metadata.corrupt_detected`, `api.agent_report.session_not_found`, `api.agent_report.transition_rejected`. Adds `"recovery"` to the `ActivityEventSource` union. Lets RCA reconstruct `ao recover` invocations, find every silent metadata overwrite, and audit rejected agent transitions. Adds `ao events list --source` and `--kind` so these forensic event queries are available from the CLI.

87
packages/cli/__tests__/commands/events.test.ts Normal file
View File

@ -0,0 +1,87 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { Command } from "commander";
const { mockQueryActivityEvents, mockSearchActivityEvents, mockGetActivityEventStats } = vi.hoisted(
() => ({
mockQueryActivityEvents: vi.fn(),
mockSearchActivityEvents: vi.fn(),
mockGetActivityEventStats: vi.fn(),
}),
);
vi.mock("@aoagents/ao-core", () => ({
queryActivityEvents: (...args: unknown[]) => mockQueryActivityEvents(...args),
searchActivityEvents: (...args: unknown[]) => mockSearchActivityEvents(...args),
getActivityEventStats: (...args: unknown[]) => mockGetActivityEventStats(...args),
droppedEventCount: () => 0,
isActivityEventsFtsEnabled: () => true,
}));
import { registerEvents } from "../../src/commands/events.js";
describe("events command", () => {
let program: Command;
let consoleLogSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
program = new Command();
program.exitOverride();
registerEvents(program);
consoleLogSpy = vi.spyOn(console, "log").mockImplementation(() => {});
mockQueryActivityEvents.mockReset();
mockSearchActivityEvents.mockReset();
mockGetActivityEventStats.mockReset();
mockQueryActivityEvents.mockReturnValue([]);
});
afterEach(() => {
consoleLogSpy.mockRestore();
});
it("filters list output by source and --kind alias", async () => {
await program.parseAsync([
"node",
"test",
"events",
"list",
"--source",
"recovery",
"--kind",
"metadata.corrupt_detected",
"--limit",
"1",
"--json",
]);
expect(mockQueryActivityEvents).toHaveBeenCalledWith(
expect.objectContaining({
source: "recovery",
kind: "metadata.corrupt_detected",
limit: 1,
}),
);
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('"source": "recovery"'));
expect(consoleLogSpy).toHaveBeenCalledWith(
expect.stringContaining('"kind": "metadata.corrupt_detected"'),
);
});
it("keeps --type as the existing event-kind filter", async () => {
await program.parseAsync([
"node",
"test",
"events",
"list",
"--type",
"recovery.session_failed",
"--json",
]);
expect(mockQueryActivityEvents).toHaveBeenCalledWith(
expect.objectContaining({
kind: "recovery.session_failed",
}),
);
});
});

240
packages/cli/__tests__/lib/project-supervisor.test.ts
View File

@ -9,6 +9,12 @@ const mockSetHealth = vi.fn();
const activeWorkers = new Set<string>();
vi.mock("@aoagents/ao-core", () => ({
ConfigNotFoundError: class ConfigNotFoundError extends Error {
constructor(message = "No agent-orchestrator.yaml found.") {
super(message);
this.name = "ConfigNotFoundError";
}
},
createCorrelationId: () => "correlation-id",
createProjectObserver: () => ({ setHealth: (...args: unknown[]) => mockSetHealth(...args) }),
getGlobalConfigPath: () => "/tmp/global-config.yaml",
@ -67,9 +73,9 @@ import {
stopProjectSupervisor,
} from "../../src/lib/project-supervisor.js";
function makeConfig(projectIds: string[]) {
function makeConfig(projectIds: string[], configPath = "/tmp/global-config.yaml") {
return {
configPath: "/tmp/global-config.yaml",
configPath,
projects: Object.fromEntries(projectIds.map((id) => [id, { name: id, path: `/tmp/${id}` }])),
};
}
@ -223,7 +229,7 @@ describe("project-supervisor", () => {
},
});
const startPromise = startProjectSupervisor(1_000);
const startPromise = startProjectSupervisor({ intervalMs: 1_000 });
await vi.waitFor(() => expect(releaseList).toBeDefined());
stopProjectSupervisor();
@ -242,7 +248,7 @@ describe("project-supervisor", () => {
throw new Error("bad config");
});
await expect(startProjectSupervisor(1_000)).rejects.toThrow("bad config");
await expect(startProjectSupervisor({ intervalMs: 1_000 })).rejects.toThrow("bad config");
});
it("allows startup when the global config does not exist yet", async () => {
@ -257,7 +263,7 @@ describe("project-supervisor", () => {
throw error;
});
const handle = await startProjectSupervisor(1_000);
const handle = await startProjectSupervisor({ intervalMs: 1_000 });
expect(handle).toEqual({
stop: expect.any(Function),
@ -266,10 +272,230 @@ describe("project-supervisor", () => {
handle.stop();
});
it("falls back to local config when the global config is missing (ENOENT)", async () => {
// The local fallback uses a DIFFERENT configPath than the global —
// a real bare `loadConfig()` discovers the local file and sets
// `config.configPath` to that path. Asserting on the local path here
// catches any bug that would propagate the global path through the
// fallback (e.g. accidentally returning the global config object).
const localConfigPath = "/tmp/cwd/agent-orchestrator.yaml";
sessionsByProject.set("app", [makeSession("app")]);
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/global-config.yaml",
});
}
return makeConfig(["app"], localConfigPath);
});
await reconcileProjectSupervisor();
expect(mockLoadConfig).toHaveBeenCalledWith("/tmp/global-config.yaml");
expect(mockLoadConfig).toHaveBeenCalledWith();
expect(mockEnsureLifecycleWorker).toHaveBeenCalledWith(
expect.objectContaining({ configPath: localConfigPath }),
"app",
undefined,
);
expect(activeWorkers.has("app")).toBe(true);
});
it("uses the caller-provided configPath as the local fallback when global is missing", async () => {
sessionsByProject.set("app", [makeSession("app")]);
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/global-config.yaml",
});
}
if (path === "/some/repo/agent-orchestrator.yaml") {
return makeConfig(["app"]);
}
throw new Error(`unexpected loadConfig path: ${path}`);
});
await reconcileProjectSupervisor({ configPath: "/some/repo/agent-orchestrator.yaml" });
expect(mockLoadConfig).toHaveBeenCalledWith("/tmp/global-config.yaml");
expect(mockLoadConfig).toHaveBeenCalledWith("/some/repo/agent-orchestrator.yaml");
// No bare cwd-walk when the caller resolved a path for us.
expect(mockLoadConfig).not.toHaveBeenCalledWith();
expect(activeWorkers.has("app")).toBe(true);
});
it("ignores the caller-provided configPath when the global config is healthy", async () => {
sessionsByProject.set("app", [makeSession("app")]);
// Both paths would return a valid config — assert we only ever consult
// the global path. The configPath is the fallback, not an override.
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") return makeConfig(["app"]);
if (path === "/repo/agent-orchestrator.yaml") {
throw new Error("supervisor should not consult configPath when global is healthy");
}
throw new Error(`unexpected loadConfig path: ${path}`);
});
await reconcileProjectSupervisor({ configPath: "/repo/agent-orchestrator.yaml" });
expect(mockLoadConfig).toHaveBeenCalledWith("/tmp/global-config.yaml");
expect(mockLoadConfig).not.toHaveBeenCalledWith("/repo/agent-orchestrator.yaml");
expect(activeWorkers.has("app")).toBe(true);
});
it("preserves workers across a global→fallback transition (multi-tick)", async () => {
// Tick 1: global exists with {alpha, beta, gamma} — supervisor attaches
// all three. Tick 2: global has been deleted; the local fallback config
// (passed-in configPath) lists only {alpha}. Without the source-aware
// detach skip, the second tick would kill beta and gamma even though
// they're still running real sessions.
sessionsByProject.set("alpha", [makeSession("alpha")]);
sessionsByProject.set("beta", [makeSession("beta")]);
sessionsByProject.set("gamma", [makeSession("gamma")]);
// Tick 1: global is the source.
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") return makeConfig(["alpha", "beta", "gamma"]);
throw new Error(`unexpected path on tick 1: ${path}`);
});
await reconcileProjectSupervisor({ configPath: "/repo/agent-orchestrator.yaml" });
expect(activeWorkers.has("alpha")).toBe(true);
expect(activeWorkers.has("beta")).toBe(true);
expect(activeWorkers.has("gamma")).toBe(true);
// Tick 2: global deleted, fallback has a narrower view.
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/global-config.yaml",
});
}
if (path === "/repo/agent-orchestrator.yaml") return makeConfig(["alpha"]);
throw new Error(`unexpected path on tick 2: ${path}`);
});
await reconcileProjectSupervisor({ configPath: "/repo/agent-orchestrator.yaml" });
// All three workers must survive — fallback isn't authoritative for removal.
expect(activeWorkers.has("alpha")).toBe(true);
expect(activeWorkers.has("beta")).toBe(true);
expect(activeWorkers.has("gamma")).toBe(true);
expect(mockRemoveProjectFromRunning).not.toHaveBeenCalledWith("beta");
expect(mockRemoveProjectFromRunning).not.toHaveBeenCalledWith("gamma");
});
it("does detach when global is restored after a fallback period (symmetric flip)", async () => {
// Documents intentional current behavior: when source flips back to
// "global", the detach pass treats the global config as authoritative,
// so projects not listed there ARE detached — including any that were
// attached during a prior fallback window. The reviewer's guidance was
// scoped to the fallback direction; protecting the symmetric flip would
// require per-worker source tracking and is out of scope here.
sessionsByProject.set("local-only", [makeSession("local-only")]);
sessionsByProject.set("from-global", [makeSession("from-global")]);
// Tick 1: no global, fallback attaches local-only.
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/global-config.yaml",
});
}
if (path === "/repo/agent-orchestrator.yaml") return makeConfig(["local-only"]);
throw new Error(`unexpected path on tick 1: ${path}`);
});
await reconcileProjectSupervisor({ configPath: "/repo/agent-orchestrator.yaml" });
expect(activeWorkers.has("local-only")).toBe(true);
// Tick 2: global appears (e.g. another `ao start <url>` wrote it),
// listing only "from-global". Source = "global" → detach pass runs.
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") return makeConfig(["from-global"]);
throw new Error(`unexpected path on tick 2: ${path}`);
});
await reconcileProjectSupervisor({ configPath: "/repo/agent-orchestrator.yaml" });
expect(activeWorkers.has("from-global")).toBe(true);
expect(activeWorkers.has("local-only")).toBe(false);
expect(mockRemoveProjectFromRunning).toHaveBeenCalledWith("local-only");
});
it("does not detach unrelated active workers when operating from local fallback", async () => {
// Simulates: daemon already supervising "other-project" (registered via
// a prior reconcile against a global config that has since been deleted).
// The current reconcile sees only "cwd-project" in the local fallback —
// it must NOT treat "other-project" as removed.
activeWorkers.add("other-project");
sessionsByProject.set("cwd-project", [makeSession("cwd-project")]);
mockLoadConfig.mockImplementation((path?: string) => {
if (path === "/tmp/global-config.yaml") {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/global-config.yaml",
});
}
return makeConfig(["cwd-project"]);
});
await reconcileProjectSupervisor({ configPath: "/repo/agent-orchestrator.yaml" });
expect(activeWorkers.has("other-project")).toBe(true);
expect(mockRemoveProjectFromRunning).not.toHaveBeenCalledWith("other-project");
// Attach pass still runs for the configured cwd project.
expect(activeWorkers.has("cwd-project")).toBe(true);
});
it("rethrows ENOENT from a nested file referenced by the global config", async () => {
mockLoadConfig.mockImplementation(() => {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/some-referenced-file.yaml",
});
});
await expect(reconcileProjectSupervisor()).rejects.toThrow("ENOENT");
expect(mockLoadConfig).toHaveBeenCalledTimes(1);
});
it("rethrows non-missing-config errors from the global config load", async () => {
mockLoadConfig.mockImplementation(() => {
throw new Error("invalid yaml");
});
await expect(reconcileProjectSupervisor()).rejects.toThrow("invalid yaml");
expect(mockLoadConfig).toHaveBeenCalledTimes(1);
});
it("exits cleanly when neither global nor local config exists", async () => {
const { ConfigNotFoundError } = await import("@aoagents/ao-core");
mockLoadConfig
.mockImplementationOnce(() => {
throw Object.assign(new Error("ENOENT"), {
code: "ENOENT",
path: "/tmp/global-config.yaml",
});
})
.mockImplementationOnce(() => {
throw new ConfigNotFoundError();
});
const handle = await startProjectSupervisor({ intervalMs: 1_000 });
expect(handle).toEqual({
stop: expect.any(Function),
reconcileNow: expect.any(Function),
});
expect(mockEnsureLifecycleWorker).not.toHaveBeenCalled();
handle.stop();
});
it("forwards the supervisor interval to lifecycle workers it starts", async () => {
sessionsByProject.set("app", [makeSession("app")]);
const handle = await startProjectSupervisor(1_234);
const handle = await startProjectSupervisor({ intervalMs: 1_234 });
expect(mockEnsureLifecycleWorker).toHaveBeenCalledWith(
expect.objectContaining({ configPath: "/tmp/global-config.yaml" }),
@ -280,7 +506,7 @@ describe("project-supervisor", () => {
});
it("reconcileNow waits for a queued reconcile when one is already running", async () => {
const handle = await startProjectSupervisor(1_000);
const handle = await startProjectSupervisor({ intervalMs: 1_000 });
let firstRelease: (() => void) | undefined;
let secondRelease: (() => void) | undefined;
let listCalls = 0;

21
packages/cli/src/commands/events.ts
View File

@ -9,6 +9,7 @@ import {
type ActivityEvent,
type ActivityEventLevel,
type ActivityEventKind,
type ActivityEventSource,
} from "@aoagents/ao-core";
interface JsonEnvelope {
@ -80,7 +81,12 @@ export function registerEvents(program: Command): void {
.description("List recent activity events")
.option("-p, --project <id>", "Filter by project ID")
.option("-s, --session <id>", "Filter by session ID")
.option("-t, --type <kind>", "Filter by event kind (e.g. session.spawned, lifecycle.transition)")
.option(
"-t, --type <kind>",
"Filter by event kind (e.g. session.spawned, lifecycle.transition)",
)
.option("--kind <kind>", "Alias for --type")
.option("--source <source>", "Filter by event source (e.g. lifecycle, recovery, api)")
.option("--log-level <level>", "Filter by log level (debug, info, warn, error)")
.option("--since <duration>", "Show events from last N minutes/hours/days (e.g. 30m, 2h, 1d)")
.option("-n, --limit <n>", "Max results", "50")
@ -91,15 +97,21 @@ export function registerEvents(program: Command): void {
if (sinceRaw) {
since = parseSinceDuration(sinceRaw);
if (!since) {
console.error(chalk.yellow(`Warning: unrecognised --since format "${sinceRaw}" (use e.g. 30m, 2h, 1d). No time filter applied.`));
console.error(
chalk.yellow(
`Warning: unrecognised --since format "${sinceRaw}" (use e.g. 30m, 2h, 1d). No time filter applied.`,
),
);
}
}
const limit = parseInt(opts["limit"] ?? "50", 10);
const kind = opts["type"] ?? opts["kind"];
const results = queryActivityEvents({
projectId: opts["project"],
sessionId: opts["session"],
kind: opts["type"] as ActivityEventKind,
kind: kind as ActivityEventKind,
source: opts["source"] as ActivityEventSource,
level: opts["logLevel"] as ActivityEventLevel,
since,
limit,
@ -111,7 +123,8 @@ export function registerEvents(program: Command): void {
query: {
projectId: opts["project"] ?? null,
sessionId: opts["session"] ?? null,
kind: opts["type"] ?? null,
kind: kind ?? null,
source: opts["source"] ?? null,
level: opts["logLevel"] ?? null,
since: sinceRaw ?? null,
limit,

2
packages/cli/src/commands/start.ts
View File

@ -985,7 +985,7 @@ async function runStartup(
if (shouldStartLifecycle) {
try {
spinner.start("Starting project supervisor");
await startProjectSupervisor();
await startProjectSupervisor({ configPath: config.configPath });
spinner.succeed("Lifecycle project supervisor started");
} catch (err) {
spinner.fail("Project supervisor failed to start");

104
packages/cli/src/lib/project-supervisor.ts
View File

@ -4,6 +4,7 @@ import {
isTerminalSession,
createCorrelationId,
createProjectObserver,
ConfigNotFoundError,
type OrchestratorConfig,
type ProjectObserver,
} from "@aoagents/ao-core";
@ -24,11 +25,36 @@ interface SupervisorHandle {
let activeSupervisor: SupervisorHandle | null = null;
export interface ReconcileProjectSupervisorOptions {
intervalMs?: number;
type SupervisorConfigSource = "global" | "local-fallback";
interface LoadedSupervisorConfig {
config: OrchestratorConfig;
source: SupervisorConfigSource;
}
function isMissingGlobalConfigError(error: unknown): boolean {
export interface ReconcileProjectSupervisorOptions {
intervalMs?: number;
/**
* Resolved config path from the caller (typically `ao start`). When the
* global config is missing, this is used as the explicit local-fallback
* source. Without it the supervisor would fall back to a cwd-walk via
* bare `loadConfig()`, which misses configs in `ao start <url>` /
* `ao start <path>` first-run flows — there the resolved config can
* live under the clone/target path while the daemon's cwd is somewhere
* else. A bare cwd-walk in that case throws ConfigNotFoundError, which
* `run()` silently swallows, leaving `running.projects` empty.
*/
configPath?: string;
}
export interface StartProjectSupervisorOptions {
intervalMs?: number;
/** See {@link ReconcileProjectSupervisorOptions.configPath}. */
configPath?: string;
}
function isMissingConfigError(error: unknown): boolean {
if (error instanceof ConfigNotFoundError) return true;
return (
error instanceof Error &&
"code" in error &&
@ -38,6 +64,29 @@ function isMissingGlobalConfigError(error: unknown): boolean {
);
}
/** Load the supervisor config: prefer the global registry, fall back to the
* caller-resolved local config path (or cwd discovery when none provided).
* Returns the source so callers can gate authoritative actions (like the
* detach pass) on whether we're looking at the real registry. */
function loadSupervisorConfig(configPath?: string): LoadedSupervisorConfig {
const globalConfigPath = getGlobalConfigPath();
try {
return { config: loadConfig(globalConfigPath), source: "global" };
} catch (error) {
if (
error instanceof Error &&
"code" in error &&
error.code === "ENOENT" &&
"path" in error &&
error.path === globalConfigPath
) {
const config = configPath ? loadConfig(configPath) : loadConfig();
return { config, source: "local-fallback" };
}
throw error;
}
}
function reportProjectSupervisorError(
observer: ProjectObserver,
projectId: string,
@ -69,23 +118,31 @@ async function projectHasNonTerminalSession(
export async function reconcileProjectSupervisor(
options: ReconcileProjectSupervisorOptions = {},
): Promise<void> {
const config = loadConfig(getGlobalConfigPath());
const { config, source } = loadSupervisorConfig(options.configPath);
const observer = createProjectObserver(config, "project-supervisor");
const configuredProjectIds = new Set(Object.keys(config.projects));
const activeProjectIds = new Set(listLifecycleWorkers());
for (const projectId of activeProjectIds) {
if (!configuredProjectIds.has(projectId)) {
try {
stopLifecycleWorker(projectId);
await removeProjectFromRunning(projectId);
} catch (error) {
reportProjectSupervisorError(
observer,
projectId,
"Failed to detach lifecycle worker for removed project",
error,
);
// Only the authoritative global registry can declare a project "removed".
// On a local fallback (e.g. global config was deleted while the daemon is
// already supervising multiple projects) the loaded config likely doesn't
// enumerate every supervised project — running the detach pass would kill
// unrelated lifecycle workers. Pre-fallback behavior was a no-op on
// missing global; preserve that property for the detach pass specifically.
if (source === "global") {
const activeProjectIds = new Set(listLifecycleWorkers());
for (const projectId of activeProjectIds) {
if (!configuredProjectIds.has(projectId)) {
try {
stopLifecycleWorker(projectId);
await removeProjectFromRunning(projectId);
} catch (error) {
reportProjectSupervisorError(
observer,
projectId,
"Failed to detach lifecycle worker for removed project",
error,
);
}
}
}
}
@ -117,16 +174,19 @@ export async function reconcileProjectSupervisor(
}
export async function startProjectSupervisor(
intervalMs: number = DEFAULT_SUPERVISOR_INTERVAL_MS,
options: StartProjectSupervisorOptions = {},
): Promise<SupervisorHandle> {
if (activeSupervisor) return activeSupervisor;
const intervalMs = options.intervalMs ?? DEFAULT_SUPERVISOR_INTERVAL_MS;
const configPath = options.configPath;
let reconciling = false;
let pending = false;
let stopped = false;
let waiters: Array<() => void> = [];
const run = async (options: { swallowErrors?: boolean } = {}): Promise<void> => {
const run = async (runOptions: { swallowErrors?: boolean } = {}): Promise<void> => {
if (stopped) return;
if (reconciling) {
pending = true;
@ -140,10 +200,10 @@ export async function startProjectSupervisor(
do {
pending = false;
try {
await reconcileProjectSupervisor({ intervalMs });
await reconcileProjectSupervisor({ intervalMs, configPath });
} catch (error) {
if (isMissingGlobalConfigError(error)) return;
if (!options.swallowErrors) throw error;
if (isMissingConfigError(error)) return;
if (!runOptions.swallowErrors) throw error;
// Best-effort background loop: transient config/state errors should not crash ao start.
}
} while (pending && !stopped);

80
packages/core/src/__tests__/activity-events-agent-report.test.ts Normal file
View File

@ -0,0 +1,80 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import { applyAgentReport } from "../agent-report.js";
import { writeMetadata, writeCanonicalLifecycle } from "../metadata.js";
import { createInitialCanonicalLifecycle } from "../lifecycle-state.js";
import { recordActivityEvent } from "../activity-events.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
let dataDir: string;
let sessionId: string;
beforeEach(() => {
dataDir = join(tmpdir(), `ao-test-agent-report-events-${randomUUID()}`);
mkdirSync(dataDir, { recursive: true });
sessionId = "ao-1";
vi.mocked(recordActivityEvent).mockClear();
});
afterEach(() => {
rmSync(dataDir, { recursive: true, force: true });
});
describe("activity events: agent-report", () => {
it("emits api.agent_report.transition_rejected when the transition is invalid", () => {
const lifecycle = createInitialCanonicalLifecycle("worker");
lifecycle.session.state = "terminated";
writeMetadata(dataDir, sessionId, {
worktree: "/tmp/worktree",
branch: "feat/x",
status: "terminated",
project: "demo",
});
writeCanonicalLifecycle(dataDir, sessionId, lifecycle);
expect(() =>
applyAgentReport(dataDir, sessionId, {
state: "working",
now: new Date(),
}),
).toThrow();
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const rejected = calls.find(
(c) => c.kind === "api.agent_report.transition_rejected",
);
expect(rejected).toBeDefined();
expect(rejected?.sessionId).toBe(sessionId);
expect(rejected?.source).toBe("api");
});
it("does not emit transition_rejected on a valid transition", () => {
const lifecycle = createInitialCanonicalLifecycle("worker");
lifecycle.session.state = "working";
lifecycle.session.reason = "task_in_progress";
lifecycle.session.startedAt = "2024-12-01T00:00:00.000Z";
lifecycle.runtime.state = "alive";
lifecycle.runtime.reason = "process_running";
writeMetadata(dataDir, sessionId, {
worktree: "/tmp/worktree",
branch: "feat/x",
status: "working",
project: "demo",
});
writeCanonicalLifecycle(dataDir, sessionId, lifecycle);
applyAgentReport(dataDir, sessionId, {
state: "needs_input",
now: new Date(),
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
expect(calls.find((c) => c.kind === "api.agent_report.transition_rejected")).toBeUndefined();
});
});

205
packages/core/src/__tests__/activity-events-config.test.ts Normal file
View File

@ -0,0 +1,205 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { loadConfig } from "../config.js";
import { recordActivityEvent } from "../activity-events.js";
import { saveGlobalConfig, type GlobalConfig } from "../global-config.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
function makeGlobalConfig(projects: GlobalConfig["projects"] = {}): GlobalConfig {
return {
port: 3000,
readyThresholdMs: 300_000,
defaults: { runtime: "tmux", agent: "claude-code", workspace: "worktree", notifiers: [] },
projects,
notifiers: {},
notificationRouting: {},
reactions: {},
};
}
describe("activity events: config loading", () => {
let tempRoot: string;
let configPath: string;
let originalHome: string | undefined;
let originalGlobalConfig: string | undefined;
beforeEach(() => {
tempRoot = join(
tmpdir(),
`ao-config-events-${Date.now()}-${Math.random().toString(16).slice(2)}`,
);
configPath = join(tempRoot, ".agent-orchestrator", "config.yaml");
mkdirSync(tempRoot, { recursive: true });
originalHome = process.env["HOME"];
originalGlobalConfig = process.env["AO_GLOBAL_CONFIG"];
process.env["HOME"] = tempRoot;
process.env["AO_GLOBAL_CONFIG"] = configPath;
vi.mocked(recordActivityEvent).mockClear();
});
afterEach(() => {
process.env["HOME"] = originalHome;
if (originalGlobalConfig === undefined) {
delete process.env["AO_GLOBAL_CONFIG"];
} else {
process.env["AO_GLOBAL_CONFIG"] = originalGlobalConfig;
}
rmSync(tempRoot, { recursive: true, force: true });
});
it("emits config.project_resolve_failed for unresolved projects without a specific config event", () => {
const projectPath = join(tempRoot, "old-format");
mkdirSync(projectPath, { recursive: true });
writeFileSync(
join(projectPath, "agent-orchestrator.yaml"),
["projects:", " old-format:", " path: .", ""].join("\n"),
);
saveGlobalConfig(
makeGlobalConfig({
"old-format": {
projectId: "old-format",
path: projectPath,
displayName: "Old format",
defaultBranch: "main",
sessionPrefix: "old-format",
},
}),
configPath,
);
loadConfig(configPath);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "config",
kind: "config.project_resolve_failed",
projectId: "old-format",
}),
);
});
it("does not emit config.project_resolve_failed for malformed local yaml", () => {
const projectPath = join(tempRoot, "malformed");
mkdirSync(projectPath, { recursive: true });
writeFileSync(join(projectPath, "agent-orchestrator.yaml"), "tracker: [\n");
saveGlobalConfig(
makeGlobalConfig({
malformed: {
projectId: "malformed",
path: projectPath,
displayName: "Malformed",
defaultBranch: "main",
sessionPrefix: "malformed",
},
}),
configPath,
);
loadConfig(configPath);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "config",
kind: "config.project_malformed",
projectId: "malformed",
}),
);
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
expect(calls.find((c) => c.kind === "config.project_resolve_failed")).toBeUndefined();
});
it("does not emit config.project_resolve_failed for healthy projects", () => {
const projectPath = join(tempRoot, "clean");
mkdirSync(projectPath, { recursive: true });
writeFileSync(
join(projectPath, "agent-orchestrator.yaml"),
["agent: codex", "runtime: tmux", "workspace: worktree", ""].join("\n"),
);
saveGlobalConfig(
makeGlobalConfig({
clean: {
projectId: "clean",
path: projectPath,
displayName: "Clean",
defaultBranch: "main",
sessionPrefix: "clean",
},
}),
configPath,
);
loadConfig(configPath);
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
expect(calls.find((c) => c.kind === "config.project_resolve_failed")).toBeUndefined();
});
it("emits config.project_malformed for unparseable local yaml", () => {
const projectPath = join(tempRoot, "malformed");
mkdirSync(projectPath, { recursive: true });
writeFileSync(join(projectPath, "agent-orchestrator.yaml"), "tracker: [\n");
saveGlobalConfig(
makeGlobalConfig({
malformed: {
projectId: "malformed",
path: projectPath,
displayName: "Malformed",
defaultBranch: "main",
sessionPrefix: "malformed",
},
}),
configPath,
);
loadConfig(configPath);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "config",
kind: "config.project_malformed",
projectId: "malformed",
}),
);
});
it("emits config.project_invalid for schema-invalid local yaml", () => {
const projectPath = join(tempRoot, "invalid");
mkdirSync(projectPath, { recursive: true });
// Valid YAML but fails LocalProjectConfigSchema (numeric agent isn't a string)
writeFileSync(join(projectPath, "agent-orchestrator.yaml"), "agent: 123\n");
saveGlobalConfig(
makeGlobalConfig({
invalid: {
projectId: "invalid",
path: projectPath,
displayName: "Invalid",
defaultBranch: "main",
sessionPrefix: "invalid",
},
}),
configPath,
);
loadConfig(configPath);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "config",
kind: "config.project_invalid",
projectId: "invalid",
}),
);
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
expect(calls.find((c) => c.kind === "config.project_resolve_failed")).toBeUndefined();
});
});

223
packages/core/src/__tests__/activity-events-migration.test.ts Normal file
View File

@ -0,0 +1,223 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { execSync } from "node:child_process";
import { migrateStorage } from "../migration/storage-v2.js";
import { recordActivityEvent } from "../activity-events.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal();
return {
...(actual as Record<string, unknown>),
execSync: vi.fn(),
};
});
function createTempDir(): string {
const dir = join(
tmpdir(),
`ao-migration-events-${Date.now()}-${Math.random().toString(36).slice(2)}`,
);
mkdirSync(dir, { recursive: true });
return dir;
}
describe("activity events: storage migration", () => {
let testDir: string;
let aoBaseDir: string;
let configPath: string;
beforeEach(() => {
testDir = createTempDir();
aoBaseDir = join(testDir, ".agent-orchestrator");
configPath = join(aoBaseDir, "config.yaml");
mkdirSync(aoBaseDir, { recursive: true });
vi.mocked(recordActivityEvent).mockClear();
vi.mocked(execSync).mockReturnValue("");
});
afterEach(() => {
rmSync(testDir, { recursive: true, force: true });
});
it("emits migration.completed when there is nothing to migrate", async () => {
const result = await migrateStorage({
aoBaseDir,
globalConfigPath: configPath,
force: true,
log: () => {},
});
expect(result.projects).toBe(0);
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const completed = calls.find((c) => c.kind === "migration.completed");
expect(completed).toBeDefined();
expect(completed?.source).toBe("migration");
expect(completed?.level).toBe("info");
expect(completed?.data).toMatchObject({
projectsMigrated: 0,
sessions: 0,
worktrees: 0,
projectErrors: 0,
});
});
it("emits migration.completed with totals when migration succeeds", async () => {
const hashDir = join(aoBaseDir, "aaaaaa000000-myproject");
mkdirSync(join(hashDir, "sessions"), { recursive: true });
mkdirSync(join(hashDir, "worktrees", "ao-1"), { recursive: true });
writeFileSync(
join(hashDir, "sessions", "ao-1"),
[
"project=myproject",
"agent=claude-code",
"status=working",
"createdAt=2026-04-21T12:00:00.000Z",
"branch=session/ao-1",
`worktree=${join(hashDir, "worktrees", "ao-1")}`,
].join("\n"),
);
writeFileSync(
configPath,
[
"projects:",
" myproject:",
" path: /home/user/myproject",
" storageKey: aaaaaa000000",
" defaultBranch: main",
"",
].join("\n"),
);
await migrateStorage({
aoBaseDir,
globalConfigPath: configPath,
force: true,
log: () => {},
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const completed = calls.find((c) => c.kind === "migration.completed");
expect(completed).toBeDefined();
const data = (completed?.data ?? {}) as Record<string, unknown>;
expect(data["projectsMigrated"]).toBe(1);
expect(data["projectErrors"]).toBe(0);
});
it("emits migration.project_failed plus migration.completed when a project fails", async () => {
// Force migrateProject to throw by pre-creating projects/badproj as a FILE
// — mkdirSync with recursive:true tolerates existing directories but NOT existing
// files at the same path, so it throws EEXIST when migrateProject tries to create
// projects/badproj/sessions.
const badDir = join(aoBaseDir, "bbbbbb000000-badproj");
mkdirSync(join(badDir, "sessions"), { recursive: true });
writeFileSync(
join(badDir, "sessions", "ao-99"),
[
"project=badproj",
"agent=claude-code",
"createdAt=2026-04-21T12:00:00.000Z",
"branch=session/ao-99",
`worktree=${join(badDir, "worktrees", "ao-99")}`,
].join("\n"),
);
// Pre-create projects/badproj as a regular file so mkdirSync inside migrateProject
// raises ENOTDIR / EEXIST when it tries to create a subdirectory under it.
mkdirSync(join(aoBaseDir, "projects"), { recursive: true });
writeFileSync(join(aoBaseDir, "projects", "badproj"), "blocker");
writeFileSync(
configPath,
[
"projects:",
" badproj:",
" path: /home/user/badproj",
" storageKey: bbbbbb000000",
" defaultBranch: main",
"",
].join("\n"),
);
await migrateStorage({
aoBaseDir,
globalConfigPath: configPath,
force: true,
log: () => {},
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const projectFailed = calls.filter((c) => c.kind === "migration.project_failed");
const completed = calls.find((c) => c.kind === "migration.completed");
expect(projectFailed.length).toBeGreaterThanOrEqual(1);
expect(projectFailed[0]?.projectId).toBe("badproj");
expect(completed).toBeDefined();
const completedData = (completed?.data ?? {}) as Record<string, unknown>;
expect(completedData["projectErrors"]).toBeGreaterThanOrEqual(1);
});
it("emits migration.blocked when active sessions are detected", async () => {
vi.mocked(execSync).mockReturnValue("ao-1\nabcdef012345-worker-7\nunrelated\n");
writeFileSync(
configPath,
"projects:\n myproject:\n path: /tmp/x\n storageKey: aaaaaa000000\n defaultBranch: main\n",
);
await expect(migrateStorage({
aoBaseDir,
globalConfigPath: configPath,
log: () => {},
})).rejects.toThrow(/Found 2 active AO tmux session/);
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const blocked = calls.find((c) => c.kind === "migration.blocked");
expect(blocked).toBeDefined();
expect(blocked?.source).toBe("migration");
expect(blocked?.level).toBe("warn");
expect(blocked?.summary).toBe("migration blocked by 2 active session(s)");
expect(blocked?.data).toMatchObject({
activeSessionCount: 2,
sample: ["ao-1", "abcdef012345-worker-7"],
});
});
it("does not emit migration.blocked when force skips active-session detection", async () => {
vi.mocked(execSync).mockReturnValue("ao-1\n");
const hashDir = join(aoBaseDir, "aaaaaa000000-myproject");
mkdirSync(join(hashDir, "sessions"), { recursive: true });
writeFileSync(
join(hashDir, "sessions", "ao-1"),
[
"project=myproject",
"agent=claude-code",
"createdAt=2026-04-21T12:00:00.000Z",
"branch=session/ao-1",
`worktree=${join(hashDir, "worktrees", "ao-1")}`,
].join("\n"),
);
mkdirSync(join(hashDir, "worktrees", "ao-1"), { recursive: true });
writeFileSync(
configPath,
"projects:\n myproject:\n path: /tmp/x\n storageKey: aaaaaa000000\n defaultBranch: main\n",
);
// force: true skips the active-session check, so no migration.blocked event.
await migrateStorage({
aoBaseDir,
globalConfigPath: configPath,
force: true,
log: () => {},
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
expect(calls.find((c) => c.kind === "migration.blocked")).toBeUndefined();
});
});

129
packages/core/src/__tests__/activity-events-plugin-registry.test.ts Normal file
View File

@ -0,0 +1,129 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createPluginRegistry } from "../plugin-registry.js";
import { recordActivityEvent } from "../activity-events.js";
import type { OrchestratorConfig, PluginManifest, PluginModule } from "../types.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
function makePlugin(slot: PluginManifest["slot"], name: string): PluginModule {
return {
manifest: {
name,
slot,
description: `Test ${slot} plugin: ${name}`,
version: "0.0.1",
},
create: vi.fn(() => ({ name })),
};
}
function makeOrchestratorConfig(overrides?: Partial<OrchestratorConfig>): OrchestratorConfig {
return {
projects: {},
...overrides,
} as OrchestratorConfig;
}
beforeEach(() => {
vi.mocked(recordActivityEvent).mockClear();
});
describe("activity events: plugin-registry", () => {
it("emits plugin-registry.load_failed when a configured external plugin fails to import", async () => {
const registry = createPluginRegistry();
const config = makeOrchestratorConfig({
plugins: [
{
name: "broken-plugin",
source: "npm",
package: "@example/broken",
enabled: true,
},
],
});
await registry.loadFromConfig(config, async (pkg: string) => {
// Built-ins all silently skip; external import throws
if (pkg === "@example/broken") {
throw new Error("module not found");
}
throw new Error(`builtin not installed: ${pkg}`);
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const loadFailed = calls.find(
(c) =>
c.kind === "plugin-registry.load_failed" &&
c.source === "plugin-registry" &&
(c.data as Record<string, unknown> | undefined)?.["builtin"] === false,
);
expect(loadFailed).toBeDefined();
});
it("emits plugin-registry.specifier_failed when a plugin specifier cannot be resolved", async () => {
const registry = createPluginRegistry();
const config = makeOrchestratorConfig({
plugins: [
{
name: "no-specifier",
source: "local",
// No path — resolvePluginSpecifier returns null
enabled: true,
},
],
});
await registry.loadFromConfig(config, async (pkg: string) => {
throw new Error(`builtin not installed: ${pkg}`);
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const specifierFailed = calls.find(
(c) => c.kind === "plugin-registry.specifier_failed",
);
expect(specifierFailed).toBeDefined();
});
it("emits plugin-registry.load_failed when a built-in plugin's register() throws", async () => {
const registry = createPluginRegistry();
// Make a plugin whose create() throws to force registration failure
const fakeRuntime: PluginModule = {
manifest: {
name: "tmux",
slot: "runtime",
description: "throwing test plugin",
version: "0.0.1",
},
create: () => {
throw new Error("boom");
},
};
await registry.loadBuiltins(undefined, async (pkg: string) => {
if (pkg === "@aoagents/ao-plugin-runtime-tmux") return fakeRuntime;
throw new Error(`Not found: ${pkg}`);
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const loadFailed = calls.find((c) => c.kind === "plugin-registry.load_failed");
expect(loadFailed).toBeDefined();
});
it("does not emit any failure events when plugins load cleanly", async () => {
const registry = createPluginRegistry();
const fakePlugin = makePlugin("runtime", "tmux");
await registry.loadBuiltins(undefined, async (pkg: string) => {
if (pkg === "@aoagents/ao-plugin-runtime-tmux") return fakePlugin;
throw new Error(`Not found: ${pkg}`);
});
const calls = vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0]);
const failures = calls.filter((c) =>
typeof c.kind === "string" && c.kind.startsWith("plugin-registry."),
);
expect(failures).toEqual([]);
});
});

28
packages/core/src/__tests__/agent-report.test.ts
View File

@ -1,4 +1,4 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { mkdirSync, readFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
@ -18,17 +18,25 @@ import {
} from "../agent-report.js";
import { writeMetadata, writeCanonicalLifecycle, readMetadataRaw } from "../metadata.js";
import { createInitialCanonicalLifecycle } from "../lifecycle-state.js";
import { recordActivityEvent } from "../activity-events.js";
import type { CanonicalSessionLifecycle } from "../types.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
let dataDir: string;
let tempRoot: string;
beforeEach(() => {
dataDir = join(tmpdir(), `ao-test-agent-report-${randomUUID()}`);
tempRoot = join(tmpdir(), `ao-test-agent-report-${randomUUID()}`);
dataDir = join(tempRoot, "projects", "project-alpha", "sessions");
mkdirSync(dataDir, { recursive: true });
vi.mocked(recordActivityEvent).mockClear();
});
afterEach(() => {
rmSync(dataDir, { recursive: true, force: true });
rmSync(tempRoot, { recursive: true, force: true });
});
function seedWorkerSession(
@ -433,6 +441,13 @@ describe("applyAgentReport", () => {
sessionState: "terminated",
},
});
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
projectId: "project-alpha",
sessionId,
kind: "api.agent_report.transition_rejected",
}),
);
});
it("throws when the session does not exist", () => {
@ -442,6 +457,13 @@ describe("applyAgentReport", () => {
now: new Date(),
}),
).toThrow(/not found/);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
projectId: "project-alpha",
sessionId: "missing-session",
kind: "api.agent_report.session_not_found",
}),
);
});
// 260 atomic-write cycles are slow on Windows (rename + AV scan); bump the

139
packages/core/src/__tests__/metadata.test.ts
View File

@ -1,5 +1,6 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { mkdirSync, rmSync, readFileSync, existsSync, writeFileSync, readdirSync } from "node:fs";
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { mkdirSync, rmSync, readFileSync, existsSync, writeFileSync, readdirSync, renameSync } from "node:fs";
import type * as NodeFs from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
@ -13,12 +14,29 @@ import {
deleteMetadata,
listMetadata,
} from "../metadata.js";
import { recordActivityEvent } from "../activity-events.js";
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof NodeFs>();
return {
...actual,
renameSync: vi.fn((...args: Parameters<typeof actual.renameSync>) =>
actual.renameSync(...args),
),
};
});
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
let dataDir: string;
beforeEach(() => {
dataDir = join(tmpdir(), `ao-test-metadata-${randomUUID()}`);
mkdirSync(dataDir, { recursive: true });
vi.mocked(recordActivityEvent).mockClear();
vi.mocked(renameSync).mockClear();
});
afterEach(() => {
@ -390,6 +408,123 @@ describe("mutateMetadata corrupt-file handling", () => {
const corruptCopies = readdirSync(dataDir).filter((f) => f.includes(".corrupt-"));
expect(corruptCopies).toHaveLength(0);
});
it("emits metadata.corrupt_detected when JSON parse fails and file is renamed", () => {
const sessionPath = join(dataDir, "ao-3.json");
writeFileSync(sessionPath, "{ broken json", "utf-8");
const result = mutateMetadata(
dataDir,
"ao-3",
(existing) => ({ ...existing, branch: "feat/x" }),
{ createIfMissing: true },
);
expect(result).not.toBeNull();
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
sessionId: "ao-3",
source: "session-manager",
kind: "metadata.corrupt_detected",
level: "error",
summary: expect.stringContaining("renamed to"),
data: expect.objectContaining({
renamedTo: expect.stringContaining(`${sessionPath}.corrupt-`),
renameSucceeded: true,
contentSample: "{ broken json",
path: sessionPath,
}),
}),
);
});
it("emits a rename-failed summary when corrupt metadata cannot be renamed", () => {
const sessionPath = join(dataDir, "ao-rename-failed.json");
writeFileSync(sessionPath, "{ broken json", "utf-8");
vi.mocked(renameSync).mockImplementationOnce(() => {
throw new Error("rename denied");
});
const result = mutateMetadata(
dataDir,
"ao-rename-failed",
(existing) => ({ ...existing, branch: "feat/x" }),
{ createIfMissing: true },
);
expect(result).not.toBeNull();
const call = vi
.mocked(recordActivityEvent)
.mock.calls.find((c) => c[0].kind === "metadata.corrupt_detected");
expect(call).toBeDefined();
expect(call![0]).toMatchObject({
sessionId: "ao-rename-failed",
summary: expect.stringContaining("failed to rename"),
data: expect.objectContaining({
renamedTo: null,
renameSucceeded: false,
path: sessionPath,
}),
});
expect(call![0].summary).not.toContain("renamed to");
});
it("uses the provided source for metadata.corrupt_detected", () => {
const sessionPath = join(dataDir, "ao-api-source.json");
writeFileSync(sessionPath, "{ broken json", "utf-8");
mutateMetadata(
dataDir,
"ao-api-source",
(existing) => ({ ...existing, branch: "feat/api" }),
{ createIfMissing: true, activityEventSource: "api" },
);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
sessionId: "ao-api-source",
source: "api",
kind: "metadata.corrupt_detected",
}),
);
});
it("truncates contentSample to 200 chars in metadata.corrupt_detected", () => {
const sessionPath = join(dataDir, "ao-4.json");
// 250 char garbage payload — sanitizer cap is 16KB but invariant B11 caps
// forensic sample at 200 chars.
const huge = "x".repeat(250);
writeFileSync(sessionPath, huge, "utf-8");
mutateMetadata(
dataDir,
"ao-4",
(existing) => ({ ...existing, branch: "feat/y" }),
{ createIfMissing: true },
);
const call = vi
.mocked(recordActivityEvent)
.mock.calls.find((c) => c[0].kind === "metadata.corrupt_detected");
expect(call).toBeDefined();
const sample = (call![0].data as Record<string, unknown>)["contentSample"] as string;
expect(sample.length).toBe(200);
expect((call![0].data as Record<string, unknown>)["contentLength"]).toBe(250);
});
it("does not emit metadata.corrupt_detected for healthy JSON", () => {
writeMetadata(dataDir, "ao-5", {
worktree: "/tmp/w",
branch: "main",
status: "working",
});
mutateMetadata(dataDir, "ao-5", (existing) => ({ ...existing, summary: "hi" }));
const corruptCalls = vi
.mocked(recordActivityEvent)
.mock.calls.filter((c) => c[0].kind === "metadata.corrupt_detected");
expect(corruptCalls).toHaveLength(0);
});
});
describe("readCanonicalLifecycle", () => {

269
packages/core/src/__tests__/recovery-manager.test.ts Normal file
View File

@ -0,0 +1,269 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import { recoverSessionById, runRecovery } from "../recovery/manager.js";
import { recordActivityEvent } from "../activity-events.js";
import { getProjectDir, getProjectSessionsDir } from "../paths.js";
import {
DEFAULT_RECOVERY_CONFIG,
type RecoveryAssessment,
type RecoveryResult,
} from "../recovery/types.js";
import * as actionsModule from "../recovery/actions.js";
import * as validatorModule from "../recovery/validator.js";
import type { OrchestratorConfig, PluginRegistry } from "../types.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
const PROJECT_ID = "app";
function makeConfig(rootDir: string): OrchestratorConfig {
return {
configPath: join(rootDir, "agent-orchestrator.yaml"),
port: 3000,
readyThresholdMs: 300_000,
power: { preventIdleSleep: false },
defaults: {
runtime: "tmux",
agent: "claude-code",
workspace: "worktree",
notifiers: ["desktop"],
},
projects: {
app: {
name: "app",
repo: "org/repo",
path: join(rootDir, "project"),
defaultBranch: "main",
sessionPrefix: "app",
},
},
notifiers: {},
notificationRouting: {
urgent: ["desktop"],
action: ["desktop"],
warning: ["desktop"],
info: ["desktop"],
},
reactions: {},
};
}
function makeRegistry(): PluginRegistry {
return {
register: vi.fn(),
get: vi.fn().mockReturnValue(null),
list: vi.fn().mockReturnValue([]),
loadBuiltins: vi.fn().mockResolvedValue(undefined),
loadFromConfig: vi.fn().mockResolvedValue(undefined),
};
}
function makeAssessment(sessionId: string): RecoveryAssessment {
return {
sessionId,
projectId: PROJECT_ID,
classification: "live",
action: "recover",
reason: "needs recovery",
runtimeProbeSucceeded: true,
processProbeSucceeded: true,
signalDisagreement: false,
recoveryRule: "auto",
runtimeAlive: true,
runtimeHandle: null,
workspaceExists: true,
workspacePath: "/tmp/worktree",
agentProcessRunning: true,
agentActivity: "active",
metadataValid: true,
metadataStatus: "working",
rawMetadata: { project: PROJECT_ID, status: "working" },
};
}
describe("runRecovery activity events", () => {
let rootDir: string;
let previousHome: string | undefined;
beforeEach(() => {
rootDir = join(tmpdir(), `ao-recovery-events-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
mkdirSync(join(rootDir, "project"), { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
previousHome = process.env["HOME"];
process.env["HOME"] = rootDir;
const sessionsDir = getProjectSessionsDir(PROJECT_ID);
mkdirSync(sessionsDir, { recursive: true });
writeFileSync(
join(sessionsDir, "app-1.json"),
JSON.stringify({ project: PROJECT_ID, status: "working" }) + "\n",
"utf-8",
);
writeFileSync(
join(sessionsDir, "app-2.json"),
JSON.stringify({ project: PROJECT_ID, status: "working" }) + "\n",
"utf-8",
);
vi.mocked(recordActivityEvent).mockClear();
vi.restoreAllMocks();
});
afterEach(() => {
if (previousHome === undefined) {
delete process.env["HOME"];
} else {
process.env["HOME"] = previousHome;
}
if (rootDir) {
const projectBaseDir = getProjectDir(PROJECT_ID);
if (existsSync(projectBaseDir)) {
rmSync(projectBaseDir, { recursive: true, force: true });
}
rmSync(rootDir, { recursive: true, force: true });
}
vi.restoreAllMocks();
});
it("emits recovery.session_failed for each session that recovery couldn't fix", async () => {
vi.spyOn(validatorModule, "validateSession").mockImplementation(async (scanned) =>
makeAssessment(scanned.sessionId),
);
const successResult: RecoveryResult = {
success: true,
sessionId: "app-1",
action: "recover",
};
const failedResult: RecoveryResult = {
success: false,
sessionId: "app-2",
action: "recover",
error: "worktree missing",
};
vi.spyOn(actionsModule, "executeAction")
.mockResolvedValueOnce(successResult)
.mockResolvedValueOnce(failedResult);
const config = makeConfig(rootDir);
const registry = makeRegistry();
const { report } = await runRecovery({
config,
registry,
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: join(rootDir, "recovery.log"),
},
});
expect(report.errors).toHaveLength(1);
expect(report.errors[0]?.sessionId).toBe("app-2");
const emitCalls = vi
.mocked(recordActivityEvent)
.mock.calls.map((c) => c[0])
.filter((e) => e.kind === "recovery.session_failed");
expect(emitCalls).toHaveLength(1);
expect(emitCalls[0]).toEqual(
expect.objectContaining({
sessionId: "app-2",
projectId: PROJECT_ID,
source: "recovery",
kind: "recovery.session_failed",
level: "error",
data: expect.objectContaining({
action: "recover",
errorMessage: "worktree missing",
}),
}),
);
});
it("emits recovery.session_failed when single-session recovery fails", async () => {
vi.spyOn(validatorModule, "validateSession").mockImplementation(async (scanned) =>
makeAssessment(scanned.sessionId),
);
const failedResult: RecoveryResult = {
success: false,
sessionId: "app-2",
action: "recover",
error: "agent process missing",
};
vi.spyOn(actionsModule, "executeAction").mockResolvedValueOnce(failedResult);
const config = makeConfig(rootDir);
const registry = makeRegistry();
const result = await recoverSessionById("app-2", {
config,
registry,
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: join(rootDir, "recovery.log"),
},
});
expect(result).toEqual(failedResult);
const emitCalls = vi
.mocked(recordActivityEvent)
.mock.calls.map((c) => c[0])
.filter((e) => e.kind === "recovery.session_failed");
expect(emitCalls).toHaveLength(1);
expect(emitCalls[0]).toEqual(
expect.objectContaining({
sessionId: "app-2",
projectId: PROJECT_ID,
source: "recovery",
kind: "recovery.session_failed",
level: "error",
data: expect.objectContaining({
action: "recover",
errorMessage: "agent process missing",
}),
}),
);
});
it("does not emit recovery.session_failed when every session recovers cleanly", async () => {
vi.spyOn(validatorModule, "validateSession").mockImplementation(async (scanned) =>
makeAssessment(scanned.sessionId),
);
vi.spyOn(actionsModule, "executeAction").mockImplementation(async (assessment) => ({
success: true,
sessionId: assessment.sessionId,
action: "recover",
}));
const config = makeConfig(rootDir);
const registry = makeRegistry();
await runRecovery({
config,
registry,
recoveryConfig: {
...DEFAULT_RECOVERY_CONFIG,
logPath: join(rootDir, "recovery.log"),
},
});
const failedEmits = vi
.mocked(recordActivityEvent)
.mock.calls.map((c) => c[0])
.filter((e) => e.kind === "recovery.session_failed");
expect(failedEmits).toHaveLength(0);
});
});

635
packages/core/src/__tests__/session-manager-instrumentation.test.ts Normal file
View File

@ -0,0 +1,635 @@
/**
* Regression tests for session-manager activity event instrumentation
* (issue #1657 — extends PR #1620 to cover the rest of the failure paths
* inside spawn / kill / restore / send / list).
*
* One test per MUST-class emit. Pattern follows the lifecycle-manager-
* instrumentation tests: mock `recordActivityEvent`, drive the manager into
* the failure path, then assert the right kind/level/data was logged.
*
* Invariants asserted by these tests (PR #1620 B1/B2 plus #1657 B25):
* - state mutation happens BEFORE event emission
* - failure-only emits — no event on a successful send/spawn
* - cleanup-stack rollbacks emit per failed step (not in aggregate)
*/
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { writeFileSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import { createSessionManager } from "../session-manager.js";
import { writeMetadata, readMetadataRaw } from "../metadata.js";
import { recordActivityEvent } from "../activity-events.js";
import { getProjectWorktreesDir } from "../paths.js";
import type { OrchestratorConfig, PluginRegistry, Agent } from "../types.js";
import {
setupTestContext,
teardownTestContext,
makeHandle,
type TestContext,
} from "./test-utils.js";
vi.mock("../activity-events.js", () => ({
recordActivityEvent: vi.fn(),
}));
let ctx: TestContext;
let sessionsDir: string;
let mockRegistry: PluginRegistry;
let config: OrchestratorConfig;
beforeEach(() => {
ctx = setupTestContext();
({ sessionsDir, mockRegistry, config } = ctx);
vi.mocked(recordActivityEvent).mockClear();
});
afterEach(() => {
teardownTestContext(ctx);
vi.restoreAllMocks();
vi.useRealTimers();
});
function findEvent(kind: string) {
return vi
.mocked(recordActivityEvent)
.mock.calls.map((c) => c[0])
.find((e) => e.kind === kind);
}
function findAllEvents(kind: string) {
return vi
.mocked(recordActivityEvent)
.mock.calls.map((c) => c[0])
.filter((e) => e.kind === kind);
}
function writeTerminatedSession(
sessionId: string,
options: { worktree: string; branch?: string },
): void {
const metadata: Record<string, unknown> = {
worktree: options.worktree,
status: "killed",
project: "my-app",
runtimeHandle: makeHandle("rt-old"),
lifecycle: {
version: 2,
session: {
kind: "worker",
state: "terminated",
reason: "manually_killed",
startedAt: "2025-01-01T00:00:00.000Z",
completedAt: null,
terminatedAt: "2025-01-01T00:00:00.000Z",
lastTransitionAt: "2025-01-01T00:00:00.000Z",
},
pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null },
runtime: {
state: "missing",
reason: "process_missing",
lastObservedAt: "2025-01-01T00:00:00.000Z",
handle: null,
tmuxName: null,
},
},
};
if (options.branch !== undefined) {
metadata["branch"] = options.branch;
}
writeMetadata(sessionsDir, sessionId, metadata as unknown as Parameters<typeof writeMetadata>[2]);
}
describe("session.kill_started (MUST)", () => {
it("emits before runtime.destroy is attempted", async () => {
let destroyCalled = false;
let killStartedEmittedBeforeDestroy = false;
vi.mocked(ctx.mockRuntime.destroy).mockImplementation(async () => {
destroyCalled = true;
killStartedEmittedBeforeDestroy = !!findEvent("session.kill_started");
});
writeMetadata(sessionsDir, "app-killed", {
worktree: "/tmp/ws",
branch: "main",
status: "working",
project: "my-app",
runtimeHandle: makeHandle("rt-1"),
});
const sm = createSessionManager({ config, registry: mockRegistry });
await sm.kill("app-killed");
expect(destroyCalled).toBe(true);
expect(killStartedEmittedBeforeDestroy).toBe(true);
const start = findEvent("session.kill_started");
expect(start).toMatchObject({
projectId: "my-app",
sessionId: "app-killed",
source: "session-manager",
kind: "session.kill_started",
});
});
it("does not emit kill_started when session is already terminated (idempotent)", async () => {
writeMetadata(sessionsDir, "app-already-killed", {
worktree: "/tmp/ws",
branch: "main",
status: "killed",
project: "my-app",
lifecycle: {
version: 2,
session: {
kind: "worker",
state: "terminated",
reason: "manually_killed",
startedAt: "2025-01-01T00:00:00.000Z",
completedAt: null,
terminatedAt: "2025-01-01T00:00:00.000Z",
lastTransitionAt: "2025-01-01T00:00:00.000Z",
},
pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null },
runtime: {
state: "missing",
reason: "manual_kill_requested",
lastObservedAt: "2025-01-01T00:00:00.000Z",
handle: null,
tmuxName: null,
},
},
});
const sm = createSessionManager({ config, registry: mockRegistry });
await sm.kill("app-already-killed");
expect(findEvent("session.kill_started")).toBeUndefined();
});
});
describe("session.spawn_failed — orchestrator path (MUST)", () => {
it("emits session.spawned after a successful orchestrator spawn", async () => {
const sm = createSessionManager({ config, registry: mockRegistry });
const session = await sm.spawnOrchestrator({
projectId: "my-app",
systemPrompt: "be helpful",
});
const events = findAllEvents("session.spawned");
const orchestratorSpawned = events.find(
(e) => e.data && (e.data as Record<string, unknown>)["role"] === "orchestrator",
);
expect(session.id).toBe("app-orchestrator");
expect(orchestratorSpawned).toMatchObject({
projectId: "my-app",
sessionId: "app-orchestrator",
source: "session-manager",
kind: "session.spawned",
summary: "spawned: app-orchestrator",
data: {
agent: "mock-agent",
branch: "orchestrator/app-orchestrator",
role: "orchestrator",
},
});
});
it("does not emit terminal spawn_failed when ensure recovers a fixed reservation conflict", async () => {
let releaseWorkspace: () => void = () => {};
const blockingWorkspace = new Promise<void>((resolve) => {
releaseWorkspace = resolve;
});
vi.mocked(ctx.mockWorkspace.create).mockImplementationOnce(async (cfg) => {
await blockingWorkspace;
return {
path: join(ctx.tmpDir, "ws-orchestrator"),
branch: cfg.branch,
sessionId: cfg.sessionId,
projectId: cfg.projectId,
};
});
const firstManager = createSessionManager({ config, registry: mockRegistry });
const secondManager = createSessionManager({ config, registry: mockRegistry });
const firstEnsure = firstManager.ensureOrchestrator({
projectId: "my-app",
systemPrompt: "be helpful",
});
await vi.waitFor(() => {
expect(ctx.mockWorkspace.create).toHaveBeenCalledTimes(1);
});
const secondEnsure = secondManager.ensureOrchestrator({
projectId: "my-app",
systemPrompt: "be helpful",
});
await vi.waitFor(() => {
expect(findEvent("session.orchestrator_conflict")).toBeDefined();
});
releaseWorkspace();
const [created, recovered] = await Promise.all([firstEnsure, secondEnsure]);
expect(created.id).toBe("app-orchestrator");
expect(recovered.id).toBe("app-orchestrator");
expect(findAllEvents("session.orchestrator_conflict")).toHaveLength(1);
expect(
findAllEvents("session.spawn_failed").filter(
(e) => e.data && (e.data as Record<string, unknown>)["role"] === "orchestrator",
),
).toHaveLength(0);
});
it("emits one terminal failure plus one stage failure when workspace.create throws", async () => {
vi.mocked(ctx.mockWorkspace.create).mockRejectedValue(new Error("disk full"));
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(
sm.spawnOrchestrator({ projectId: "my-app", systemPrompt: "be helpful" }),
).rejects.toThrow("disk full");
const events = findAllEvents("session.spawn_failed");
expect(events).toHaveLength(1);
const orchestratorFailure = events.find(
(e) => e.data && (e.data as Record<string, unknown>)["role"] === "orchestrator",
);
expect(orchestratorFailure).toBeDefined();
expect(orchestratorFailure!.level).toBe("error");
expect(orchestratorFailure!.projectId).toBe("my-app");
const stepEvents = findAllEvents("session.spawn_step_failed");
expect(stepEvents).toHaveLength(1);
expect(stepEvents[0]!.sessionId).toBe("app-orchestrator");
expect(stepEvents[0]!.data).toMatchObject({
role: "orchestrator",
stage: "workspace_create",
});
});
it("emits one terminal failure plus one stage failure when runtime.create throws", async () => {
vi.mocked(ctx.mockRuntime.create).mockRejectedValue(new Error("tmux not found"));
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(
sm.spawnOrchestrator({ projectId: "my-app", systemPrompt: "be helpful" }),
).rejects.toThrow("tmux not found");
const events = findAllEvents("session.spawn_failed");
expect(events).toHaveLength(1);
const orchestratorFailure = events.find(
(e) => e.data && (e.data as Record<string, unknown>)["role"] === "orchestrator",
);
expect(orchestratorFailure).toBeDefined();
expect(orchestratorFailure!.level).toBe("error");
const stepEvents = findAllEvents("session.spawn_step_failed");
expect(stepEvents).toHaveLength(1);
expect(stepEvents[0]!.sessionId).toBe("app-orchestrator");
expect(stepEvents[0]!.data).toMatchObject({
role: "orchestrator",
stage: "runtime_create",
});
});
});
describe("session.rollback_started/session.rollback_step_failed (MUST)", () => {
it("includes reserved sessionId when worker spawn rolls back after reservation", async () => {
const workspacePath = join(getProjectWorktreesDir("my-app"), "app-1");
vi.mocked(ctx.mockWorkspace.create).mockResolvedValue({
path: workspacePath,
branch: "feat/test",
sessionId: "app-1",
projectId: "my-app",
});
vi.mocked(ctx.mockWorkspace.destroy).mockRejectedValue(new Error("destroy failed"));
vi.mocked(ctx.mockRuntime.create).mockRejectedValue(new Error("runtime failed"));
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.spawn({ projectId: "my-app" })).rejects.toThrow("runtime failed");
const rollbackStarted = findEvent("session.rollback_started");
expect(rollbackStarted).toBeDefined();
expect(rollbackStarted!.sessionId).toBe("app-1");
const rollbackStepFailed = findEvent("session.rollback_step_failed");
expect(rollbackStepFailed).toBeDefined();
expect(rollbackStepFailed!.sessionId).toBe("app-1");
expect(rollbackStepFailed!.data).toMatchObject({ reason: "destroy failed" });
});
});
describe("session.workspace_hooks_failed (MUST)", () => {
it("emits when setupWorkspaceHooks throws during orchestrator spawn", async () => {
const hookFailingAgent: Agent = {
...ctx.mockAgent,
name: "hook-failing-agent",
setupWorkspaceHooks: vi.fn().mockRejectedValue(new Error("settings.json EACCES")),
};
const originalGet = mockRegistry.get;
mockRegistry.get = vi.fn().mockImplementation((slot: string, name?: string) => {
if (slot === "agent") return hookFailingAgent;
return (originalGet as any)(slot, name);
});
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(
sm.spawnOrchestrator({ projectId: "my-app", systemPrompt: "be helpful" }),
).rejects.toThrow("settings.json EACCES");
const event = findEvent("session.workspace_hooks_failed");
expect(event).toBeDefined();
expect(event!.level).toBe("error");
expect(event!.projectId).toBe("my-app");
});
});
describe("runtime.lost_detected (MUST)", () => {
it("emits when sm.list() persists runtime_lost for a dead runtime", async () => {
writeMetadata(sessionsDir, "app-dead", {
worktree: "/tmp/ws",
branch: "main",
status: "working",
project: "my-app",
runtimeHandle: makeHandle("rt-dead"),
});
// Runtime claims dead, agent process gone
vi.mocked(ctx.mockRuntime.isAlive).mockResolvedValue(false);
vi.mocked(ctx.mockAgent.isProcessRunning).mockResolvedValue(false);
const sm = createSessionManager({ config, registry: mockRegistry });
await sm.list();
const event = findEvent("runtime.lost_detected");
expect(event).toBeDefined();
expect(event!.projectId).toBe("my-app");
expect(event!.sessionId).toBe("app-dead");
expect(event!.level).toBe("warn");
// B1: state mutation BEFORE event emission — verify metadata was persisted
const persisted = readMetadataRaw(sessionsDir, "app-dead");
expect(persisted).not.toBeNull();
const lifecycleStr = persisted!["lifecycle"];
expect(lifecycleStr).toBeDefined();
const lc = JSON.parse(lifecycleStr!) as { session: { state: string; reason: string } };
expect(lc.session.state).toBe("detecting");
expect(lc.session.reason).toBe("runtime_lost");
});
});
describe("session.send_failed (MUST)", () => {
it("emits after send retry-with-restore exhausts", async () => {
writeMetadata(sessionsDir, "app-send", {
worktree: "/tmp/ws",
branch: "main",
status: "killed",
project: "my-app",
runtimeHandle: makeHandle("rt-1"),
lifecycle: {
version: 2,
session: {
kind: "worker",
state: "terminated",
reason: "manually_killed",
startedAt: "2025-01-01T00:00:00.000Z",
completedAt: null,
terminatedAt: "2025-01-01T00:00:00.000Z",
lastTransitionAt: "2025-01-01T00:00:00.000Z",
},
pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null },
runtime: {
state: "missing",
reason: "process_missing",
lastObservedAt: "2025-01-01T00:00:00.000Z",
handle: null,
tmuxName: null,
},
},
});
vi.mocked(ctx.mockRuntime.sendMessage).mockRejectedValue(new Error("send broke"));
// restore() throws so retry-with-restore exhausts
vi.mocked(ctx.mockRuntime.create).mockRejectedValue(new Error("restore failed"));
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.send("app-send", "hi")).rejects.toThrow();
const event = findEvent("session.send_failed");
expect(event).toBeDefined();
expect(event!.level).toBe("error");
expect(event!.sessionId).toBe("app-send");
// B11: data must not contain message content
expect(JSON.stringify(event!.data ?? {})).not.toContain("hi");
});
it("does not double-emit session.restore_failed when restore fails during send", async () => {
const wsPath = join(ctx.tmpDir, "missing-send-ws");
writeTerminatedSession("app-send-restore", { worktree: wsPath, branch: "feat/send" });
ctx.mockWorkspace.exists = vi.fn().mockResolvedValue(false);
ctx.mockWorkspace.restore = vi.fn().mockRejectedValue(new Error("restore failed"));
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.send("app-send-restore", "hi")).rejects.toThrow();
const restoreFailed = findAllEvents("session.restore_failed");
expect(restoreFailed).toHaveLength(1);
expect(restoreFailed[0]!.data).toMatchObject({ stage: "workspace_restore" });
expect(findEvent("session.send_failed")).toBeDefined();
});
it("tags restore-for-delivery timeout restore_failed with ready_timeout stage", async () => {
vi.useFakeTimers();
try {
const wsPath = join(ctx.tmpDir, "send-restore-ready-timeout");
mkdirSync(wsPath, { recursive: true });
writeTerminatedSession("app-send-timeout", { worktree: wsPath, branch: "feat/send" });
vi.mocked(ctx.mockRuntime.isAlive).mockImplementation(async (handle) => {
return handle.id !== "rt-restored";
});
vi.mocked(ctx.mockAgent.isProcessRunning).mockImplementation(async (handle) => {
return handle.id !== "rt-restored";
});
vi.mocked(ctx.mockRuntime.create).mockResolvedValue(makeHandle("rt-restored"));
vi.mocked(ctx.mockRuntime.getOutput).mockResolvedValue("");
vi.mocked(ctx.mockAgent.detectActivity).mockReturnValue("idle");
const sm = createSessionManager({ config, registry: mockRegistry });
const sendPromise = sm.send("app-send-timeout", "hi");
const rejection = expect(sendPromise).rejects.toThrow(
"restored session did not become ready for delivery",
);
await vi.runAllTimersAsync();
await rejection;
const restoreFailed = findAllEvents("session.restore_failed");
expect(restoreFailed).toHaveLength(1);
expect(restoreFailed[0]!.data).toMatchObject({
stage: "ready_timeout",
reason: "restored session did not become ready for delivery",
trigger: "send",
});
expect(findEvent("session.send_failed")).toBeDefined();
} finally {
vi.useRealTimers();
}
});
});
describe("session.restore_failed (MUST)", () => {
it("emits when restore's workspace restore throws", async () => {
const wsPath = join(ctx.tmpDir, "missing-ws");
writeMetadata(sessionsDir, "app-rest", {
worktree: wsPath,
branch: "feat/x",
status: "killed",
project: "my-app",
runtimeHandle: makeHandle("rt-old"),
lifecycle: {
version: 2,
session: {
kind: "worker",
state: "terminated",
reason: "manually_killed",
startedAt: "2025-01-01T00:00:00.000Z",
completedAt: null,
terminatedAt: "2025-01-01T00:00:00.000Z",
lastTransitionAt: "2025-01-01T00:00:00.000Z",
},
pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null },
runtime: {
state: "missing",
reason: "process_missing",
lastObservedAt: "2025-01-01T00:00:00.000Z",
handle: null,
tmuxName: null,
},
},
});
// workspace doesn't exist + restore throws
vi.mocked(ctx.mockWorkspace.exists ?? (() => false)).mockResolvedValue?.(false);
ctx.mockWorkspace.exists = vi.fn().mockResolvedValue(false);
ctx.mockWorkspace.restore = vi.fn().mockRejectedValue(new Error("clone failed"));
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.restore("app-rest")).rejects.toThrow();
const event = findEvent("session.restore_failed");
expect(event).toBeDefined();
expect(event!.level).toBe("error");
expect(event!.sessionId).toBe("app-rest");
});
it("emits SessionNotRestorableError when session is not restorable", async () => {
writeMetadata(sessionsDir, "app-not-rest", {
worktree: "/tmp/ws",
branch: "feat/x",
status: "working",
project: "my-app",
runtimeHandle: makeHandle("rt-1"),
lifecycle: {
version: 2,
session: {
kind: "worker",
state: "working",
reason: "task_in_progress",
startedAt: "2025-01-01T00:00:00.000Z",
completedAt: null,
terminatedAt: null,
lastTransitionAt: "2025-01-01T00:00:00.000Z",
},
pr: { state: "none", reason: "not_created", number: null, url: null, lastObservedAt: null },
runtime: {
state: "alive",
reason: "process_running",
lastObservedAt: "2025-01-01T00:00:00.000Z",
handle: makeHandle("rt-1"),
tmuxName: null,
},
},
});
// active session — not restorable
vi.mocked(ctx.mockRuntime.isAlive).mockResolvedValue(true);
vi.mocked(ctx.mockAgent.isProcessRunning).mockResolvedValue(true);
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.restore("app-not-rest")).rejects.toThrow();
const event = findEvent("session.restore_failed");
expect(event).toBeDefined();
});
it("emits when workspace is missing and the workspace plugin cannot restore", async () => {
const wsPath = join(ctx.tmpDir, "missing-no-restore");
writeTerminatedSession("app-no-restore", { worktree: wsPath, branch: "feat/no-restore" });
ctx.mockWorkspace.exists = vi.fn().mockResolvedValue(false);
ctx.mockWorkspace.restore = undefined;
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.restore("app-no-restore")).rejects.toThrow();
const event = findEvent("session.restore_failed");
expect(event).toBeDefined();
expect(event!.sessionId).toBe("app-no-restore");
expect(event!.data).toMatchObject({
stage: "workspace_restore",
workspacePath: wsPath,
reason: "workspace plugin does not support restore",
});
});
it("emits when workspace is missing and branch metadata is absent", async () => {
const wsPath = join(ctx.tmpDir, "missing-no-branch");
writeTerminatedSession("app-no-branch", { worktree: wsPath });
ctx.mockWorkspace.exists = vi.fn().mockResolvedValue(false);
ctx.mockWorkspace.restore = vi.fn().mockResolvedValue({
path: wsPath,
branch: "unused",
sessionId: "app-no-branch",
projectId: "my-app",
});
const sm = createSessionManager({ config, registry: mockRegistry });
await expect(sm.restore("app-no-branch")).rejects.toThrow();
const event = findEvent("session.restore_failed");
expect(event).toBeDefined();
expect(event!.sessionId).toBe("app-no-branch");
expect(event!.data).toMatchObject({
stage: "workspace_restore",
workspacePath: wsPath,
reason: "branch metadata is missing",
});
});
});
describe("metadata.corrupt_detected (MUST)", () => {
it("emits when mutateMetadata side-renames a corrupt file", async () => {
// Simulate a corrupt metadata file in the sessions dir
const sessionPath = join(sessionsDir, "app-corrupt.json");
mkdirSync(sessionsDir, { recursive: true });
writeFileSync(sessionPath, "{ this is not json", "utf-8");
const { mutateMetadata } = await import("../metadata.js");
mutateMetadata(sessionsDir, "app-corrupt", () => ({ branch: "feat/x", project: "my-app" }), {
createIfMissing: true,
activityEventSource: "api",
});
const event = findEvent("metadata.corrupt_detected");
expect(event).toBeDefined();
expect(event!.level).toBe("error");
expect(event!.source).toBe("api");
expect(event!.sessionId).toBe("app-corrupt");
const data = event!.data as Record<string, unknown>;
expect(data["renameSucceeded"]).toBe(true);
expect(data["renamedTo"]).toMatch(/\.corrupt-\d+$/);
});
});

78
packages/core/src/activity-events.ts
View File

@ -25,13 +25,35 @@ export type ActivityEventSource =
| "notifier"
| "reaction"
| "report-watcher"
| "cli";
| "cli"
| "config"
| "plugin-registry"
| "migration"
| "recovery";
export type ActivityEventKind =
| "session.spawn_started"
| "session.spawned"
| "session.spawn_failed"
| "session.spawn_step_failed"
| "session.killed"
| "session.kill_started"
| "session.send_failed"
| "session.restore_failed"
| "session.restore_fallback"
| "session.rollback_started"
| "session.rollback_step_failed"
| "session.workspace_hooks_failed"
| "session.cleanup_error"
| "session.orchestrator_conflict"
| "runtime.lost_detected"
| "runtime.lost_persist_failed"
| "runtime.destroy_failed"
| "workspace.destroy_failed"
| "agent.opencode_purge_failed"
| "tracker.issue_fetch_failed"
| "tracker.generate_prompt_failed"
| "metadata.corrupt_detected"
| "activity.transition"
| "lifecycle.transition"
| "ci.failing"
@ -70,7 +92,38 @@ export type ActivityEventKind =
| "lifecycle.poll_failed"
| "detecting.escalated"
// Report watcher
| "report_watcher.triggered";
| "report_watcher.triggered"
// Config/plugin-registry/storage migration
| "config.project_resolve_failed"
| "config.project_malformed"
| "config.project_invalid"
| "config.migrated"
| "plugin-registry.load_failed"
| "plugin-registry.validation_failed"
| "plugin-registry.specifier_failed"
| "migration.blocked"
| "migration.project_failed"
| "migration.rename_failed"
| "migration.completed"
| "migration.rollback_skipped"
// Webhook ingress (api source)
| "api.webhook_unverified"
| "api.webhook_rejected"
| "api.webhook_received"
| "api.webhook_failed"
// WebSocket terminal mux (ui source — Node-side server only)
| "ui.terminal_connected"
| "ui.terminal_disconnected"
| "ui.terminal_heartbeat_lost"
| "ui.terminal_pty_lost"
| "ui.terminal_protocol_error"
| "ui.session_broadcast_failed"
// Recovery/forensic instrumentation
| "recovery.session_failed"
| "recovery.action_failed"
| "api.agent_report.session_not_found"
| "api.agent_report.transition_rejected"
| "api.agent_report.apply_failed";
export type ActivityEventLevel = "debug" | "info" | "warn" | "error";
@ -110,14 +163,12 @@ export function droppedEventCount(): number {
}
function pruneOldEvents(db: ReturnType<typeof getDb>, cutoff: number): void {
db
?.prepare(
`DELETE FROM activity_events
db?.prepare(
`DELETE FROM activity_events
WHERE rowid IN (
SELECT rowid FROM activity_events WHERE ts_epoch < ? LIMIT ?
)`,
)
.run(cutoff, PRUNE_BATCH_SIZE);
).run(cutoff, PRUNE_BATCH_SIZE);
}
// Patterns that indicate sensitive field names
@ -134,7 +185,10 @@ function redactCredentialUrls(input: string): string {
const proto = result.indexOf("://", offset);
if (proto === -1) break;
// Only match http:// or https:// (case-insensitive, matching old /gi flag)
if (proto < 4) { offset = proto + 3; continue; }
if (proto < 4) {
offset = proto + 3;
continue;
}
const schemeEnd = result.slice(Math.max(0, proto - 5), proto).toLowerCase();
if (!schemeEnd.endsWith("http") && !schemeEnd.endsWith("https")) {
offset = proto + 3;
@ -145,7 +199,7 @@ function redactCredentialUrls(input: string): string {
while (cursor < result.length) {
const ch = result.charCodeAt(cursor);
// Space/control chars or '/' mean no '@' is coming in userinfo
if (ch <= 0x20 || ch === 0x2F) break;
if (ch <= 0x20 || ch === 0x2f) break;
if (ch === 0x40) {
// '@' found — redact everything between :// and @
// Lowercase the scheme to match the old /gi regex behavior
@ -158,7 +212,11 @@ function redactCredentialUrls(input: string): string {
cursor++;
}
// No '@' found — not a credential URL, move past this ://
if (cursor >= result.length || result.charCodeAt(cursor) <= 0x20 || result.charCodeAt(cursor) === 0x2F) {
if (
cursor >= result.length ||
result.charCodeAt(cursor) <= 0x20 ||
result.charCodeAt(cursor) === 0x2f
) {
offset = proto + 3;
}
}

216
packages/core/src/agent-report.ts
View File

@ -18,7 +18,7 @@
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { basename, dirname, join } from "node:path";
import type {
CanonicalSessionLifecycle,
CanonicalSessionReason,
@ -26,8 +26,14 @@ import type {
SessionId,
SessionStatus,
} from "./types.js";
import { recordActivityEvent } from "./activity-events.js";
import { mutateMetadata, readMetadataRaw } from "./metadata.js";
import { buildLifecycleMetadataPatch, cloneLifecycle, deriveLegacyStatus, parseCanonicalLifecycle } from "./lifecycle-state.js";
import {
buildLifecycleMetadataPatch,
cloneLifecycle,
deriveLegacyStatus,
parseCanonicalLifecycle,
} from "./lifecycle-state.js";
import { parsePrFromUrl } from "./utils/pr.js";
import { assertValidSessionIdComponent } from "./utils/session-id.js";
import { validateStatus } from "./utils/validation.js";
@ -254,6 +260,11 @@ function buildAuditDir(dataDir: string): string {
return join(dataDir, ".agent-report-audit");
}
function inferProjectIdFromDataDir(dataDir: string): string | undefined {
// dataDir is `.../projects/{projectId}/sessions`; recover projectId for filtering.
return basename(dirname(dataDir)) || undefined;
}
const AGENT_REPORT_AUDIT_MAX_BYTES = 256 * 1024;
const AGENT_REPORT_AUDIT_MAX_ENTRIES = 200;
@ -383,8 +394,22 @@ export function applyAgentReport(
sessionId: SessionId,
input: ApplyAgentReportInput,
): ApplyAgentReportResult {
const projectId = inferProjectIdFromDataDir(dataDir);
const raw = readMetadataRaw(dataDir, sessionId);
if (!raw) {
recordActivityEvent({
projectId,
sessionId,
source: "api",
kind: "api.agent_report.session_not_found",
level: "warn",
summary: `applyAgentReport: session not found: ${sessionId}`,
data: {
reportState: input.state,
actor: input.actor,
source: input.source,
},
});
throw new Error(`Session not found: ${sessionId}`);
}
@ -428,93 +453,124 @@ export function applyAgentReport(
let legacyStatus: SessionStatus | null = null;
let previousLegacyStatus: SessionStatus | null = null;
const nextMetadata = mutateMetadata(dataDir, sessionId, (existing) => {
const current = cloneLifecycle(
parseCanonicalLifecycle(existing, {
sessionId,
status: validateStatus(existing["status"]),
}),
);
previousLegacyStatus = deriveLegacyStatus(current);
before = buildAuditSnapshot(current, previousLegacyStatus);
const validation = validateAgentReportTransition(current, input.state);
if (!validation.ok) {
appendAgentReportAuditEntry(dataDir, sessionId, {
timestamp: now,
actor,
source,
reportState: input.state,
note: trimmedNote,
prNumber,
prUrl: trimmedPrUrl,
prIsDraft,
accepted: false,
rejectionReason: validation.reason ?? "transition rejected",
before,
after: before,
});
throw new Error(validation.reason ?? "transition rejected");
}
const mapped = mapAgentReportToLifecycle(input.state);
previousState = current.session.state;
nextState = mapped.sessionState;
current.session.state = mapped.sessionState;
current.session.reason = mapped.sessionReason;
current.session.lastTransitionAt = now;
if (isPRWorkflowReport(input.state)) {
const effectivePrUrl = trimmedPrUrl ?? current.pr.url ?? existingPrUrl;
const effectivePrNumber =
prNumber ?? current.pr.number ?? existingPrNumber ?? parsedPrFromUrl?.number;
const canAdvancePrState =
effectivePrUrl !== undefined ||
effectivePrNumber !== undefined ||
current.pr.state !== "none";
if (canAdvancePrState) {
current.pr.state = "open";
current.pr.reason =
input.state === "ready_for_review" ? "review_pending" : "in_progress";
current.pr.lastObservedAt = now;
const nextMetadata = mutateMetadata(
dataDir,
sessionId,
(existing) => {
const current = cloneLifecycle(
parseCanonicalLifecycle(existing, {
sessionId,
status: validateStatus(existing["status"]),
}),
);
previousLegacyStatus = deriveLegacyStatus(current);
before = buildAuditSnapshot(current, previousLegacyStatus);
const validation = validateAgentReportTransition(current, input.state);
if (!validation.ok) {
const rejectionReason = validation.reason ?? "transition rejected";
appendAgentReportAuditEntry(dataDir, sessionId, {
timestamp: now,
actor,
source,
reportState: input.state,
note: trimmedNote,
prNumber,
prUrl: trimmedPrUrl,
prIsDraft,
accepted: false,
rejectionReason,
before,
after: before,
});
recordActivityEvent({
projectId,
sessionId,
source: "api",
kind: "api.agent_report.transition_rejected",
level: "warn",
summary: `applyAgentReport rejected ${input.state} for ${sessionId}: ${rejectionReason}`,
data: {
reportState: input.state,
rejectionReason,
actor,
reportSource: source,
fromState: current.session.state,
fromReason: current.session.reason,
legacyStatus: previousLegacyStatus,
},
});
throw new Error(rejectionReason);
}
if (effectivePrUrl) {
current.pr.url = effectivePrUrl;
const mapped = mapAgentReportToLifecycle(input.state);
previousState = current.session.state;
nextState = mapped.sessionState;
current.session.state = mapped.sessionState;
current.session.reason = mapped.sessionReason;
current.session.lastTransitionAt = now;
if (isPRWorkflowReport(input.state)) {
const effectivePrUrl = trimmedPrUrl ?? current.pr.url ?? existingPrUrl;
const effectivePrNumber =
prNumber ?? current.pr.number ?? existingPrNumber ?? parsedPrFromUrl?.number;
const canAdvancePrState =
effectivePrUrl !== undefined ||
effectivePrNumber !== undefined ||
current.pr.state !== "none";
if (canAdvancePrState) {
current.pr.state = "open";
current.pr.reason = input.state === "ready_for_review" ? "review_pending" : "in_progress";
current.pr.lastObservedAt = now;
}
if (effectivePrUrl) {
current.pr.url = effectivePrUrl;
}
if (effectivePrNumber !== undefined) {
current.pr.number = effectivePrNumber;
}
}
if (effectivePrNumber !== undefined) {
current.pr.number = effectivePrNumber;
if (mapped.sessionState === "working" && current.session.startedAt === null) {
current.session.startedAt = now;
}
}
if (mapped.sessionState === "working" && current.session.startedAt === null) {
current.session.startedAt = now;
}
legacyStatus = deriveLegacyStatus(current);
const next = { ...existing };
Object.assign(
next,
buildLifecycleMetadataPatch(current),
{
legacyStatus = deriveLegacyStatus(current);
const next = { ...existing };
Object.assign(next, buildLifecycleMetadataPatch(current), {
[AGENT_REPORT_METADATA_KEYS.STATE]: input.state,
[AGENT_REPORT_METADATA_KEYS.AT]: now,
},
);
if (trimmedNote) {
next[AGENT_REPORT_METADATA_KEYS.NOTE] = trimmedNote;
} else {
next[AGENT_REPORT_METADATA_KEYS.NOTE] = "";
}
if (isPRWorkflowReport(input.state)) {
if (trimmedPrUrl) {
next[AGENT_REPORT_METADATA_KEYS.PR_URL] = trimmedPrUrl;
});
if (trimmedNote) {
next[AGENT_REPORT_METADATA_KEYS.NOTE] = trimmedNote;
} else {
next[AGENT_REPORT_METADATA_KEYS.NOTE] = "";
}
if (prNumber !== undefined) {
next[AGENT_REPORT_METADATA_KEYS.PR_NUMBER] = String(prNumber);
if (isPRWorkflowReport(input.state)) {
if (trimmedPrUrl) {
next[AGENT_REPORT_METADATA_KEYS.PR_URL] = trimmedPrUrl;
}
if (prNumber !== undefined) {
next[AGENT_REPORT_METADATA_KEYS.PR_NUMBER] = String(prNumber);
}
if (prIsDraft !== undefined) {
next[AGENT_REPORT_METADATA_KEYS.PR_IS_DRAFT] = prIsDraft ? "true" : "false";
}
}
if (prIsDraft !== undefined) {
next[AGENT_REPORT_METADATA_KEYS.PR_IS_DRAFT] = prIsDraft ? "true" : "false";
}
}
return next;
});
return next;
},
{ activityEventSource: "api" },
);
if (!nextMetadata || !before || !previousState || !nextState || !legacyStatus) {
recordActivityEvent({
projectId,
sessionId,
source: "api",
kind: "api.agent_report.apply_failed",
level: "error",
summary: `failed to apply agent report ${input.state} for ${sessionId}`,
data: {
reportState: input.state,
actor,
source,
},
});
throw new Error(`Failed to apply agent report for session ${sessionId}`);
}

12
packages/core/src/config.ts
View File

@ -32,6 +32,7 @@ import {
loadGlobalConfig,
} from "./global-config.js";
import { loadEffectiveProjectConfig } from "./project-resolver.js";
import { recordActivityEvent } from "./activity-events.js";
function inferScmPlugin(project: {
repo?: string;
@ -876,6 +877,17 @@ function buildEffectiveConfigFromGlobalConfigPath(configPath: string): LoadedCon
path: entry.path,
resolveError: error.message,
};
if (error.reasonKind === "malformed" || error.reasonKind === "invalid") {
continue;
}
recordActivityEvent({
projectId,
source: "config",
kind: "config.project_resolve_failed",
level: "error",
summary: `project ${projectId} failed to resolve`,
data: { path: entry.path, error: error.message },
});
}
}

43
packages/core/src/global-config.ts
View File

@ -7,10 +7,11 @@ import { z } from "zod";
import { atomicWriteFileSync } from "./atomic-write.js";
import { detectScmPlatform } from "./config-generator.js";
import { withFileLockSync } from "./file-lock.js";
import { ProjectResolveError } from "./types.js";
import { ProjectResolveError, type ProjectResolveErrorKind } from "./types.js";
import { generateSessionPrefix } from "./paths.js";
import { normalizeOriginUrl } from "./storage-key.js";
import { getDefaultRuntime } from "./platform.js";
import { recordActivityEvent } from "./activity-events.js";
function globalConfigLockPath(configPath: string): string {
return `${configPath}.lock`;
@ -347,6 +348,12 @@ export function loadGlobalConfig(
if (migrationSummary) {
// eslint-disable-next-line no-console -- required migration visibility for stale shadow stripping
console.info(migrationSummary);
recordActivityEvent({
source: "config",
kind: "config.migrated",
summary: "global config migrated",
data: { migrationSummary },
});
}
const config = GlobalConfigSchema.parse(parsed);
@ -836,6 +843,7 @@ export function resolveProjectIdentity(
defaultBranch: string;
sessionPrefix: string;
resolveError?: string;
resolveErrorKind?: ProjectResolveErrorKind;
})
| null {
const entry = globalConfig.projects[projectId] as
@ -914,15 +922,48 @@ export function resolveProjectIdentity(
};
}
if (localConfigResult.kind === "malformed") {
recordActivityEvent({
projectId,
source: "config",
kind: "config.project_malformed",
level: "error",
summary: `local config for ${projectId} could not be parsed`,
data: {
path: localConfigResult.path,
error: localConfigResult.error,
},
});
} else if (localConfigResult.kind === "invalid") {
recordActivityEvent({
projectId,
source: "config",
kind: "config.project_invalid",
level: "error",
summary: `local config for ${projectId} failed validation`,
data: {
path: localConfigResult.path,
error: localConfigResult.error,
},
});
}
const resolveError =
localConfigResult.kind !== "missing"
? (localConfigResult.error ?? "Failed to load local config")
: undefined;
const resolveErrorKind: ProjectResolveErrorKind | undefined =
localConfigResult.kind === "malformed" ||
localConfigResult.kind === "invalid" ||
localConfigResult.kind === "old-format"
? localConfigResult.kind
: undefined;
return {
...(resolveError ? {} : applyBehaviorDefaults({})),
...identityFields,
...(resolveError ? { resolveError } : {}),
...(resolveErrorKind ? { resolveErrorKind } : {}),
};
}

203
packages/core/src/metadata.ts
View File

@ -22,9 +22,15 @@ import {
closeSync,
constants,
} from "node:fs";
import { join, dirname } from "node:path";
import type { CanonicalSessionLifecycle, RuntimeHandle, SessionId, SessionMetadata } from "./types.js";
import { basename, join, dirname } from "node:path";
import type {
CanonicalSessionLifecycle,
RuntimeHandle,
SessionId,
SessionMetadata,
} from "./types.js";
import { atomicWriteFileSync } from "./atomic-write.js";
import { recordActivityEvent, type ActivityEventSource } from "./activity-events.js";
import {
buildLifecycleMetadataPatch,
cloneLifecycle,
@ -94,7 +100,9 @@ function parseRuntimeHandleField(value: unknown): RuntimeHandle | undefined {
if (typeof parsed["id"] === "string" && typeof parsed["runtimeName"] === "string") {
return parsed as unknown as RuntimeHandle;
}
} catch { /* not valid JSON */ }
} catch {
/* not valid JSON */
}
}
return undefined;
}
@ -106,13 +114,16 @@ function parseDashboardField(raw: Record<string, unknown>): SessionMetadata["das
return {
port: typeof d["port"] === "number" ? d["port"] : undefined,
terminalWsPort: typeof d["terminalWsPort"] === "number" ? d["terminalWsPort"] : undefined,
directTerminalWsPort: typeof d["directTerminalWsPort"] === "number" ? d["directTerminalWsPort"] : undefined,
directTerminalWsPort:
typeof d["directTerminalWsPort"] === "number" ? d["directTerminalWsPort"] : undefined,
};
}
// Legacy format: flat fields
const port = typeof raw["dashboardPort"] === "number" ? raw["dashboardPort"] : undefined;
const terminalWsPort = typeof raw["terminalWsPort"] === "number" ? raw["terminalWsPort"] : undefined;
const directTerminalWsPort = typeof raw["directTerminalWsPort"] === "number" ? raw["directTerminalWsPort"] : undefined;
const terminalWsPort =
typeof raw["terminalWsPort"] === "number" ? raw["terminalWsPort"] : undefined;
const directTerminalWsPort =
typeof raw["directTerminalWsPort"] === "number" ? raw["directTerminalWsPort"] : undefined;
if (port !== undefined || terminalWsPort !== undefined || directTerminalWsPort !== undefined) {
return { port, terminalWsPort, directTerminalWsPort };
}
@ -159,8 +170,15 @@ export function readMetadata(dataDir: string, sessionId: SessionId): SessionMeta
issueTitle: raw["issueTitle"] as string | undefined,
pr: raw["pr"] as string | undefined,
prAutoDetect:
raw["prAutoDetect"] === "off" || raw["prAutoDetect"] === "false" || raw["prAutoDetect"] === false ? false :
raw["prAutoDetect"] === "on" || raw["prAutoDetect"] === "true" || raw["prAutoDetect"] === true ? true : undefined,
raw["prAutoDetect"] === "off" ||
raw["prAutoDetect"] === "false" ||
raw["prAutoDetect"] === false
? false
: raw["prAutoDetect"] === "on" ||
raw["prAutoDetect"] === "true" ||
raw["prAutoDetect"] === true
? true
: undefined,
summary: raw["summary"] as string | undefined,
project: raw["project"] as string | undefined,
agent: raw["agent"] as string | undefined,
@ -220,8 +238,12 @@ export function readMetadataRaw(
/** Fields that are stored as JSON objects and should be parsed when unflattening. */
const jsonFields = new Set([
"runtimeHandle", "lifecycle", "statePayload", "dashboard",
"agentReport", "reportWatcher",
"runtimeHandle",
"lifecycle",
"statePayload",
"dashboard",
"agentReport",
"reportWatcher",
]);
/** Unflatten a Record<string, string> to proper types for JSON storage. */
@ -233,7 +255,12 @@ function unflattenFromStringRecord(data: Record<string, string>): Record<string,
for (const [key, value] of Object.entries(data)) {
if (value === undefined || value === "") continue;
if (booleanFields.has(key)) {
result[key] = value === "on" || value === "true" ? true : value === "off" || value === "false" ? false : value;
result[key] =
value === "on" || value === "true"
? true
: value === "off" || value === "false"
? false
: value;
} else if (numberFields.has(key)) {
const num = Number(value);
result[key] = Number.isFinite(num) ? num : value;
@ -306,9 +333,14 @@ export function updateMetadata(
sessionId: SessionId,
updates: Partial<Record<string, string>>,
): void {
mutateMetadata(dataDir, sessionId, (existing) => {
return applyMetadataUpdates(existing, updates);
}, { createIfMissing: true });
mutateMetadata(
dataDir,
sessionId,
(existing) => {
return applyMetadataUpdates(existing, updates);
},
{ createIfMissing: true },
);
}
export function applyMetadataUpdates(
@ -336,59 +368,94 @@ function normalizeMetadataRecord(data: Record<string, string>): Record<string, s
);
}
export interface MutateMetadataOptions {
createIfMissing?: boolean;
activityEventSource?: ActivityEventSource;
}
export function mutateMetadata(
dataDir: string,
sessionId: SessionId,
updater: (existing: Record<string, string>) => Record<string, string>,
options: { createIfMissing?: boolean } = {},
options: MutateMetadataOptions = {},
): Record<string, string> | null {
const path = metadataPath(dataDir, sessionId);
const lockPath = `${path}.lock`;
return withFileLockSync(lockPath, () => {
let existing: Record<string, string> = {};
return withFileLockSync(
lockPath,
() => {
let existing: Record<string, string> = {};
let content: string | undefined;
try {
content = readFileSync(path, "utf-8").trim();
} catch {
// File doesn't exist
}
let content: string | undefined;
try {
content = readFileSync(path, "utf-8").trim();
} catch {
// File doesn't exist
}
if (content !== undefined) {
if (content) {
const raw = parseMetadataContent(content);
if (raw) {
existing = flattenToStringRecord(raw);
} else {
// Corrupt JSON. Preserve forensic evidence by side-renaming
// the file before we overwrite it with the merged update.
// Without this, the very next mutateMetadata call destroys
// the corrupt bytes permanently and the user has no signal
// that anything was wrong — the file just becomes "not
// corrupt anymore — and missing fields".
const corruptPath = `${path}.corrupt-${Date.now()}`;
try {
renameSync(path, corruptPath);
// eslint-disable-next-line no-console
console.warn(
`[metadata] corrupt JSON at ${path}; preserved as ${corruptPath} before rewriting`,
);
} catch {
// best effort — proceed even if the rename fails (e.g. EACCES)
if (content !== undefined) {
if (content) {
const raw = parseMetadataContent(content);
if (raw) {
existing = flattenToStringRecord(raw);
} else {
// Corrupt JSON. Preserve forensic evidence by side-renaming
// the file before we overwrite it with the merged update.
// Without this, the very next mutateMetadata call destroys
// the corrupt bytes permanently and the user has no signal
// that anything was wrong — the file just becomes "not
// corrupt anymore — and missing fields".
const corruptPath = `${path}.corrupt-${Date.now()}`;
let renamed = false;
try {
renameSync(path, corruptPath);
renamed = true;
// eslint-disable-next-line no-console
console.warn(
`[metadata] corrupt JSON at ${path}; preserved as ${corruptPath} before rewriting`,
);
} catch {
// best effort — proceed even if the rename fails (e.g. EACCES)
}
// Forensic activity event so RCA can find every silent overwrite.
// Truncate the bad-JSON sample to 200 chars (B11 invariant — full file
// could be 16KB+ and would be dropped by the sanitizer cap).
const contentSample = content.length > 200 ? content.slice(0, 200) : content;
// dataDir is `.../projects/{projectId}/sessions`; recover projectId for filtering.
const inferredProjectId = basename(dirname(dataDir));
const summary = renamed
? `Corrupt metadata for session ${sessionId} renamed to ${basename(corruptPath)}`
: `Corrupt metadata detected for session ${sessionId}; failed to rename forensic copy before rewrite`;
recordActivityEvent({
projectId: inferredProjectId || undefined,
sessionId,
source: options.activityEventSource ?? "session-manager",
kind: "metadata.corrupt_detected",
level: "error",
summary,
data: {
path,
renamedTo: renamed ? corruptPath : null,
renameSucceeded: renamed,
contentSample,
contentLength: content.length,
},
});
}
}
} else if (!options.createIfMissing) {
return null;
}
} else if (!options.createIfMissing) {
return null;
}
const next = normalizeMetadataRecord(updater({ ...existing }));
const next = normalizeMetadataRecord(updater({ ...existing }));
mkdirSync(dirname(path), { recursive: true });
atomicWriteFileSync(path, serializeMetadata(unflattenFromStringRecord(next)));
return next;
}, { timeoutMs: 5_000, staleMs: 30_000 });
mkdirSync(dirname(path), { recursive: true });
atomicWriteFileSync(path, serializeMetadata(unflattenFromStringRecord(next)));
return next;
},
{ timeoutMs: 5_000, staleMs: 30_000 },
);
}
export function readCanonicalLifecycle(
@ -405,11 +472,7 @@ export function writeCanonicalLifecycle(
sessionId: SessionId,
lifecycle: CanonicalSessionLifecycle,
): void {
updateMetadata(
dataDir,
sessionId,
buildLifecycleMetadataPatch(cloneLifecycle(lifecycle)),
);
updateMetadata(dataDir, sessionId, buildLifecycleMetadataPatch(cloneLifecycle(lifecycle)));
}
export function updateCanonicalLifecycle(
@ -450,18 +513,20 @@ export function listMetadata(dataDir: string): SessionId[] {
const dir = dataDir;
if (!existsSync(dir)) return [];
return readdirSync(dir).filter((name) => {
// Must be a .json file
if (!name.endsWith(JSON_EXTENSION)) return false;
const baseName = name.slice(0, -JSON_EXTENSION.length);
if (!baseName || baseName.startsWith(".")) return false;
if (!SESSION_ID_COMPONENT_PATTERN.test(baseName)) return false;
try {
return statSync(join(dir, name)).isFile();
} catch {
return false;
}
}).map((name) => name.slice(0, -JSON_EXTENSION.length));
return readdirSync(dir)
.filter((name) => {
// Must be a .json file
if (!name.endsWith(JSON_EXTENSION)) return false;
const baseName = name.slice(0, -JSON_EXTENSION.length);
if (!baseName || baseName.startsWith(".")) return false;
if (!SESSION_ID_COMPONENT_PATTERN.test(baseName)) return false;
try {
return statSync(join(dir, name)).isFile();
} catch {
return false;
}
})
.map((name) => name.slice(0, -JSON_EXTENSION.length));
}
/**

86
packages/core/src/migration/storage-v2.ts
View File

@ -30,6 +30,7 @@ import { parseKeyValueContent } from "../key-value.js";
import { generateSessionPrefix } from "../paths.js";
import { atomicWriteFileSync } from "../atomic-write.js";
import { withFileLockSync } from "../file-lock.js";
import { recordActivityEvent } from "../activity-events.js";
// ---------------------------------------------------------------------------
// Constants
@ -1209,6 +1210,16 @@ export async function migrateStorage(options: MigrationOptions = {}): Promise<Mi
const knownPrefixes = extractProjectPrefixes(effectiveConfigPath);
const activeSessions = await detectActiveSessions(knownPrefixes);
if (activeSessions.length > 0) {
recordActivityEvent({
source: "migration",
kind: "migration.blocked",
level: "warn",
summary: `migration blocked by ${activeSessions.length} active session(s)`,
data: {
activeSessionCount: activeSessions.length,
sample: activeSessions.slice(0, 5),
},
});
throw new Error(
`Found ${activeSessions.length} active AO tmux session(s): ${activeSessions.slice(0, 5).join(", ")}${activeSessions.length > 5 ? "..." : ""}. ` +
`Kill active sessions first (ao session kill --all) or use --force to migrate anyway.`,
@ -1228,7 +1239,7 @@ export async function migrateStorage(options: MigrationOptions = {}): Promise<Mi
if (!dryRun && existsSync(markerPath)) {
try { unlinkSync(markerPath); } catch { /* best-effort */ }
}
return {
const totals: MigrationResult = {
projects: 0,
sessions: 0,
worktrees: 0,
@ -1237,6 +1248,24 @@ export async function migrateStorage(options: MigrationOptions = {}): Promise<Mi
claudeSessionsRelinked: 0,
codexSessionsRewritten: 0,
};
recordActivityEvent({
source: "migration",
kind: "migration.completed",
level: "info",
summary: "migration completed: 0 project(s), 0 session(s)",
data: {
dryRun,
projectsMigrated: totals.projects,
sessions: totals.sessions,
worktrees: totals.worktrees,
strayWorktreesMoved: totals.strayWorktreesMoved,
claudeSessionsRelinked: totals.claudeSessionsRelinked,
codexSessionsRewritten: totals.codexSessionsRewritten,
emptyDirsDeleted: totals.emptyDirsDeleted,
projectErrors: 0,
},
});
return totals;
}
log(`Found ${hashDirs.length} legacy director${hashDirs.length === 1 ? "y" : "ies"}.`);
@ -1315,6 +1344,18 @@ export async function migrateStorage(options: MigrationOptions = {}): Promise<Mi
const msg = err instanceof Error ? err.message : String(err);
log(` ERROR migrating project ${projectId}: ${msg}`);
projectErrors.push({ projectId, error: msg });
recordActivityEvent({
projectId,
source: "migration",
kind: "migration.project_failed",
level: "error",
summary: `migration failed for project ${projectId}`,
data: {
dryRun,
hashDirCount: dirs.length,
error: msg,
},
});
continue;
}
@ -1344,6 +1385,18 @@ export async function migrateStorage(options: MigrationOptions = {}): Promise<Mi
projectId,
error: `Failed to rename ${basename(dir.path)} to ${basename(migratedPath)}: ${msg}`,
});
recordActivityEvent({
projectId,
source: "migration",
kind: "migration.rename_failed",
level: "error",
summary: `failed to rename ${basename(dir.path)} to .migrated`,
data: {
from: basename(dir.path),
to: basename(migratedPath),
error: msg,
},
});
}
}
}
@ -1414,6 +1467,27 @@ export async function migrateStorage(options: MigrationOptions = {}): Promise<Mi
try { unlinkSync(markerPath); } catch { /* best-effort */ }
}
recordActivityEvent({
source: "migration",
kind: "migration.completed",
level: projectErrors.length > 0 ? "warn" : "info",
summary:
projectErrors.length > 0
? `migration finished with ${projectErrors.length} error(s)`
: `migration completed: ${totals.projects} project(s), ${totals.sessions} session(s)`,
data: {
dryRun,
projectsMigrated: totals.projects,
sessions: totals.sessions,
worktrees: totals.worktrees,
strayWorktreesMoved: totals.strayWorktreesMoved,
claudeSessionsRelinked: totals.claudeSessionsRelinked,
codexSessionsRewritten: totals.codexSessionsRewritten,
emptyDirsDeleted: totals.emptyDirsDeleted,
projectErrors: projectErrors.length,
},
});
return totals;
}
@ -1587,6 +1661,16 @@ export async function rollbackStorage(options: RollbackOptions = {}): Promise<vo
if (postMigrationSessions > 0) {
log(` Warning: projects/${projectId} has ${postMigrationSessions} session(s) created after migration — skipping deletion.`);
log(` These sessions exist only in projects/${projectId}/ and would be lost. Remove manually after verifying.`);
recordActivityEvent({
projectId,
source: "migration",
kind: "migration.rollback_skipped",
level: "warn",
summary: `rollback skipped projects/${projectId} — ${postMigrationSessions} post-migration session(s)`,
data: {
postMigrationSessions,
},
});
} else {
safeToDeleteProjects.add(projectId);
}

57
packages/core/src/plugin-registry.ts
View File

@ -19,6 +19,7 @@ import type {
PluginRegistry,
OrchestratorConfig,
} from "./types.js";
import { recordActivityEvent } from "./activity-events.js";
/** Map from "slot:name" → plugin instance */
type PluginMap = Map<string, { manifest: PluginManifest; instance: unknown }>;
@ -461,9 +462,23 @@ export function createPluginRegistry(): PluginRegistry {
this.register(mod);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
process.stderr.write(
`[plugin-registry] Failed to load built-in plugin "${builtin.name}": ${error}\n`,
);
recordActivityEvent({
source: "plugin-registry",
kind: "plugin-registry.load_failed",
level: "error",
summary: `built-in plugin ${builtin.name} failed to load`,
data: {
plugin: builtin.name,
slot: builtin.slot,
pkg: builtin.pkg,
builtin: true,
error: message,
},
});
}
}
}
@ -486,6 +501,16 @@ export function createPluginRegistry(): PluginRegistry {
const specifier = resolvePluginSpecifier(plugin, config);
if (!specifier) {
process.stderr.write(`[plugin-registry] Could not resolve specifier for plugin "${plugin.name}" (source: ${plugin.source})\n`);
recordActivityEvent({
source: "plugin-registry",
kind: "plugin-registry.specifier_failed",
level: "error",
summary: `could not resolve specifier for plugin ${plugin.name}`,
data: {
plugin: plugin.name,
source: plugin.source ?? null,
},
});
continue;
}
@ -508,9 +533,27 @@ export function createPluginRegistry(): PluginRegistry {
// Log validation errors but don't abort - other projects can still use the plugin.
// The misconfigured project will fail later when it tries to use the plugin
// with the wrong name, giving a clearer error at point of use.
const message =
validationError instanceof Error
? validationError.message
: String(validationError);
process.stderr.write(
`[plugin-registry] Config validation failed for ${externalEntry.source}: ${validationError}\n`,
);
recordActivityEvent({
source: "plugin-registry",
kind: "plugin-registry.validation_failed",
level: "error",
summary: `plugin manifest validation failed for ${plugin.name}`,
data: {
plugin: plugin.name,
externalSource: externalEntry.source,
specifier,
manifestName: mod.manifest.name,
manifestSlot: mod.manifest.slot,
error: message,
},
});
}
}
@ -520,7 +563,21 @@ export function createPluginRegistry(): PluginRegistry {
this.register(mod);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
process.stderr.write(`[plugin-registry] Failed to load plugin "${specifier}": ${error}\n`);
recordActivityEvent({
source: "plugin-registry",
kind: "plugin-registry.load_failed",
level: "error",
summary: `external plugin ${plugin.name} failed to load`,
data: {
plugin: plugin.name,
specifier,
source: plugin.source ?? null,
builtin: false,
error: message,
},
});
}
}
},

2
packages/core/src/project-resolver.ts
View File

@ -16,7 +16,7 @@ export function loadEffectiveProjectConfig(
throw new ProjectResolveError(projectId, `Unknown project: ${projectId}`);
}
if (typeof resolved.resolveError === "string" && resolved.resolveError.length > 0) {
throw new ProjectResolveError(projectId, resolved.resolveError);
throw new ProjectResolveError(projectId, resolved.resolveError, resolved.resolveErrorKind);
}
return resolved;
}

17
packages/core/src/recovery/actions.ts
View File

@ -5,6 +5,7 @@ import type {
Runtime,
Workspace,
} from "../types.js";
import { recordActivityEvent } from "../activity-events.js";
import { updateMetadata } from "../metadata.js";
import { getProjectSessionsDir } from "../paths.js";
import { validateStatus } from "../utils/validation.js";
@ -146,11 +147,25 @@ export async function recoverSession(
session,
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
recordActivityEvent({
projectId,
sessionId,
source: "recovery",
kind: "recovery.action_failed",
level: "error",
summary: `recoverSession threw for ${sessionId}: ${errorMessage}`,
data: {
action: "recover",
recoveryCount,
errorMessage,
},
});
return {
success: false,
sessionId,
action: "recover",
error: error instanceof Error ? error.message : String(error),
error: errorMessage,
};
}
}

34
packages/core/src/recovery/manager.ts
View File

@ -1,4 +1,5 @@
import type { OrchestratorConfig, PluginRegistry, Session } from "../types.js";
import { recordActivityEvent } from "../activity-events.js";
import { scanAllSessions, getRecoveryLogPath } from "./scanner.js";
import { validateSession } from "./validator.js";
import { executeAction } from "./actions.js";
@ -106,9 +107,10 @@ export async function runRecovery(options: RecoveryManagerOptions): Promise<Reco
break;
}
} else {
const errorMessage = recordRecoverySessionFailed(assessment, result);
report.errors.push({
sessionId: assessment.sessionId,
error: result.error || "Unknown error",
error: errorMessage,
});
}
@ -133,6 +135,32 @@ export async function runRecovery(options: RecoveryManagerOptions): Promise<Reco
};
}
function recordRecoverySessionFailed(
assessment: RecoveryAssessment,
result: RecoveryResult,
): string {
const errorMessage = result.error || "Unknown error";
// Forensic event so RCA can answer: did `ao recover` actually fix
// anything? Which sessions did it skip? B22: one event per failed
// session, not per probe step.
recordActivityEvent({
projectId: assessment.projectId,
sessionId: assessment.sessionId,
source: "recovery",
kind: "recovery.session_failed",
level: "error",
summary: `Recovery failed for session ${assessment.sessionId}: ${errorMessage}`,
data: {
action: result.action,
classification: assessment.classification,
recoveryRule: assessment.recoveryRule,
metadataStatus: assessment.metadataStatus,
errorMessage,
},
});
return errorMessage;
}
function mapActionToLogAction(
action: string,
success: boolean,
@ -184,6 +212,10 @@ export async function recoverSessionById(
return result;
}
if (!result.success) {
recordRecoverySessionFailed(assessment, result);
}
const logAction = mapActionToLogAction(result.action, result.success);
writeRecoveryLog(
recoveryConfig.logPath,

497
packages/core/src/session-manager.ts
View File

@ -1221,6 +1221,18 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
// Branch will be generated as feat/{issueId} (line 329-331)
} else {
// Other error (auth, network, etc) - fail fast
recordActivityEvent({
projectId: spawnConfig.projectId,
source: "session-manager",
kind: "tracker.issue_fetch_failed",
level: "error",
summary: `tracker getIssue failed for ${spawnConfig.issueId}`,
data: {
issueId: spawnConfig.issueId,
tracker: plugins.tracker.name,
reason: err instanceof Error ? err.message : String(err),
},
});
throw new Error(`Failed to fetch issue ${spawnConfig.issueId}: ${err}`, { cause: err });
}
}
@ -1235,10 +1247,13 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
// step now requires pushing one cleanup, with no risk of forgetting prior
// ones.
const cleanupStack = new CleanupStack();
let sessionId: string | undefined;
try {
// Determine session ID — atomically reserve to prevent concurrent collisions
const { sessionId, tmuxName } = await reserveNextSessionIdentity(project, sessionsDir);
cleanupStack.push(() => deleteMetadata(sessionsDir, sessionId));
let tmuxName: string | undefined;
({ sessionId, tmuxName } = await reserveNextSessionIdentity(project, sessionsDir));
const reservedSessionId = sessionId;
cleanupStack.push(() => deleteMetadata(sessionsDir, reservedSessionId));
// Determine branch name — explicit branch always takes priority
let branch: string;
@ -1295,9 +1310,23 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
if (spawnConfig.issueId && plugins.tracker && resolvedIssue) {
try {
issueContext = await plugins.tracker.generatePrompt(spawnConfig.issueId, project);
} catch {
// Non-fatal: continue without detailed issue context
// Silently ignore errors - caller can check if issueContext is undefined
} catch (err) {
// Non-fatal: continue without detailed issue context. Surface the
// failure via AE so RCA can answer "did the agent get an enriched
// prompt or just the bare issue ID?"
recordActivityEvent({
projectId: spawnConfig.projectId,
sessionId,
source: "session-manager",
kind: "tracker.generate_prompt_failed",
level: "warn",
summary: `tracker generatePrompt failed for ${spawnConfig.issueId}`,
data: {
issueId: spawnConfig.issueId,
tracker: plugins.tracker.name,
reason: err instanceof Error ? err.message : String(err),
},
});
}
}
@ -1514,14 +1543,81 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
// Log cleanup failures so they don't disappear silently. The original
// code used /* best effort */ swallows; the stack preserves that
// behavior (cleanup errors don't propagate) but surfaces them for debug.
recordActivityEvent({
projectId: spawnConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.rollback_started",
level: "warn",
summary: "spawn rollback started",
data: { reason: err instanceof Error ? err.message : String(err) },
});
await cleanupStack.runAll((cleanupErr) => {
console.error("[session-manager] spawn rollback step failed:", cleanupErr);
// B25: emit per-step rollback failure so each leaked resource is queryable.
recordActivityEvent({
projectId: spawnConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.rollback_step_failed",
level: "error",
summary: "spawn rollback step failed",
data: {
reason: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr),
},
});
});
throw err;
}
}
async function spawnOrchestrator(orchestratorConfig: OrchestratorSpawnConfig): Promise<Session> {
function recordOrchestratorSpawnFailed(
orchestratorConfig: OrchestratorSpawnConfig,
err: unknown,
sessionId?: string,
): void {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
...(sessionId ? { sessionId } : {}),
source: "session-manager",
kind: "session.spawn_failed",
level: "error",
summary: "orchestrator spawn failed",
data: {
role: "orchestrator",
reason: err instanceof Error ? err.message : String(err),
},
});
}
async function spawnOrchestrator(
orchestratorConfig: OrchestratorSpawnConfig,
options?: { suppressFixedReservationFailure?: boolean },
): Promise<Session> {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
source: "session-manager",
kind: "session.spawn_started",
summary: "orchestrator spawn started",
data: { agent: orchestratorConfig.agent ?? undefined, role: "orchestrator" },
});
try {
return await _spawnOrchestratorInner(orchestratorConfig);
} catch (err) {
const project = config.projects[orchestratorConfig.projectId];
const sessionId = project ? getOrchestratorSessionId(project) : undefined;
const shouldSuppressRecoverableConflict =
options?.suppressFixedReservationFailure === true &&
sessionId !== undefined &&
isFixedOrchestratorReservationError(err, sessionId);
if (!shouldSuppressRecoverableConflict) {
recordOrchestratorSpawnFailed(orchestratorConfig, err, sessionId);
}
throw err;
}
}
async function _spawnOrchestratorInner(orchestratorConfig: OrchestratorSpawnConfig): Promise<Session> {
const project = config.projects[orchestratorConfig.projectId];
if (!project) {
throw new Error(`Unknown project: ${orchestratorConfig.projectId}`);
@ -1582,6 +1678,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
workspacePath = wsInfo.path;
adoptedManagedWorkspace = adoptedInfo !== undefined && adoptedInfo !== null;
} catch (err) {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawn_step_failed",
level: "error",
summary: "orchestrator workspace.create failed",
data: {
role: "orchestrator",
stage: "workspace_create",
reason: err instanceof Error ? err.message : String(err),
},
});
try {
deleteMetadata(sessionsDir, sessionId);
} catch {
@ -1627,6 +1736,21 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
await setupPathWrapperWorkspace(workspacePath);
}
} catch (err) {
// PR tracking and CI fetch hooks are wired here — emit a dedicated AE
// before rolling back so RCA can answer "did the orchestrator launch
// succeed but lose its hook integration?".
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.workspace_hooks_failed",
level: "error",
summary: "orchestrator workspace hooks installation failed",
data: {
agent: plugins.agent.name,
reason: err instanceof Error ? err.message : String(err),
},
});
await cleanupWorktreeAndMetadata();
throw err;
}
@ -1642,6 +1766,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
systemPromptFile = join(projectDir, `orchestrator-prompt-${sessionId}.md`);
writeFileSync(systemPromptFile, orchestratorConfig.systemPrompt, "utf-8");
} catch (err) {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawn_step_failed",
level: "error",
summary: "orchestrator systemPrompt write failed",
data: {
role: "orchestrator",
stage: "system_prompt_write",
reason: err instanceof Error ? err.message : String(err),
},
});
await cleanupWorktreeAndMetadata(systemPromptFile);
throw err;
}
@ -1651,6 +1788,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
try {
writeWorkspaceOpenCodeAgentsMd(workspacePath, systemPromptFile);
} catch (err) {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawn_step_failed",
level: "error",
summary: "orchestrator AGENTS.md write failed",
data: {
role: "orchestrator",
stage: "agents_md_write",
reason: err instanceof Error ? err.message : String(err),
},
});
await cleanupWorktreeAndMetadata(systemPromptFile);
throw err;
}
@ -1675,6 +1825,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
});
}
} catch (err) {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawn_step_failed",
level: "error",
summary: "orchestrator opencode session resolution failed",
data: {
role: "orchestrator",
stage: "opencode_session_reuse",
reason: err instanceof Error ? err.message : String(err),
},
});
await cleanupWorktreeAndMetadata(systemPromptFile);
throw err;
}
@ -1732,6 +1895,22 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
},
});
} catch (err) {
// Outer envelope catches and emits session.spawn_failed; this step emit
// tags the runtime.create failure path specifically so RCA can answer
// "did the orchestrator runtime fail to start at all?".
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawn_step_failed",
level: "error",
summary: "orchestrator runtime.create failed",
data: {
role: "orchestrator",
stage: "runtime_create",
reason: err instanceof Error ? err.message : String(err),
},
});
await cleanupWorktreeAndMetadata(systemPromptFile);
throw err;
}
@ -1819,6 +1998,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
}
invalidateCache();
} catch (err) {
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawn_step_failed",
level: "error",
summary: "orchestrator post-launch metadata write failed",
data: {
role: "orchestrator",
stage: "post_launch_metadata",
reason: err instanceof Error ? err.message : String(err),
},
});
// Clean up runtime on post-launch failure
try {
await plugins.runtime.destroy(handle);
@ -1829,6 +2021,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
throw err;
}
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.spawned",
summary: `spawned: ${sessionId}`,
data: {
agent: plugins.agent.name,
branch: session.branch ?? undefined,
role: "orchestrator",
},
});
return session;
}
@ -1897,14 +2102,27 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
}
try {
return await spawnOrchestrator(orchestratorConfig);
return await spawnOrchestrator(orchestratorConfig, {
suppressFixedReservationFailure: true,
});
} catch (err) {
if (!isFixedOrchestratorReservationError(err, sessionId)) {
throw err;
}
recordActivityEvent({
projectId: orchestratorConfig.projectId,
sessionId,
source: "session-manager",
kind: "session.orchestrator_conflict",
level: "warn",
summary: "concurrent orchestrator reservation conflict",
data: { reason: err instanceof Error ? err.message : String(err) },
});
const concurrent = await waitForConcurrentOrchestrator(sessionId);
if (concurrent) return concurrent;
recordOrchestratorSpawnFailed(orchestratorConfig, err, sessionId);
throw err;
}
}
@ -2068,20 +2286,44 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
onDiskLifecycle.session.state !== "done" &&
onDiskLifecycle.session.state !== "detecting"
) {
const runtimeStateBefore = session.lifecycle.runtime.state;
const runtimeReasonBefore = session.lifecycle.runtime.reason;
try {
const persisted = buildUpdatedLifecycle(sessionName, raw, (next) => {
next.session.state = "detecting";
next.session.reason = "runtime_lost";
next.session.lastTransitionAt = new Date().toISOString();
next.runtime.state = session.lifecycle!.runtime.state;
next.runtime.reason = session.lifecycle!.runtime.reason;
next.runtime.state = runtimeStateBefore;
next.runtime.reason = runtimeReasonBefore;
next.runtime.lastObservedAt = new Date().toISOString();
});
// B1: persist BEFORE emitting the event
updateMetadata(sessionsDir, sessionName, lifecycleMetadataUpdates(raw, persisted));
session.lifecycle = persisted;
session.status = deriveLegacyStatus(persisted);
} catch {
recordActivityEvent({
projectId: sessionProjectId,
sessionId: sessionName,
source: "session-manager",
kind: "runtime.lost_detected",
level: "warn",
summary: `runtime lost reconciled: ${sessionName}`,
data: {
runtimeState: runtimeStateBefore,
runtimeReason: runtimeReasonBefore,
},
});
} catch (err) {
// Persist failed — in-memory state is still correct for this request
recordActivityEvent({
projectId: sessionProjectId,
sessionId: sessionName,
source: "session-manager",
kind: "runtime.lost_persist_failed",
level: "error",
summary: `runtime_lost persist failed: ${sessionName}`,
data: { reason: err instanceof Error ? err.message : String(err) },
});
}
}
@ -2192,6 +2434,17 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
const killReason: LifecycleKillReason = options?.reason ?? "manually_killed";
const cleanupAgent = resolveSelectionForSession(project, sessionId, raw).agentName;
// Emit kill_started up-front — this is the only signal that the kill
// intent reached the manager (the destroys below are silent on failure).
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.kill_started",
summary: `kill started: ${sessionId}`,
data: { reason: killReason },
});
// Destroy runtime — prefer handle.runtimeName to find the correct plugin
if (raw["runtimeHandle"]) {
const handle = safeJsonParse<RuntimeHandle>(raw["runtimeHandle"]);
@ -2204,8 +2457,20 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
if (runtimePlugin) {
try {
await runtimePlugin.destroy(handle);
} catch {
// Runtime might already be gone
} catch (err) {
// Runtime might already be gone — surface as AE so leaks are queryable.
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "runtime.destroy_failed",
level: "warn",
summary: `runtime.destroy failed during kill: ${sessionId}`,
data: {
runtime: handle.runtimeName ?? null,
reason: err instanceof Error ? err.message : String(err),
},
});
}
}
}
@ -2219,8 +2484,21 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
if (workspacePlugin) {
try {
await workspacePlugin.destroy(worktree);
} catch {
// Workspace might already be gone
} catch (err) {
// Workspace might already be gone — emit AE so abandoned worktrees
// surface for cleanup tooling.
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "workspace.destroy_failed",
level: "warn",
summary: `workspace.destroy failed during kill: ${sessionId}`,
data: {
workspace: workspacePlugin.name,
reason: err instanceof Error ? err.message : String(err),
},
});
}
}
}
@ -2238,8 +2516,20 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
try {
await deleteOpenCodeSession(mappedOpenCodeSessionId);
didPurgeOpenCodeSession = true;
} catch {
void 0;
} catch (err) {
// Dangling opencode session is a real leak — surface for RCA.
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "agent.opencode_purge_failed",
level: "warn",
summary: `opencode session purge failed: ${sessionId}`,
data: {
opencodeSessionId: mappedOpenCodeSessionId,
reason: err instanceof Error ? err.message : String(err),
},
});
}
}
}
@ -2372,9 +2662,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
pushSkipped(session.projectId, session.id);
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err);
result.errors.push({
sessionId: session.id,
error: err instanceof Error ? err.message : String(err),
error: errorMessage,
});
recordActivityEvent({
projectId: session.projectId,
sessionId: session.id,
source: "session-manager",
kind: "session.cleanup_error",
level: "warn",
summary: `cleanup error: ${session.id}`,
data: { reason: errorMessage },
});
}
}
@ -2416,11 +2716,24 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
...existing,
opencodeSessionId: "",
opencodeCleanedAt: new Date().toISOString(),
}));
}), { activityEventSource: "session-manager" });
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err);
result.errors.push({
sessionId: terminatedId,
error: `Failed to delete OpenCode session ${mappedOpenCodeSessionId}: ${err instanceof Error ? err.message : String(err)}`,
error: `Failed to delete OpenCode session ${mappedOpenCodeSessionId}: ${errorMessage}`,
});
recordActivityEvent({
projectId: projectKey,
sessionId: terminatedId,
source: "session-manager",
kind: "agent.opencode_purge_failed",
level: "warn",
summary: `opencode session purge failed during cleanup: ${terminatedId}`,
data: {
opencodeSessionId: mappedOpenCodeSessionId,
reason: errorMessage,
},
});
continue;
}
@ -2451,7 +2764,7 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
}
async function send(sessionId: SessionId, message: string): Promise<void> {
const { raw, sessionsDir, project } = requireSessionRecord(sessionId);
const { raw, sessionsDir, project, projectId } = requireSessionRecord(sessionId);
const selection = resolveSelectionForSession(project, sessionId, raw);
const selectedAgent = selection.agentName;
@ -2603,19 +2916,31 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
throw new Error(`Cannot send to session ${sessionId}: ${reason}`);
}
let restored: Session;
try {
const restored = await restore(sessionId);
const ready = await waitForRestoredSession(restored);
if (!ready) {
throw new Error("restored session did not become ready for delivery");
}
return restored;
restored = await restore(sessionId);
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new Error(`Cannot send to session ${sessionId}: ${reason} (${detail})`, {
cause: err,
});
}
const ready = await waitForRestoredSession(restored);
if (!ready) {
const detail = "restored session did not become ready for delivery";
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.restore_failed",
level: "error",
summary: `restore for delivery failed: ${sessionId}`,
data: { stage: "ready_timeout", reason: detail, trigger: "send" },
});
throw new Error(`Cannot send to session ${sessionId}: ${reason} (${detail})`);
}
return restored;
};
const prepareSession = async (forceRestore = false): Promise<Session> => {
@ -2707,29 +3032,52 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
return;
};
let prepared = await prepareSession();
// Top-level try/catch: any final send failure (initial preparation,
// retry-with-restore, etc.) emits a single `session.send_failed` event
// (B16 — failure-only). Stage tag distinguishes which branch failed.
let stage: "prepare" | "initial" | "restore_retry" = "prepare";
try {
await sendWithConfirmation(prepared);
} catch (err) {
const shouldRetryWithRestore = prepared.restoredAt === undefined && isRestorable(prepared);
let prepared = await prepareSession();
if (!shouldRetryWithRestore) {
if (err instanceof Error) {
throw err;
}
throw new Error(String(err), { cause: err });
}
prepared = await prepareSession(true);
try {
stage = "initial";
await sendWithConfirmation(prepared);
} catch (retryErr) {
if (retryErr instanceof Error) {
throw retryErr;
} catch (err) {
const shouldRetryWithRestore =
prepared.restoredAt === undefined && isRestorable(prepared);
if (!shouldRetryWithRestore) {
if (err instanceof Error) {
throw err;
}
throw new Error(String(err), { cause: err });
}
stage = "restore_retry";
prepared = await prepareSession(true);
try {
await sendWithConfirmation(prepared);
} catch (retryErr) {
if (retryErr instanceof Error) {
throw retryErr;
}
throw new Error(String(retryErr), { cause: retryErr });
}
throw new Error(String(retryErr), { cause: retryErr });
}
} catch (err) {
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.send_failed",
level: "error",
summary: `send failed: ${sessionId}`,
data: {
stage,
reason: err instanceof Error ? err.message : String(err),
},
});
throw err;
}
}
@ -2930,6 +3278,20 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
const reason = NON_RESTORABLE_STATUSES.has(session.status)
? `status "${session.status}" is not restorable`
: `session is not in a terminal state (status: "${session.status}", activity: "${session.activity}")`;
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.restore_failed",
level: "error",
summary: `restore not allowed: ${sessionId}`,
data: {
stage: "validation",
status: session.status,
activity: session.activity,
reason,
},
});
throw new SessionNotRestorableError(sessionId, reason);
}
@ -2950,9 +3312,35 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
if (!workspaceExists) {
// Try to restore workspace if plugin supports it
if (!plugins.workspace?.restore) {
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.restore_failed",
level: "error",
summary: `restore workspace failed: ${sessionId}`,
data: {
stage: "workspace_restore",
workspacePath,
reason: "workspace plugin does not support restore",
},
});
throw new WorkspaceMissingError(workspacePath, "workspace plugin does not support restore");
}
if (!session.branch) {
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.restore_failed",
level: "error",
summary: `restore workspace failed: ${sessionId}`,
data: {
stage: "workspace_restore",
workspacePath,
reason: "branch metadata is missing",
},
});
throw new WorkspaceMissingError(workspacePath, "branch metadata is missing");
}
try {
@ -2972,6 +3360,19 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
await plugins.workspace.postCreate(wsInfo, project);
}
} catch (err) {
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.restore_failed",
level: "error",
summary: `workspace restore failed: ${sessionId}`,
data: {
stage: "workspace_restore",
workspacePath,
reason: err instanceof Error ? err.message : String(err),
},
});
throw new WorkspaceMissingError(
workspacePath,
`restore failed: ${err instanceof Error ? err.message : String(err)}`,
@ -3063,6 +3464,16 @@ export function createSessionManager(deps: SessionManagerDeps): OpenCodeSessionM
updateMetadata(sessionsDir, sessionId, {
restoreFallbackReason: reason,
});
// Surface that AO fell back to a fresh launch instead of native restore.
recordActivityEvent({
projectId,
sessionId,
source: "session-manager",
kind: "session.restore_fallback",
level: "warn",
summary: `using fresh launch instead of native restore: ${sessionId}`,
data: { agent: plugins.agent.name, reason },
});
launchCommand = plugins.agent.getLaunchCommand(agentLaunchConfig);
}
} else {

3
packages/core/src/types.ts
View File

@ -2005,11 +2005,14 @@ export class ConfigNotFoundError extends Error {
}
}
export type ProjectResolveErrorKind = "malformed" | "invalid" | "old-format";
/** Thrown when a project cannot be resolved into an effective runtime config. */
export class ProjectResolveError extends Error {
constructor(
public readonly projectId: string,
message: string,
public readonly reasonKind?: ProjectResolveErrorKind,
) {
super(message);
this.name = "ProjectResolveError";

36
packages/plugins/tracker-linear/src/index.ts
View File

@ -75,12 +75,19 @@ class LinearNetworkError extends Error {
}
}
class LinearTimeoutError extends Error {
constructor() {
super("Linear API request timed out after 30s");
}
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function isRetryableDirectTransportError(err: unknown): boolean {
if (err instanceof LinearHttpError) return err.transient;
if (err instanceof LinearTimeoutError) return true;
return err instanceof LinearNetworkError;
}
@ -159,21 +166,22 @@ function createDirectTransport(): GraphQLTransport {
},
);
req.setTimeout(30_000, () => {
settle(() => {
req.destroy();
recordTransportActivityEvent({
source: "tracker",
kind: "tracker.api_timeout",
level: "warn",
summary: "Linear API request timed out after 30s",
data: {
plugin: "tracker-linear",
transport: "direct",
timeoutMs: 30_000,
},
req.setTimeout(30_000, () => {
settle(() => {
req.destroy();
recordTransportActivityEvent({
source: "tracker",
kind: "tracker.api_timeout",
level: "warn",
summary: "Linear API request timed out after 30s",
data: {
plugin: "tracker-linear",
transport: "direct",
timeoutMs: 30_000,
},
});
reject(new LinearTimeoutError());
});
reject(new Error("Linear API request timed out after 30s"));
});
req.on("error", (err) => settle(() => reject(new LinearNetworkError(err.message))));

526
packages/web/server/__tests__/mux-websocket.test.ts
View File

@ -1,15 +1,26 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { EventEmitter } from "node:events";
import type { Socket } from "node:net";
import { WebSocket } from "ws";
import type { SessionBroadcaster as SessionBroadcasterType } from "../mux-websocket";
// vi.mock factories run before module-level statements. Hoist the mock
// fns so the factories close over the same instances the tests use.
const { mockSpawn, mockPtySpawn, mockTmuxHasSession } = vi.hoisted(() => ({
const { mockSpawn, mockPtySpawn, mockTmuxHasSession, recordActivityEvent } = vi.hoisted(() => ({
mockSpawn: vi.fn(),
mockPtySpawn: vi.fn(),
mockTmuxHasSession: vi.fn(),
recordActivityEvent: vi.fn(),
}));
vi.mock("@aoagents/ao-core", async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>;
return {
...actual,
recordActivityEvent: (event: unknown) => recordActivityEvent(event),
};
});
vi.mock("node:child_process", async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>;
const spawnFn = (...args: unknown[]) => mockSpawn(...args);
@ -34,21 +45,72 @@ vi.mock("../tmux-utils.js", () => ({
findTmux: () => "/usr/bin/tmux",
validateSessionId: () => true,
resolveTmuxSession: () => "ao-177",
resolvePipePath: () => null,
tmuxHasSession: (...args: unknown[]) => mockTmuxHasSession(...args),
}));
const { SessionBroadcaster, TerminalManager } = await import("../mux-websocket");
const { SessionBroadcaster, TerminalManager, createMuxWebSocket, handleWindowsPipeMessage } =
await import("../mux-websocket");
// Mock global fetch
const mockFetch = vi.fn();
global.fetch = mockFetch;
type MockPty = {
dataHandlers: Array<(data: string) => void>;
exitHandlers: Array<(event: { exitCode: number }) => void>;
onData: ReturnType<typeof vi.fn>;
onExit: ReturnType<typeof vi.fn>;
write: ReturnType<typeof vi.fn>;
resize: ReturnType<typeof vi.fn>;
kill: ReturnType<typeof vi.fn>;
emitData: (data: string) => void;
emitExit: (exitCode: number) => Promise<void>;
};
const ptyInstances: MockPty[] = [];
function createMockPty(): MockPty {
const pty = {} as MockPty;
pty.dataHandlers = [];
pty.exitHandlers = [];
pty.onData = vi.fn((handler: (data: string) => void) => {
pty.dataHandlers.push(handler);
});
pty.onExit = vi.fn((handler: (event: { exitCode: number }) => void) => {
pty.exitHandlers.push(handler);
});
pty.write = vi.fn();
pty.resize = vi.fn();
pty.kill = vi.fn();
pty.emitData = (data: string) => {
for (const handler of pty.dataHandlers) handler(data);
};
pty.emitExit = async (exitCode: number) => {
await Promise.all([...pty.exitHandlers].map((handler) => handler({ exitCode })));
};
ptyInstances.push(pty);
return pty;
}
function resetPtyMock(): void {
ptyInstances.length = 0;
mockSpawn.mockReset();
mockPtySpawn.mockReset();
mockTmuxHasSession.mockReset();
mockTmuxHasSession.mockResolvedValue(true);
mockSpawn.mockImplementation(() => new EventEmitter());
mockPtySpawn.mockImplementation(createMockPty);
}
describe("SessionBroadcaster", () => {
let broadcaster: SessionBroadcasterType;
beforeEach(() => {
vi.useFakeTimers();
mockFetch.mockReset();
recordActivityEvent.mockClear();
resetPtyMock();
broadcaster = new SessionBroadcaster("3000");
});
@ -262,6 +324,451 @@ describe("SessionBroadcaster", () => {
expect(mockFetch).toHaveBeenCalledTimes(1);
});
});
describe("ui.session_broadcast_failed activity events", () => {
function failedKinds(): string[] {
return recordActivityEvent.mock.calls
.map(([e]) => (e as { kind: string }).kind)
.filter((k) => k === "ui.session_broadcast_failed");
}
it("emits exactly once on the healthy→failing transition", async () => {
// First fetch fails — triggers emission
mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));
// Second fetch (3s later) also fails — should NOT emit again
mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));
broadcaster.subscribe(vi.fn());
await vi.advanceTimersByTimeAsync(10);
await vi.advanceTimersByTimeAsync(3010);
expect(failedKinds()).toEqual(["ui.session_broadcast_failed"]);
});
it("re-arms after recovery (success → failure emits again)", async () => {
// fail → succeed → fail
mockFetch.mockRejectedValueOnce(new Error("net down"));
mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ sessions: [] }) });
mockFetch.mockRejectedValueOnce(new Error("net down again"));
broadcaster.subscribe(vi.fn());
await vi.advanceTimersByTimeAsync(10);
await vi.advanceTimersByTimeAsync(3010); // poll #1 → success
await vi.advanceTimersByTimeAsync(3010); // poll #2 → failure
expect(failedKinds().length).toBe(2);
});
it("emits with source=ui, level=warn, and the failure URL in data", async () => {
mockFetch.mockRejectedValueOnce(new Error("ETIMEDOUT"));
broadcaster.subscribe(vi.fn());
await vi.advanceTimersByTimeAsync(10);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "ui",
kind: "ui.session_broadcast_failed",
level: "warn",
}),
);
const call = recordActivityEvent.mock.calls.find(
([e]) => (e as { kind: string }).kind === "ui.session_broadcast_failed",
)![0] as { data: Record<string, unknown> };
expect(call.data["url"]).toContain("/api/sessions/patches");
expect(call.data["errorMessage"]).toContain("ETIMEDOUT");
});
it("includes httpStatus when fetch returns non-OK response", async () => {
mockFetch.mockResolvedValueOnce({ ok: false, status: 503 });
broadcaster.subscribe(vi.fn());
await vi.advanceTimersByTimeAsync(10);
const call = recordActivityEvent.mock.calls.find(
([e]) => (e as { kind: string }).kind === "ui.session_broadcast_failed",
)![0] as { data: Record<string, unknown> };
expect(call.data["httpStatus"]).toBe(503);
});
});
});
// ── Connection-level activity events ──────────────────────────────────
// These verify ui.terminal_* events fire at the right WS lifecycle points.
// We exercise the connection handler directly by emitting "connection" on
// the WebSocketServer and feeding a fake ws + IncomingMessage stand-in.
class FakeWS extends EventEmitter {
readyState: 0 | 1 | 2 | 3 = WebSocket.OPEN;
bufferedAmount = 0;
ping = vi.fn();
terminate = vi.fn(() => {
this.readyState = WebSocket.CLOSED;
});
send = vi.fn();
}
class FakePipeSocket extends EventEmitter {
write = vi.fn();
end = vi.fn(() => {
this.emit("close");
});
destroy = vi.fn(() => {
this.emit("close");
});
}
function makeFakeRequest(opts?: { remoteAddress?: string; xff?: string }) {
return {
headers: opts?.xff ? { "x-forwarded-for": opts.xff } : {},
socket: { remoteAddress: opts?.remoteAddress ?? "127.0.0.1" },
};
}
describe("mux WebSocket connection events", () => {
beforeEach(() => {
vi.useFakeTimers();
recordActivityEvent.mockClear();
resetPtyMock();
});
afterEach(() => {
vi.clearAllTimers();
vi.useRealTimers();
});
function emitConnection(opts?: Parameters<typeof makeFakeRequest>[0]) {
const wss = createMuxWebSocket();
if (!wss) {
throw new Error("mux WS server not created — node-pty unavailable");
}
const ws = new FakeWS();
wss.emit("connection", ws as unknown as WebSocket, makeFakeRequest(opts));
return { wss, ws };
}
function findEvent(kind: string): { data: Record<string, unknown> } | undefined {
const found = recordActivityEvent.mock.calls.find(
([e]) => (e as { kind: string }).kind === kind,
);
return found?.[0] as { data: Record<string, unknown> } | undefined;
}
it("emits ui.terminal_connected on a new mux connection (with remoteAddr)", () => {
emitConnection({ xff: "198.51.100.5, 10.0.0.1" });
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({ source: "ui", kind: "ui.terminal_connected" }),
);
const evt = findEvent("ui.terminal_connected")!;
expect(evt.data["remoteAddr"]).toBe("198.51.100.5");
});
it("emits ui.terminal_disconnected exactly once on close", () => {
const { ws } = emitConnection();
recordActivityEvent.mockClear();
ws.emit("close", 1000, Buffer.from("normal"));
const calls = recordActivityEvent.mock.calls.filter(
([e]) => (e as { kind: string }).kind === "ui.terminal_disconnected",
);
expect(calls.length).toBe(1);
const evt = findEvent("ui.terminal_disconnected")!;
expect(evt.data["code"]).toBe(1000);
expect(evt.data["reason"]).toBe("normal");
});
it("emits ui.terminal_heartbeat_lost once on 3 missed pongs and terminates", () => {
const { ws } = emitConnection();
recordActivityEvent.mockClear();
// Each 15s interval sends a ping and increments missedPongs by 1.
// After 3 ticks (45s) it should hit MAX_MISSED_PONGS=3 and terminate.
vi.advanceTimersByTime(15_000);
vi.advanceTimersByTime(15_000);
vi.advanceTimersByTime(15_000);
const calls = recordActivityEvent.mock.calls.filter(
([e]) => (e as { kind: string }).kind === "ui.terminal_heartbeat_lost",
);
expect(calls.length).toBe(1);
expect(ws.terminate).toHaveBeenCalled();
// Issue invariant: at most one emit per state change — extra ticks must not
// produce another event.
vi.advanceTimersByTime(15_000);
expect(
recordActivityEvent.mock.calls.filter(
([e]) => (e as { kind: string }).kind === "ui.terminal_heartbeat_lost",
).length,
).toBe(1);
});
it("does NOT emit heartbeat_lost when pong arrives before 3 missed pings", () => {
const { ws } = emitConnection();
recordActivityEvent.mockClear();
vi.advanceTimersByTime(15_000); // missedPongs=1
ws.emit("pong"); // resets to 0
vi.advanceTimersByTime(15_000); // missedPongs=1
vi.advanceTimersByTime(15_000); // missedPongs=2
expect(
recordActivityEvent.mock.calls.filter(
([e]) => (e as { kind: string }).kind === "ui.terminal_heartbeat_lost",
).length,
).toBe(0);
expect(ws.terminate).not.toHaveBeenCalled();
});
it("emits ui.terminal_protocol_error on malformed client message", () => {
const { ws } = emitConnection();
recordActivityEvent.mockClear();
ws.emit("message", Buffer.from("not-json{{{"));
const calls = recordActivityEvent.mock.calls.filter(
([e]) => (e as { kind: string }).kind === "ui.terminal_protocol_error",
);
expect(calls.length).toBe(1);
const evt = findEvent("ui.terminal_protocol_error")!;
expect(evt.data["errorMessage"]).toBeTruthy();
});
});
describe("Windows pipe ui.terminal_pty_lost activity events", () => {
beforeEach(() => {
recordActivityEvent.mockClear();
});
function framedMessage(type: number, payload: unknown): Buffer {
const body = Buffer.from(JSON.stringify(payload), "utf-8");
const header = Buffer.alloc(5);
header.writeUInt8(type, 0);
header.writeUInt32BE(body.length, 1);
return Buffer.concat([header, body]);
}
function openPipe() {
const ws = new FakeWS();
const pipe = new FakePipeSocket();
const winPipes = new Map<string, Socket>();
const winPipeBuffers = new Map<string, Buffer>();
const deps = {
connect: vi.fn(() => pipe as unknown as Socket),
resolvePipePath: vi.fn(() => "\\\\.\\pipe\\ao-pty-app-1"),
};
handleWindowsPipeMessage(
{ id: "app-1", type: "open", projectId: "proj-1" },
ws,
winPipes,
winPipeBuffers,
deps,
);
pipe.emit("connect");
recordActivityEvent.mockClear();
ws.send.mockClear();
return { ws, pipe, winPipes, winPipeBuffers, deps };
}
function ptyLostEvents(): Array<{ data: Record<string, unknown>; sessionId?: string }> {
return recordActivityEvent.mock.calls
.map(([e]) => e as { kind: string; data: Record<string, unknown>; sessionId?: string })
.filter((event) => event.kind === "ui.terminal_pty_lost");
}
it("emits ui.terminal_pty_lost when the PTY host pipe closes while the socket is open", () => {
const { ws, pipe } = openPipe();
pipe.emit("close");
expect(ptyLostEvents()).toEqual([
expect.objectContaining({
sessionId: "app-1",
data: expect.objectContaining({
sessionId: "app-1",
transport: "windows_pipe",
reason: "pipe_closed",
}),
}),
]);
expect(ws.send).toHaveBeenCalledWith(
JSON.stringify({ ch: "terminal", id: "app-1", type: "exited", code: 0, projectId: "proj-1" }),
);
});
it("emits ui.terminal_pty_lost when the PTY host reports not alive", () => {
const { ws, pipe } = openPipe();
pipe.emit("data", framedMessage(0x07, { alive: false }));
pipe.emit("close");
const events = ptyLostEvents();
expect(events).toHaveLength(1);
expect(events[0]).toEqual(
expect.objectContaining({
sessionId: "app-1",
data: expect.objectContaining({
sessionId: "app-1",
transport: "windows_pipe",
reason: "host_not_alive",
}),
}),
);
expect(ws.send).toHaveBeenCalledWith(
JSON.stringify({ ch: "terminal", id: "app-1", type: "exited", code: 0, projectId: "proj-1" }),
);
});
it("does not emit ui.terminal_pty_lost for an intentional client close", () => {
const { ws, winPipes, winPipeBuffers, deps } = openPipe();
handleWindowsPipeMessage(
{ id: "app-1", type: "close", projectId: "proj-1" },
ws,
winPipes,
winPipeBuffers,
deps,
);
expect(ptyLostEvents()).toEqual([]);
});
});
describe("TerminalManager ui.terminal_pty_lost activity events", () => {
beforeEach(() => {
recordActivityEvent.mockClear();
resetPtyMock();
});
function ptyLostEvents(): Array<{ data: Record<string, unknown>; sessionId?: string }> {
return recordActivityEvent.mock.calls
.map(([e]) => e as { kind: string; data: Record<string, unknown>; sessionId?: string })
.filter((event) => event.kind === "ui.terminal_pty_lost");
}
it("emits ui.terminal_pty_lost when a subscribed PTY exits and reattach fails", async () => {
const manager = new TerminalManager("/usr/bin/tmux");
manager.open("app-1", "proj-1", "tmux-app-1");
const pty = ptyInstances[0];
expect(pty).toBeDefined();
manager.subscribe("app-1", "proj-1", vi.fn(), vi.fn());
mockPtySpawn.mockImplementationOnce(() => {
throw new Error("reattach unavailable");
});
await pty!.emitExit(9);
const events = ptyLostEvents();
expect(events).toHaveLength(1);
expect(events[0]).toEqual(
expect.objectContaining({
sessionId: "app-1",
data: expect.objectContaining({
sessionId: "app-1",
exitCode: 9,
subscriberCount: 1,
reattachError: "reattach unavailable",
}),
}),
);
});
it("emits ui.terminal_pty_lost when a subscribed PTY exits and reattach succeeds", async () => {
const manager = new TerminalManager("/usr/bin/tmux");
manager.open("app-1", "proj-1", "tmux-app-1");
const pty = ptyInstances[0];
expect(pty).toBeDefined();
manager.subscribe("app-1", "proj-1", vi.fn(), vi.fn());
await pty!.emitExit(9);
const events = ptyLostEvents();
expect(mockPtySpawn).toHaveBeenCalledTimes(2);
expect(events).toHaveLength(1);
expect(events[0]).toEqual(
expect.objectContaining({
sessionId: "app-1",
data: expect.objectContaining({
sessionId: "app-1",
exitCode: 9,
subscriberCount: 1,
reattachRecovered: true,
reattachExhausted: false,
}),
}),
);
});
it("does not emit ui.terminal_pty_lost when the last subscriber already left", async () => {
const manager = new TerminalManager("/usr/bin/tmux");
manager.open("app-1", "proj-1", "tmux-app-1");
const pty = ptyInstances[0];
expect(pty).toBeDefined();
const unsubscribe = manager.subscribe("app-1", "proj-1", vi.fn(), vi.fn());
unsubscribe();
await pty!.emitExit(0);
expect(ptyLostEvents()).toEqual([]);
});
it("emits ui.terminal_pty_lost at most once across reattach cycles", async () => {
const manager = new TerminalManager("/usr/bin/tmux");
manager.open("app-1", "proj-1", "tmux-app-1");
const firstPty = ptyInstances[0];
expect(firstPty).toBeDefined();
manager.subscribe("app-1", "proj-1", vi.fn(), vi.fn());
mockPtySpawn.mockImplementationOnce(() => {
throw new Error("first reattach unavailable");
});
await firstPty!.emitExit(7);
expect(ptyLostEvents()).toHaveLength(1);
// A client may try to re-open the terminal after the first PTY loss.
// A second failed reattach should not produce another activity event for
// the same terminal entry.
manager.open("app-1", "proj-1", "tmux-app-1");
const secondPty = ptyInstances[1];
expect(secondPty).toBeDefined();
mockPtySpawn.mockImplementationOnce(() => {
throw new Error("second reattach unavailable");
});
await secondPty!.emitExit(8);
expect(ptyLostEvents()).toHaveLength(1);
});
it("re-arms ui.terminal_pty_lost after a successful reattach survives the grace period", async () => {
vi.useFakeTimers();
try {
const manager = new TerminalManager("/usr/bin/tmux");
manager.open("app-1", "proj-1", "tmux-app-1");
const firstPty = ptyInstances[0];
expect(firstPty).toBeDefined();
manager.subscribe("app-1", "proj-1", vi.fn(), vi.fn());
await firstPty!.emitExit(7);
expect(ptyLostEvents()).toHaveLength(1);
await vi.advanceTimersByTimeAsync(5_000);
const secondPty = ptyInstances[1];
expect(secondPty).toBeDefined();
await secondPty!.emitExit(8);
expect(ptyLostEvents()).toHaveLength(2);
} finally {
vi.useRealTimers();
}
});
});
describe("TerminalManager.open — tmux target args (regression for #1714)", () => {
@ -325,6 +832,7 @@ describe("TerminalManager.open — re-attach skipped when tmux session is gone (
mockSpawn.mockReset();
mockPtySpawn.mockReset();
mockTmuxHasSession.mockReset();
recordActivityEvent.mockClear();
capturedOnExit = undefined;
mockSpawn.mockImplementation(() => new EventEmitter());
@ -355,6 +863,20 @@ describe("TerminalManager.open — re-attach skipped when tmux session is gone (
// Subscribers were notified with the original exit code.
expect(exitCb).toHaveBeenCalledTimes(1);
expect(exitCb).toHaveBeenCalledWith(0);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "ui",
kind: "ui.terminal_pty_lost",
level: "warn",
sessionId: "ao-177",
data: expect.objectContaining({
sessionId: "ao-177",
exitCode: 0,
reattachSkipped: true,
tmuxSessionPresent: false,
}),
}),
);
});
it("still re-attaches when has-session reports the tmux session is alive", async () => {

237
packages/web/server/mux-websocket.ts
View File

@ -16,7 +16,7 @@ import {
tmuxHasSession,
validateSessionId,
} from "./tmux-utils.js";
import { getEnvDefaults, isWindows } from "@aoagents/ao-core";
import { getEnvDefaults, isWindows, recordActivityEvent } from "@aoagents/ao-core";
// These types mirror src/lib/mux-protocol.ts exactly.
// tsconfig.server.json constrains rootDir to "server/", so we cannot import
@ -63,6 +63,9 @@ export class SessionBroadcaster {
private errorSubscribers = new Set<(error: string) => void>();
private intervalId: ReturnType<typeof setInterval> | null = null;
private polling = false;
// Tracks the last fetch outcome so we only emit ui.session_broadcast_failed on
// the healthy → failing transition (not every 3s during an outage).
private lastFetchOk = true;
private readonly baseUrl: string;
constructor(nextPort: string) {
@ -158,18 +161,42 @@ export class SessionBroadcaster {
if (!res.ok) {
const msg = `Session fetch failed: HTTP ${res.status}`;
console.warn(`[SessionBroadcaster] ${msg}`);
this.recordFetchFailure(msg, { httpStatus: res.status });
return { sessions: null, error: msg };
}
const data = (await res.json()) as { sessions?: SessionPatch[] };
this.lastFetchOk = true;
return { sessions: data.sessions ?? null, error: null };
} catch (err) {
clearTimeout(timeoutId);
const msg = err instanceof Error ? err.message : String(err);
console.warn("[SessionBroadcaster] fetchSnapshot error:", msg);
this.recordFetchFailure(msg);
return { sessions: null, error: msg };
}
}
/**
* Emit ui.session_broadcast_failed once per healthy→failing transition.
* The broadcaster polls every 3s; emitting on every failure during a long
* outage would flood the events table (~20/min). Recovery resets the flag.
*/
private recordFetchFailure(message: string, extra?: Record<string, unknown>): void {
if (!this.lastFetchOk) return;
this.lastFetchOk = false;
recordActivityEvent({
source: "ui",
kind: "ui.session_broadcast_failed",
level: "warn",
summary: `session broadcaster fetch failed: ${message}`,
data: {
url: `${this.baseUrl}/api/sessions/patches`,
errorMessage: message,
...extra,
},
});
}
private disconnect(): void {
if (this.intervalId !== null) {
clearInterval(this.intervalId);
@ -199,6 +226,7 @@ interface ManagedTerminal {
buffer: string[];
bufferBytes: number;
reattachAttempts: number;
ptyLostEmitted: boolean;
/**
* Pending grace-period timer that resets reattachAttempts when the
* currently-attached PTY survives REATTACH_RESET_GRACE_MS. Tracked so
@ -276,6 +304,7 @@ export class TerminalManager {
buffer: [],
bufferBytes: 0,
reattachAttempts: 0,
ptyLostEmitted: false,
};
this.terminals.set(key, terminal);
}
@ -346,6 +375,7 @@ export class TerminalManager {
terminal.resetTimer = undefined;
if (terminal.pty === pty) {
terminal.reattachAttempts = 0;
terminal.ptyLostEmitted = false;
}
}, REATTACH_RESET_GRACE_MS);
terminal.resetTimer.unref();
@ -383,6 +413,7 @@ export class TerminalManager {
pty.onExit(async ({ exitCode }) => {
console.log(`[MuxServer] PTY exited for ${id} with code ${exitCode}`);
terminal.pty = null;
let reattachError: string | undefined;
// Skip the re-attach loop entirely when the underlying tmux session is
// gone (e.g. user pressed Ctrl-C in the pane and the launch command
@ -392,15 +423,33 @@ export class TerminalManager {
// clean user-initiated termination — see issue #1756. The
// MAX_REATTACH_ATTEMPTS bound from #1640 still covers tmux server
// hiccups where the session does still exist.
if (
terminal.subscribers.size > 0 &&
!(await tmuxHasSession(this.TMUX, tmuxSessionId))
) {
if (terminal.subscribers.size > 0 && !(await tmuxHasSession(this.TMUX, tmuxSessionId))) {
console.log(`[MuxServer] tmux session ${tmuxSessionId} is gone, not re-attaching`);
if (terminal.resetTimer) {
clearTimeout(terminal.resetTimer);
terminal.resetTimer = undefined;
}
if (!terminal.ptyLostEmitted) {
terminal.ptyLostEmitted = true;
recordActivityEvent({
projectId,
sessionId: id,
source: "ui",
kind: "ui.terminal_pty_lost",
level: "warn",
summary: `terminal PTY exited (code ${exitCode}) — tmux session gone`,
data: {
sessionId: id,
exitCode,
reattachAttempts: terminal.reattachAttempts,
maxReattachAttempts: MAX_REATTACH_ATTEMPTS,
reattachExhausted: false,
reattachSkipped: true,
tmuxSessionPresent: false,
subscriberCount: terminal.subscribers.size,
},
});
}
for (const cb of terminal.exitCallbacks) {
cb(exitCode);
}
@ -423,14 +472,63 @@ export class TerminalManager {
);
try {
this.open(id, projectId, tmuxSessionId);
if (!terminal.ptyLostEmitted) {
terminal.ptyLostEmitted = true;
recordActivityEvent({
projectId,
sessionId: id,
source: "ui",
kind: "ui.terminal_pty_lost",
level: "warn",
summary: `terminal PTY exited (code ${exitCode}) — reattached`,
data: {
sessionId: id,
exitCode,
reattachAttempts: terminal.reattachAttempts,
maxReattachAttempts: MAX_REATTACH_ATTEMPTS,
reattachExhausted: false,
reattachRecovered: true,
subscriberCount: terminal.subscribers.size,
},
});
}
return; // re-attached — don't notify exit
} catch (err) {
reattachError = err instanceof Error ? err.message : String(err);
console.error(`[MuxServer] Failed to re-attach ${id}:`, err);
}
} else if (terminal.reattachAttempts >= MAX_REATTACH_ATTEMPTS) {
console.error(`[MuxServer] Max re-attach attempts reached for ${id}, giving up`);
}
// PTY actually died (vs user closed browser): only emit when subscribers
// are still attached — otherwise the exit is just normal cleanup.
// Keep this event one-shot for the terminal entry. Clients may re-open
// the same terminal after a failed reattach; repeated PTY exits should
// not flood the activity log for the same loss condition.
if (terminal.subscribers.size > 0 && !terminal.ptyLostEmitted) {
terminal.ptyLostEmitted = true;
recordActivityEvent({
projectId,
sessionId: id,
source: "ui",
kind: "ui.terminal_pty_lost",
level: "warn",
summary: `terminal PTY exited (code ${exitCode})${
terminal.reattachAttempts >= MAX_REATTACH_ATTEMPTS ? " — reattach exhausted" : ""
}`,
data: {
sessionId: id,
exitCode,
reattachAttempts: terminal.reattachAttempts,
maxReattachAttempts: MAX_REATTACH_ATTEMPTS,
reattachExhausted: terminal.reattachAttempts >= MAX_REATTACH_ATTEMPTS,
subscriberCount: terminal.subscribers.size,
...(reattachError ? { reattachError } : {}),
},
});
}
// Notify subscribers that the terminal has exited (re-attach failed or no subscribers)
for (const cb of terminal.exitCallbacks) {
cb(exitCode);
@ -515,6 +613,8 @@ export class TerminalManager {
// ── Windows Pipe Relay (extracted for testability) ──
const intentionalWinPipeCloses = new WeakSet<Socket>();
/** Minimal WebSocket-like interface for the pipe relay handler */
export interface WsSink {
send(data: string): void;
@ -586,8 +686,36 @@ export function handleWindowsPipeMessage(
const pipeSocket = deps.connect(pipePath);
winPipes.set(pipeKey, pipeSocket);
winPipeBuffers.set(pipeKey, Buffer.alloc(0));
let ptyLostEmitted = false;
const recordWindowsPtyLost = (
reason: "pipe_closed" | "host_not_alive" | "pipe_error",
extra?: Record<string, unknown>,
): void => {
if (ptyLostEmitted || ws.readyState !== WS_OPEN) return;
ptyLostEmitted = true;
recordActivityEvent({
projectId,
sessionId: id,
source: "ui",
kind: "ui.terminal_pty_lost",
level: "warn",
summary:
reason === "host_not_alive"
? `terminal PTY host reported not alive for ${id}`
: reason === "pipe_error"
? `terminal PTY host pipe errored for ${id}`
: `terminal PTY host pipe closed for ${id}`,
data: {
sessionId: id,
transport: "windows_pipe",
reason,
...extra,
},
});
};
pipeSocket.on("error", (err) => {
recordWindowsPtyLost("pipe_error", { errorMessage: err.message });
winPipes.delete(pipeKey);
winPipeBuffers.delete(pipeKey);
pipeSocket.destroy();
@ -637,9 +765,8 @@ export function handleWindowsPipeMessage(
try {
const status = JSON.parse(payload.toString("utf-8")) as { alive: boolean };
if (!status.alive && ws.readyState === WS_OPEN) {
ws.send(
JSON.stringify({ ch: "terminal", id, type: "exited", code: 0, ...echo }),
);
recordWindowsPtyLost("host_not_alive");
ws.send(JSON.stringify({ ch: "terminal", id, type: "exited", code: 0, ...echo }));
}
} catch {
/* ignore parse errors */
@ -651,7 +778,11 @@ export function handleWindowsPipeMessage(
pipeSocket.on("close", () => {
winPipes.delete(pipeKey);
winPipeBuffers.delete(pipeKey);
const intentionalClose = intentionalWinPipeCloses.delete(pipeSocket);
if (ws.readyState === WS_OPEN) {
if (!intentionalClose) {
recordWindowsPtyLost("pipe_closed");
}
ws.send(JSON.stringify({ ch: "terminal", id, type: "exited", code: 0, ...echo }));
}
});
@ -678,6 +809,7 @@ export function handleWindowsPipeMessage(
} else if (type === "close") {
const pipeSocket = winPipes.get(pipeKey);
if (pipeSocket) {
intentionalWinPipeCloses.add(pipeSocket);
pipeSocket.end();
winPipes.delete(pipeKey);
winPipeBuffers.delete(pipeKey);
@ -706,9 +838,26 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
const wss = new WebSocketServer({ noServer: true });
wss.on("connection", (ws) => {
wss.on("connection", (ws, request) => {
console.log("[MuxServer] New mux connection");
const connectedAt = Date.now();
// Best-effort remote addr — proxy headers if present, else socket peer.
const xff = request?.headers["x-forwarded-for"];
const xffStr = Array.isArray(xff) ? xff[0] : xff;
const remoteAddr =
(typeof xffStr === "string" ? xffStr.split(",")[0]?.trim() : undefined) ??
request?.socket?.remoteAddress ??
undefined;
recordActivityEvent({
source: "ui",
kind: "ui.terminal_connected",
level: "info",
summary: "mux WebSocket connection opened",
data: { remoteAddr },
});
const subscriptions = new Map<string, () => void>();
// Windows: named pipe sockets keyed by session ID
const winPipes = new Map<string, ReturnType<typeof netConnect>>();
@ -716,6 +865,7 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
const winPipeBuffers = new Map<string, Buffer>();
let sessionUnsubscribe: (() => void) | null = null;
let missedPongs = 0;
let heartbeatLostEmitted = false;
const MAX_MISSED_PONGS = 3;
// Heartbeat: send native WebSocket ping every 15s.
@ -728,6 +878,22 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
missedPongs += 1;
if (missedPongs >= MAX_MISSED_PONGS) {
console.log("[MuxServer] Too many missed pongs, terminating connection");
if (!heartbeatLostEmitted) {
heartbeatLostEmitted = true;
recordActivityEvent({
source: "ui",
kind: "ui.terminal_heartbeat_lost",
level: "warn",
summary: `mux WebSocket heartbeat lost (${missedPongs} missed pongs)`,
data: {
missedPongs,
maxMissedPongs: MAX_MISSED_PONGS,
connectionAgeMs: Date.now() - connectedAt,
remoteAddr,
subscriberCount: subscriptions.size,
},
});
}
ws.terminate();
}
}
@ -759,7 +925,14 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
if (type === "open") {
if (isWindows()) {
handleWindowsPipeMessage(
msg as { id: string; type: string; projectId?: string; data?: string; cols?: number; rows?: number },
msg as {
id: string;
type: string;
projectId?: string;
data?: string;
cols?: number;
rows?: number;
},
ws,
winPipes,
winPipeBuffers,
@ -841,7 +1014,13 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
} else if (type === "resize" && "cols" in msg && "rows" in msg) {
if (isWindows()) {
handleWindowsPipeMessage(
msg as { id: string; type: string; projectId?: string; cols: number; rows: number },
msg as {
id: string;
type: string;
projectId?: string;
cols: number;
rows: number;
},
ws,
winPipes,
winPipeBuffers,
@ -853,7 +1032,14 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
} else if (type === "close") {
if (isWindows()) {
handleWindowsPipeMessage(
msg as { id: string; type: string; projectId?: string; data?: string; cols?: number; rows?: number },
msg as {
id: string;
type: string;
projectId?: string;
data?: string;
cols?: number;
rows?: number;
},
ws,
winPipes,
winPipeBuffers,
@ -903,6 +1089,17 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
}
} catch (err) {
console.error("[MuxServer] Failed to parse message:", err);
recordActivityEvent({
source: "ui",
kind: "ui.terminal_protocol_error",
level: "warn",
summary: "invalid mux client message — parse failed",
data: {
errorMessage: err instanceof Error ? err.message : String(err),
remoteAddr,
subscriberCount: subscriptions.size,
},
});
const errorMsg: ServerMessage = {
ch: "system",
type: "error",
@ -917,8 +1114,22 @@ export function createMuxWebSocket(tmuxPath?: string | null): WebSocketServer |
/**
* Handle connection close
*/
ws.on("close", () => {
ws.on("close", (code, reason) => {
console.log("[MuxServer] Mux connection closed");
recordActivityEvent({
source: "ui",
kind: "ui.terminal_disconnected",
level: "info",
summary: "mux WebSocket connection closed",
data: {
code,
reason: reason?.toString("utf8") || undefined,
connectionAgeMs: Date.now() - connectedAt,
subscriberCount: subscriptions.size,
heartbeatLost: heartbeatLostEmitted,
remoteAddr,
},
});
clearInterval(heartbeatInterval);
sessionUnsubscribe?.();
sessionUnsubscribe = null;

164
packages/web/src/__tests__/activity-events-projects.test.ts Normal file
View File

@ -0,0 +1,164 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { execSync } from "node:child_process";
import { NextRequest } from "next/server";
import { recordActivityEvent, registerProjectInGlobalConfig } from "@aoagents/ao-core";
vi.mock("@aoagents/ao-core", async () => {
const actual = await vi.importActual("@aoagents/ao-core");
return {
...(actual as Record<string, unknown>),
recordActivityEvent: vi.fn(),
};
});
const invalidatePortfolioServicesCache = vi.fn();
const getServices = vi.fn();
vi.mock("@/lib/services", () => ({
invalidatePortfolioServicesCache,
getServices,
}));
function makeRequest(method: string, url: string, body?: Record<string, unknown>): NextRequest {
return new NextRequest(url, {
method,
body: body ? JSON.stringify(body) : undefined,
headers: body ? { "Content-Type": "application/json" } : undefined,
});
}
const recorded = vi.mocked(recordActivityEvent);
describe("Activity events — project mutation routes", () => {
let oldGlobalConfig: string | undefined;
let oldConfigPath: string | undefined;
let oldHome: string | undefined;
let tempRoot: string;
let configPath: string;
beforeEach(() => {
vi.resetModules();
recorded.mockClear();
invalidatePortfolioServicesCache.mockReset();
getServices.mockReset();
getServices.mockResolvedValue({
registry: { get: vi.fn().mockReturnValue(null) },
sessionManager: {
list: vi.fn().mockResolvedValue([]),
kill: vi.fn().mockResolvedValue(undefined),
},
});
oldGlobalConfig = process.env["AO_GLOBAL_CONFIG"];
oldConfigPath = process.env["AO_CONFIG_PATH"];
oldHome = process.env["HOME"];
tempRoot = mkdtempSync(path.join(tmpdir(), "ao-activity-projects-"));
configPath = path.join(tempRoot, "config.yaml");
process.env["AO_GLOBAL_CONFIG"] = configPath;
process.env["AO_CONFIG_PATH"] = configPath;
process.env["HOME"] = tempRoot;
});
afterEach(() => {
if (oldGlobalConfig === undefined) delete process.env["AO_GLOBAL_CONFIG"];
else process.env["AO_GLOBAL_CONFIG"] = oldGlobalConfig;
if (oldConfigPath === undefined) delete process.env["AO_CONFIG_PATH"];
else process.env["AO_CONFIG_PATH"] = oldConfigPath;
if (oldHome === undefined) delete process.env["HOME"];
else process.env["HOME"] = oldHome;
rmSync(tempRoot, { recursive: true, force: true });
});
it("POST /api/projects emits api.project_added on success", async () => {
const repoDir = path.join(tempRoot, "demo-add");
mkdirSync(repoDir, { recursive: true });
execSync("git init -q", { cwd: repoDir });
const { POST } = await import("@/app/api/projects/route");
const res = await POST(
makeRequest("POST", "http://localhost:3000/api/projects", {
path: repoDir,
projectId: "demo-add",
name: "Demo Add",
}),
);
expect(res.status).toBe(201);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.project_added",
}),
);
});
it("PATCH /api/projects/:id emits api.project_updated with changed keys (not values)", async () => {
const repoDir = path.join(tempRoot, "demo-patch");
mkdirSync(repoDir, { recursive: true });
const effectiveId = registerProjectInGlobalConfig("demo-patch", "Demo Patch", repoDir);
const { PATCH } = await import("@/app/api/projects/[id]/route");
const res = await PATCH(
makeRequest("PATCH", `http://localhost:3000/api/projects/${effectiveId}`, {
agent: "codex",
runtime: "tmux",
someUnknownField: "do-not-record",
}),
{ params: Promise.resolve({ id: effectiveId }) },
);
expect(res.status).toBe(200);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.project_updated",
projectId: effectiveId,
data: expect.objectContaining({
changedKeys: expect.arrayContaining(["agent", "runtime"]),
}),
}),
);
// Ensure no value content (e.g. "codex", "tmux") leaked into the event payload
const calls = recorded.mock.calls.filter(
(c) => (c[0] as { kind: string }).kind === "api.project_updated",
);
expect(calls[0]?.[0]).toEqual(
expect.objectContaining({
data: expect.objectContaining({
changedKeys: ["agent", "runtime"],
}),
}),
);
for (const [event] of calls) {
const json = JSON.stringify(event);
expect(json).not.toContain("codex");
expect(json).not.toContain('"tmux"');
expect(json).not.toContain("someUnknownField");
expect(json).not.toContain("do-not-record");
}
});
it("DELETE /api/projects/:id emits api.project_removed on success", async () => {
const repoDir = path.join(tempRoot, "demo-delete");
mkdirSync(repoDir, { recursive: true });
const effectiveId = registerProjectInGlobalConfig("demo-delete", "Demo Delete", repoDir);
const { DELETE } = await import("@/app/api/projects/[id]/route");
const res = await DELETE(
makeRequest("DELETE", `http://localhost:3000/api/projects/${effectiveId}`),
{ params: Promise.resolve({ id: effectiveId }) },
);
expect(res.status).toBe(200);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.project_removed",
projectId: effectiveId,
}),
);
});
});

498
packages/web/src/__tests__/activity-events-routes.test.ts Normal file
View File

@ -0,0 +1,498 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
import {
SessionNotFoundError,
SessionNotRestorableError,
WorkspaceMissingError,
recordActivityEvent,
createInitialCanonicalLifecycle,
createActivitySignal,
type Session,
type SessionManager,
type OrchestratorConfig,
type PluginRegistry,
type SCM,
} from "@aoagents/ao-core";
// Partial mock so we replace recordActivityEvent but keep types/helpers
vi.mock("@aoagents/ao-core", async () => {
const actual = await vi.importActual("@aoagents/ao-core");
return {
...(actual as Record<string, unknown>),
recordActivityEvent: vi.fn(),
};
});
vi.mock("@/lib/observability", async () => {
const actual = await vi.importActual("@/lib/observability");
return {
...(actual as Record<string, unknown>),
recordApiObservation: vi.fn(),
};
});
function makeSession(overrides: Partial<Session> & { id: string }): Session {
const lifecycle = createInitialCanonicalLifecycle("worker", new Date());
lifecycle.session.state = "working";
lifecycle.session.reason = "task_in_progress";
lifecycle.session.startedAt = lifecycle.session.lastTransitionAt;
lifecycle.runtime.state = "alive";
lifecycle.runtime.reason = "process_running";
return {
projectId: "my-app",
status: "working",
activity: "active",
activitySignal: createActivitySignal("valid", {
activity: "active",
timestamp: new Date(),
source: "native",
}),
lifecycle,
branch: null,
issueId: null,
pr: null,
workspacePath: null,
runtimeHandle: null,
agentInfo: null,
createdAt: new Date(),
lastActivityAt: new Date(),
metadata: {},
...overrides,
};
}
const baseSessions: Session[] = [
makeSession({ id: "backend-3" }),
makeSession({
id: "backend-7",
pr: {
number: 432,
url: "https://github.com/acme/my-app/pull/432",
title: "feat: health check",
owner: "acme",
repo: "my-app",
branch: "feat/health-check",
baseBranch: "main",
isDraft: false,
},
}),
makeSession({ id: "frontend-1", status: "killed", activity: "exited" }),
];
const mockSessionManager: SessionManager = {
list: vi.fn(async () => baseSessions),
listCached: vi.fn(async () => baseSessions),
invalidateCache: vi.fn(),
get: vi.fn(async (id: string) => baseSessions.find((s) => s.id === id) ?? null),
spawn: vi.fn(async (cfg) =>
makeSession({
id: `session-${Date.now()}`,
projectId: cfg.projectId,
issueId: cfg.issueId ?? null,
status: "spawning",
}),
),
kill: vi.fn(async (id: string) => {
if (!baseSessions.find((s) => s.id === id)) {
throw new SessionNotFoundError(id);
}
}),
send: vi.fn(async (id: string) => {
if (!baseSessions.find((s) => s.id === id)) {
throw new SessionNotFoundError(id);
}
}),
cleanup: vi.fn(async () => ({ killed: [], skipped: [], errors: [] })),
spawnOrchestrator: vi.fn(async () =>
makeSession({
id: "my-app-orchestrator",
projectId: "my-app",
metadata: { role: "orchestrator" },
}),
),
relaunchOrchestrator: vi.fn(async () =>
makeSession({
id: "my-app-orchestrator",
projectId: "my-app",
metadata: { role: "orchestrator" },
}),
),
ensureOrchestrator: vi.fn(),
remap: vi.fn(async () => "ses_mock"),
restore: vi.fn(async (id: string) => {
const session = baseSessions.find((s) => s.id === id);
if (!session) throw new SessionNotFoundError(id);
return { ...session, status: "spawning" as const, activity: "active" as const };
}),
};
const mockSCM: SCM = {
name: "github",
detectPR: vi.fn(async () => null),
getPRState: vi.fn(async () => "open" as const),
mergePR: vi.fn(async () => {}),
closePR: vi.fn(async () => {}),
getCIChecks: vi.fn(async () => []),
getCISummary: vi.fn(async () => "passing" as const),
getReviews: vi.fn(async () => []),
getReviewDecision: vi.fn(async () => "approved" as const),
getPendingComments: vi.fn(async () => []),
getAutomatedComments: vi.fn(async () => []),
getMergeability: vi.fn(async () => ({
mergeable: true,
ciPassing: true,
approved: true,
noConflicts: true,
blockers: [],
})),
};
const mockRegistry: PluginRegistry = {
register: vi.fn(),
get: vi.fn(() => mockSCM) as PluginRegistry["get"],
list: vi.fn(() => []),
loadBuiltins: vi.fn(async () => {}),
loadFromConfig: vi.fn(async () => {}),
};
const mockConfig: OrchestratorConfig = {
configPath: "/tmp/ao-test/agent-orchestrator.yaml",
port: 3000,
readyThresholdMs: 300_000,
defaults: { runtime: "tmux", agent: "claude-code", workspace: "worktree", notifiers: [] },
projects: {
"my-app": {
name: "My App",
repo: "acme/my-app",
path: "/tmp/my-app",
defaultBranch: "main",
sessionPrefix: "my-app",
scm: { plugin: "github" },
},
},
notifiers: {},
notificationRouting: { urgent: [], action: [], warning: [], info: [] },
reactions: {},
};
vi.mock("@/lib/services", () => ({
getServices: vi.fn(async () => ({
config: mockConfig,
registry: mockRegistry,
sessionManager: mockSessionManager,
})),
getVerifyIssues: vi.fn(async () => []),
getSCM: vi.fn(() => mockSCM),
invalidatePortfolioServicesCache: vi.fn(),
}));
import { getServices } from "@/lib/services";
import { recordApiObservation } from "@/lib/observability";
import { POST as spawnPOST } from "@/app/api/spawn/route";
import { POST as killPOST } from "@/app/api/sessions/[id]/kill/route";
import { POST as sendPOST } from "@/app/api/sessions/[id]/send/route";
import { POST as messagePOST } from "@/app/api/sessions/[id]/message/route";
import { POST as restorePOST } from "@/app/api/sessions/[id]/restore/route";
import { POST as orchestratorsPOST } from "@/app/api/orchestrators/route";
import { POST as mergePOST } from "@/app/api/prs/[id]/merge/route";
function makeRequest(url: string, init?: RequestInit): NextRequest {
return new NextRequest(
new URL(url, "http://localhost:3000"),
init as ConstructorParameters<typeof NextRequest>[1],
);
}
const recorded = vi.mocked(recordActivityEvent);
beforeEach(() => {
recorded.mockClear();
vi.mocked(recordApiObservation).mockClear();
vi.mocked(getServices).mockClear();
});
describe("API mutation routes emit activity events (api source)", () => {
describe("MUST emits — session mutations", () => {
it("POST /api/spawn emits api.session_spawn_requested on success", async () => {
const req = makeRequest("/api/spawn", {
method: "POST",
body: JSON.stringify({ projectId: "my-app", issueId: "INT-100" }),
headers: { "Content-Type": "application/json" },
});
await spawnPOST(req);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_spawn_requested",
projectId: "my-app",
}),
);
});
it("POST /api/sessions/:id/kill emits api.session_kill_requested on success", async () => {
const req = makeRequest("/api/sessions/backend-3/kill", { method: "POST" });
await killPOST(req, { params: Promise.resolve({ id: "backend-3" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_kill_requested",
sessionId: "backend-3",
}),
);
});
it("POST /api/sessions/:id/send emits api.session_message_sent with messageLength", async () => {
const req = makeRequest("/api/sessions/backend-3/send", {
method: "POST",
body: JSON.stringify({ message: "Fix the tests" }),
headers: { "Content-Type": "application/json" },
});
await sendPOST(req, { params: Promise.resolve({ id: "backend-3" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_message_sent",
sessionId: "backend-3",
data: expect.objectContaining({ messageLength: "Fix the tests".length }),
}),
);
});
it("POST /api/sessions/:id/send does NOT include the raw message in data", async () => {
const secret = "very-secret-PII content";
const req = makeRequest("/api/sessions/backend-3/send", {
method: "POST",
body: JSON.stringify({ message: secret }),
headers: { "Content-Type": "application/json" },
});
await sendPOST(req, { params: Promise.resolve({ id: "backend-3" }) });
const calls = recorded.mock.calls.filter(
(c) => (c[0] as { kind: string }).kind === "api.session_message_sent",
);
expect(calls.length).toBeGreaterThan(0);
for (const [event] of calls) {
const json = JSON.stringify(event);
expect(json).not.toContain(secret);
}
});
it("POST /api/sessions/:id/message emits api.session_message_sent with messageLength", async () => {
const req = makeRequest("/api/sessions/backend-3/message", {
method: "POST",
body: JSON.stringify({ message: "Hi" }),
headers: { "Content-Type": "application/json" },
});
await messagePOST(req, { params: Promise.resolve({ id: "backend-3" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_message_sent",
sessionId: "backend-3",
data: expect.objectContaining({ messageLength: 2 }),
}),
);
});
it("POST /api/sessions/:id/restore emits api.session_restore_requested on success", async () => {
const req = makeRequest("/api/sessions/frontend-1/restore", { method: "POST" });
await restorePOST(req, { params: Promise.resolve({ id: "frontend-1" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_restore_requested",
sessionId: "frontend-1",
}),
);
});
});
describe("MUST emits — orchestrator + PR mutations", () => {
it("POST /api/orchestrators emits api.orchestrator_spawn_requested on success", async () => {
const req = makeRequest("/api/orchestrators", {
method: "POST",
body: JSON.stringify({ projectId: "my-app" }),
headers: { "Content-Type": "application/json" },
});
await orchestratorsPOST(req);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.orchestrator_spawn_requested",
projectId: "my-app",
}),
);
});
it("POST /api/prs/:id/merge emits api.pr_merge_requested on success", async () => {
const req = makeRequest("/api/prs/432/merge", { method: "POST" });
await mergePOST(req, { params: Promise.resolve({ id: "432" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.pr_merge_requested",
data: expect.objectContaining({ prNumber: 432 }),
}),
);
});
});
describe("SHOULD emits — failure paths", () => {
it("POST /api/spawn emits api.session_spawn_rejected for unknown project", async () => {
const req = makeRequest("/api/spawn", {
method: "POST",
body: JSON.stringify({ projectId: "unknown-app" }),
headers: { "Content-Type": "application/json" },
});
await spawnPOST(req);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_spawn_rejected",
projectId: "unknown-app",
}),
);
});
it("POST /api/spawn does not emit api.session_spawn_failed when core spawn throws", async () => {
(mockSessionManager.spawn as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
new Error("runtime failed"),
);
const req = makeRequest("/api/spawn", {
method: "POST",
body: JSON.stringify({ projectId: "my-app", issueId: "INT-101" }),
headers: { "Content-Type": "application/json" },
});
const res = await spawnPOST(req);
expect(res.status).toBe(500);
expect(
recorded.mock.calls.some(
([event]) => (event as { kind: string }).kind === "api.session_spawn_failed",
),
).toBe(false);
});
it.each([
["spawn", false, mockSessionManager.spawnOrchestrator],
["clean relaunch", true, mockSessionManager.relaunchOrchestrator],
])(
"POST /api/orchestrators does not emit api.orchestrator_spawn_failed when core %s throws",
async (_name, clean, method) => {
(method as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("runtime failed"));
const req = makeRequest("/api/orchestrators", {
method: "POST",
body: JSON.stringify({ projectId: "my-app", clean }),
headers: { "Content-Type": "application/json" },
});
const res = await orchestratorsPOST(req);
expect(res.status).toBe(500);
expect(
recorded.mock.calls.some(
([event]) => (event as { kind: string }).kind === "api.orchestrator_spawn_failed",
),
).toBe(false);
},
);
it("POST /api/sessions/:id/send emits api.session_message_failed on unexpected error", async () => {
(mockSessionManager.send as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
new Error("write failed"),
);
const req = makeRequest("/api/sessions/backend-3/send", {
method: "POST",
body: JSON.stringify({ message: "hi" }),
headers: { "Content-Type": "application/json" },
});
await sendPOST(req, { params: Promise.resolve({ id: "backend-3" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_message_failed",
sessionId: "backend-3",
data: expect.objectContaining({ messageLength: 2 }),
}),
);
});
it.each([
["non-restorable session", new SessionNotRestorableError("my-app-123", "still working"), 409],
["missing workspace", new WorkspaceMissingError("/tmp/missing-workspace"), 422],
["unexpected restore error", new Error("restore failed"), 500],
])(
"POST /api/sessions/:id/restore emits attributed api.session_restore_failed for %s",
async (_name, error, statusCode) => {
(mockSessionManager.restore as ReturnType<typeof vi.fn>).mockRejectedValueOnce(error);
const req = makeRequest("/api/sessions/my-app-123/restore", { method: "POST" });
const res = await restorePOST(req, { params: Promise.resolve({ id: "my-app-123" }) });
expect(res.status).toBe(statusCode);
expect(vi.mocked(getServices)).toHaveBeenCalledTimes(1);
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.session_restore_failed",
projectId: "my-app",
sessionId: "my-app-123",
data: expect.objectContaining({ statusCode }),
}),
);
},
);
it("POST /api/prs/:id/merge emits api.pr_merge_rejected for non-mergeable PR", async () => {
(mockSCM.getMergeability as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
mergeable: false,
ciPassing: false,
approved: false,
noConflicts: true,
blockers: ["CI checks failing"],
});
const req = makeRequest("/api/prs/432/merge", { method: "POST" });
await mergePOST(req, { params: Promise.resolve({ id: "432" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.pr_merge_rejected",
data: expect.objectContaining({ prNumber: 432 }),
}),
);
});
it("POST /api/prs/:id/merge emits api.pr_merge_failed when mergePR throws", async () => {
(mockSCM.mergePR as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("github 500"));
const req = makeRequest("/api/prs/432/merge", { method: "POST" });
await mergePOST(req, { params: Promise.resolve({ id: "432" }) });
expect(recorded).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.pr_merge_failed",
projectId: "my-app",
sessionId: "backend-7",
data: expect.objectContaining({ prNumber: 432, reason: "github 500" }),
}),
);
expect(recordApiObservation).toHaveBeenCalledWith(
expect.objectContaining({
outcome: "failure",
statusCode: 500,
projectId: "my-app",
sessionId: "backend-7",
data: expect.objectContaining({ prNumber: 432 }),
}),
);
});
});
});

328
packages/web/src/__tests__/webhook-route.test.ts Normal file
View File

@ -0,0 +1,328 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import {
createInitialCanonicalLifecycle,
createActivitySignal,
type Session,
type SessionManager,
type OrchestratorConfig,
type PluginRegistry,
type SCM,
type LifecycleManager,
} from "@aoagents/ao-core";
// Activity event recording is mocked so we can assert what fires without
// touching the real SQLite layer.
const recordActivityEvent = vi.fn();
vi.mock("@aoagents/ao-core", async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>;
return {
...actual,
recordActivityEvent: (event: unknown) => recordActivityEvent(event),
};
});
// ── Mock services + plugin registry ───────────────────────────────────
function makeSession(id: string, overrides: Partial<Session> = {}): Session {
const lifecycle = createInitialCanonicalLifecycle("worker", new Date());
lifecycle.session.state = "working";
lifecycle.session.reason = "task_in_progress";
lifecycle.session.startedAt = lifecycle.session.lastTransitionAt;
lifecycle.runtime.state = "alive";
lifecycle.runtime.reason = "process_running";
return {
id,
projectId: "my-app",
status: "working",
activity: "active",
activitySignal: createActivitySignal("valid", {
activity: "active",
timestamp: new Date(),
source: "native",
}),
lifecycle,
branch: "feat/x",
issueId: null,
pr: {
number: 42,
url: "u",
title: "t",
owner: "acme",
repo: "my-app",
branch: "feat/x",
baseBranch: "main",
isDraft: false,
},
workspacePath: null,
runtimeHandle: null,
agentInfo: null,
createdAt: new Date(),
lastActivityAt: new Date(),
metadata: {},
...overrides,
};
}
const verifyWebhook = vi.fn();
const parseWebhook = vi.fn();
const mockSCM: SCM = {
name: "github",
detectPR: vi.fn(),
getPRState: vi.fn(),
mergePR: vi.fn(),
closePR: vi.fn(),
getCIChecks: vi.fn(),
getCISummary: vi.fn(),
getReviews: vi.fn(),
getReviewDecision: vi.fn(),
getPendingComments: vi.fn(),
getAutomatedComments: vi.fn(),
getMergeability: vi.fn(),
verifyWebhook,
parseWebhook,
} as unknown as SCM;
const mockRegistry: PluginRegistry = {
register: vi.fn(),
get: vi.fn(() => mockSCM) as PluginRegistry["get"],
list: vi.fn(() => []),
loadBuiltins: vi.fn(),
loadFromConfig: vi.fn(),
};
const mockConfig: OrchestratorConfig = {
configPath: "/tmp/agent-orchestrator.yaml",
port: 3000,
readyThresholdMs: 300_000,
defaults: { runtime: "tmux", agent: "claude-code", workspace: "worktree", notifiers: [] },
projects: {
"my-app": {
name: "My App",
repo: "acme/my-app",
path: "/tmp/my-app",
defaultBranch: "main",
sessionPrefix: "my-app",
scm: {
plugin: "github",
webhook: { enabled: true, path: "/api/webhooks/github", maxBodyBytes: 1024 },
},
},
},
notifiers: {},
notificationRouting: { urgent: [], action: [], warning: [], info: [] },
reactions: {},
};
const mockSessionManager = {
list: vi.fn(async () => [makeSession("s1")]),
} as unknown as SessionManager;
const mockLifecycle = {
check: vi.fn(async () => {}),
} as unknown as LifecycleManager;
vi.mock("@/lib/services", () => ({
getServices: vi.fn(async () => ({
config: mockConfig,
registry: mockRegistry,
sessionManager: mockSessionManager,
lifecycleManager: mockLifecycle,
})),
}));
import { POST as webhookPOST } from "@/app/api/webhooks/[...slug]/route";
function makeWebhookRequest(opts?: {
body?: string;
contentLength?: number;
headers?: Record<string, string>;
}): Request {
const body = opts?.body ?? JSON.stringify({ action: "synchronize", number: 42 });
const headers: Record<string, string> = {
"content-type": "application/json",
...opts?.headers,
};
if (opts?.contentLength !== undefined) {
headers["content-length"] = String(opts.contentLength);
}
return new Request("http://localhost:3000/api/webhooks/github", {
method: "POST",
headers,
body,
});
}
beforeEach(() => {
vi.clearAllMocks();
recordActivityEvent.mockClear();
verifyWebhook.mockResolvedValue({ ok: true });
parseWebhook.mockResolvedValue({
provider: "github",
kind: "pull_request",
action: "synchronize",
rawEventType: "pull_request",
repository: { owner: "acme", name: "my-app" },
prNumber: 42,
branch: "feat/x",
data: {},
});
});
// ── Tests ─────────────────────────────────────────────────────────────
describe("POST /api/webhooks/[...slug] — activity events", () => {
it("rejects unverified webhook with 401 and emits api.webhook_unverified", async () => {
verifyWebhook.mockResolvedValueOnce({ ok: false, reason: "bad signature" });
const req = makeWebhookRequest({
headers: { "x-hub-signature-256": "sha256=bogus", "x-forwarded-for": "203.0.113.7" },
});
const res = await webhookPOST(req);
expect(res.status).toBe(401);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.webhook_unverified",
level: "warn",
}),
);
const call = recordActivityEvent.mock.calls[0]![0] as {
summary: string;
data: Record<string, unknown>;
};
// Critical: signature value must NOT be in data (or anywhere)
expect(JSON.stringify(call)).not.toContain("bogus");
expect(call.data["slug"]).toBe("github");
expect(call.data["remoteAddr"]).toBe("203.0.113.7");
expect(call.data["verificationSupported"]).toBe(true);
expect(call.data["reason"]).toBe("bad signature");
});
it("keeps unsupported webhook verification as 404 while recording audit context", async () => {
vi.mocked(mockRegistry.get).mockReturnValueOnce({
...mockSCM,
verifyWebhook: undefined,
} as unknown as SCM);
const res = await webhookPOST(makeWebhookRequest());
expect(res.status).toBe(404);
expect(verifyWebhook).not.toHaveBeenCalled();
expect(parseWebhook).not.toHaveBeenCalled();
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.webhook_unverified",
level: "warn",
summary: expect.stringContaining("verification unsupported"),
}),
);
const call = recordActivityEvent.mock.calls[0]![0] as {
data: Record<string, unknown>;
};
expect(call.data["slug"]).toBe("github");
expect(call.data["verificationSupported"]).toBe(false);
expect(call.data["unsupportedVerificationCount"]).toBe(1);
expect(call.data["reason"]).toBe("verification_unsupported");
});
it("emits api.webhook_rejected when content-length exceeds maxBodyBytes (413)", async () => {
const req = makeWebhookRequest({ contentLength: 2048 }); // > 1024 max
const res = await webhookPOST(req);
expect(res.status).toBe(413);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.webhook_rejected",
level: "warn",
}),
);
const call = recordActivityEvent.mock.calls[0]![0] as { data: Record<string, unknown> };
expect(call.data["slug"]).toBe("github");
expect(call.data["contentLength"]).toBe(2048);
expect(call.data["maxBodyBytes"]).toBe(1024);
// No body content captured
expect(JSON.stringify(call)).not.toContain("synchronize");
});
it("emits api.webhook_received with counts (not body) on 202 success", async () => {
const req = makeWebhookRequest({
headers: { "x-forwarded-for": "192.0.2.1" },
});
const res = await webhookPOST(req);
expect(res.status).toBe(202);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.webhook_received",
}),
);
const call = recordActivityEvent.mock.calls[0]![0] as { data: Record<string, unknown> };
expect(call.data["slug"]).toBe("github");
expect(call.data["remoteAddr"]).toBe("192.0.2.1");
expect(call.data["matchedSessions"]).toBe(1);
expect(call.data["parseErrorCount"]).toBe(0);
expect(call.data["lifecycleErrorCount"]).toBe(0);
expect(call.data["projectIds"]).toEqual(["my-app"]);
// Critical: payload body is NOT included
expect(JSON.stringify(call)).not.toContain("synchronize");
});
it("emits api.webhook_received with elevated level when parse/lifecycle errors occurred", async () => {
parseWebhook.mockRejectedValueOnce(new Error("boom"));
// Verification passes so we get into the parse path
verifyWebhook.mockResolvedValueOnce({ ok: true });
const res = await webhookPOST(makeWebhookRequest());
expect(res.status).toBe(202);
const received = recordActivityEvent.mock.calls.find(
([e]) => (e as { kind: string }).kind === "api.webhook_received",
);
expect(received).toBeDefined();
expect((received![0] as { level: string }).level).toBe("warn");
expect((received![0] as { data: Record<string, unknown> }).data["parseErrorCount"]).toBe(1);
});
it("emits api.webhook_failed on 500 outer crash", async () => {
// Force the list() call inside POST to throw, hitting the outer catch.
(mockSessionManager.list as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
new Error("session manager exploded"),
);
const res = await webhookPOST(makeWebhookRequest());
expect(res.status).toBe(500);
expect(recordActivityEvent).toHaveBeenCalledWith(
expect.objectContaining({
source: "api",
kind: "api.webhook_failed",
level: "error",
}),
);
const call = recordActivityEvent.mock.calls[0]![0] as { data: Record<string, unknown> };
expect(call.data["slug"]).toBe("github");
expect(call.data["errorMessage"]).toContain("session manager exploded");
});
it("does not emit any event for 404 (unknown path)", async () => {
// Empty config so no candidates match
vi.mocked(mockRegistry.get).mockReturnValueOnce(undefined as unknown as SCM);
const req = new Request("http://localhost:3000/api/webhooks/unknown", {
method: "POST",
headers: { "content-type": "application/json" },
body: "{}",
});
const res = await webhookPOST(req);
expect(res.status).toBe(404);
// No webhook events for 404 — it's a config issue, not an external signal
const kinds = recordActivityEvent.mock.calls.map(([e]) => (e as { kind: string }).kind);
expect(kinds.filter((k) => k.startsWith("api.webhook_"))).toEqual([]);
});
});

24
packages/web/src/app/api/issues/route.ts
View File

@ -1,7 +1,7 @@
import { type NextRequest, NextResponse } from "next/server";
import { getServices } from "@/lib/services";
import { validateString, validateConfiguredProject } from "@/lib/validation";
import type { Tracker } from "@aoagents/ao-core";
import { recordActivityEvent, type Tracker } from "@aoagents/ao-core";
export const dynamic = "force-dynamic";
@ -95,11 +95,25 @@ export async function POST(request: NextRequest) {
project,
);
recordActivityEvent({
projectId,
source: "api",
kind: "api.issue_created",
summary: `issue created: ${issue.id}`,
data: { issueId: issue.id, addToBacklog: Boolean(body.addToBacklog) },
});
return NextResponse.json({ issue: { projectId, ...issue } }, { status: 201 });
} catch (err) {
return NextResponse.json(
{ error: err instanceof Error ? err.message : "Failed to create issue" },
{ status: 500 },
);
const reason = err instanceof Error ? err.message : "Failed to create issue";
recordActivityEvent({
projectId,
source: "api",
kind: "api.issue_create_failed",
level: "error",
summary: `issue create failed: ${reason}`,
data: { reason },
});
return NextResponse.json({ error: reason }, { status: 500 });
}
}

15
packages/web/src/app/api/orchestrators/route.ts
View File

@ -1,9 +1,12 @@
import { type NextRequest, NextResponse } from "next/server";
import { generateOrchestratorPrompt } from "@aoagents/ao-core";
import { generateOrchestratorPrompt, recordActivityEvent } from "@aoagents/ao-core";
import { getServices } from "@/lib/services";
import { validateIdentifier, validateConfiguredProject } from "@/lib/validation";
function classifySpawnError(projectId: string, error: unknown): {
function classifySpawnError(
projectId: string,
error: unknown,
): {
status: number;
payload: Record<string, unknown>;
} {
@ -61,6 +64,14 @@ export async function POST(request: NextRequest) {
? await sessionManager.relaunchOrchestrator({ projectId, systemPrompt })
: await sessionManager.spawnOrchestrator({ projectId, systemPrompt });
recordActivityEvent({
projectId,
sessionId: session.id,
source: "api",
kind: "api.orchestrator_spawn_requested",
summary: `orchestrator spawn requested for ${projectId}`,
});
return NextResponse.json(
{
orchestrator: {

95
packages/web/src/app/api/projects/[id]/route.ts
View File

@ -9,6 +9,7 @@ import {
loadConfig,
loadGlobalConfig,
loadLocalProjectConfigDetailed,
recordActivityEvent,
repairWrappedLocalProjectConfig,
unregisterProject,
writeLocalProjectConfig,
@ -23,6 +24,7 @@ import { stopStaleWindowsPtyHosts } from "@/lib/windows-pty-cleanup";
export const dynamic = "force-dynamic";
const IDENTITY_FIELDS = new Set(["projectId", "path", "repo", "defaultBranch"]);
const EDITABLE_CONFIG_FIELDS = new Set(["agent", "runtime", "tracker", "scm", "reactions"]);
function sanitizeString(value: unknown): string | undefined {
if (typeof value !== "string") return undefined;
@ -112,7 +114,10 @@ function getProjectState(projectId: string) {
};
}
function degradedPayload(projectId: string, degradedProject: NonNullable<ReturnType<typeof getProjectState>["degradedProject"]>) {
function degradedPayload(
projectId: string,
degradedProject: NonNullable<ReturnType<typeof getProjectState>["degradedProject"]>,
) {
return {
error: degradedProject.resolveError,
projectId,
@ -126,10 +131,7 @@ function degradedPayload(projectId: string, degradedProject: NonNullable<ReturnT
};
}
export async function GET(
_request: NextRequest,
context: { params: Promise<{ id: string }> },
) {
export async function GET(_request: NextRequest, context: { params: Promise<{ id: string }> }) {
try {
const { id } = await context.params;
const state = getProjectState(id);
@ -172,10 +174,7 @@ export async function GET(
}
}
export async function PATCH(
request: NextRequest,
context: { params: Promise<{ id: string }> },
) {
export async function PATCH(request: NextRequest, context: { params: Promise<{ id: string }> }) {
try {
const { id } = await context.params;
const body = (await request.json().catch(() => null)) as Record<string, unknown> | null;
@ -200,7 +199,10 @@ export async function PATCH(
}
const projectPath = state.globalEntry?.path;
if (!projectPath) {
return NextResponse.json({ error: `Project "${id}" is missing a registry path.` }, { status: 409 });
return NextResponse.json(
{ error: `Project "${id}" is missing a registry path.` },
{ status: 409 },
);
}
const localConfigResult = loadLocalProjectConfigDetailed(projectPath);
@ -211,7 +213,8 @@ export async function PATCH(
return NextResponse.json({ error: localConfigResult.error }, { status: 400 });
}
const currentConfig: LocalProjectConfig = localConfigResult.kind === "loaded" ? { ...localConfigResult.config } : {};
const currentConfig: LocalProjectConfig =
localConfigResult.kind === "loaded" ? { ...localConfigResult.config } : {};
const nextConfig: LocalProjectConfig = {
...currentConfig,
};
@ -231,8 +234,7 @@ export async function PATCH(
...(body["tracker"] as Record<string, unknown>),
} as LocalProjectConfig["tracker"])
: undefined;
nextConfig.tracker =
nextTracker;
nextConfig.tracker = nextTracker;
}
if (hasOwn("scm")) {
const nextScm =
@ -242,8 +244,7 @@ export async function PATCH(
...(body["scm"] as Record<string, unknown>),
} as LocalProjectConfig["scm"])
: undefined;
nextConfig.scm =
nextScm;
nextConfig.scm = nextScm;
}
if (hasOwn("reactions")) {
nextConfig.reactions =
@ -261,6 +262,16 @@ export async function PATCH(
invalidatePortfolioServicesCache();
revalidateProjectPaths(id);
// Record only changed *keys*, never values — config can contain tokens.
const changedKeys = Object.keys(body).filter((k) => EDITABLE_CONFIG_FIELDS.has(k));
recordActivityEvent({
projectId: id,
source: "api",
kind: "api.project_updated",
summary: `project updated: ${id}`,
data: { changedKeys },
});
return NextResponse.json({ ok: true });
} catch (error) {
return NextResponse.json(
@ -270,19 +281,14 @@ export async function PATCH(
}
}
export async function PUT(
request: NextRequest,
context: { params: Promise<{ id: string }> },
) {
export async function PUT(request: NextRequest, context: { params: Promise<{ id: string }> }) {
return PATCH(request, context);
}
export async function DELETE(
_request: NextRequest,
context: { params: Promise<{ id: string }> },
) {
export async function DELETE(_request: NextRequest, context: { params: Promise<{ id: string }> }) {
let id: string | undefined;
try {
const { id } = await context.params;
({ id } = await context.params);
const state = getProjectState(id);
if (!state.globalEntry && !state.project && !state.degradedProject) {
return NextResponse.json({ error: `Unknown project: ${id}` }, { status: 404 });
@ -299,7 +305,8 @@ export async function DELETE(
return NextResponse.json({ error: `Invalid project ID: ${id}` }, { status: 400 });
}
const workspacePluginName = state.project?.workspace ?? state.config.defaults.workspace ?? "worktree";
const workspacePluginName =
state.project?.workspace ?? state.config.defaults.workspace ?? "worktree";
const { registry, sessionManager } = await getServices();
await stopProjectSessions(id, sessionManager);
await stopStaleWindowsPtyHosts(projectDir);
@ -310,23 +317,34 @@ export async function DELETE(
invalidatePortfolioServicesCache();
revalidateProjectPaths(id);
recordActivityEvent({
projectId: id,
source: "api",
kind: "api.project_removed",
summary: `project removed: ${id}`,
data: { removedStorageDir: hadStorageDir },
});
return NextResponse.json({
ok: true,
projectId: id,
removedStorageDir: hadStorageDir,
});
} catch (error) {
return NextResponse.json(
{ error: error instanceof Error ? error.message : "Failed to delete project" },
{ status: 500 },
);
const reason = error instanceof Error ? error.message : "Failed to delete project";
recordActivityEvent({
projectId: id,
source: "api",
kind: "api.project_remove_failed",
level: "error",
summary: `project remove failed: ${reason}`,
data: { reason },
});
return NextResponse.json({ error: reason }, { status: 500 });
}
}
export async function POST(
_request: NextRequest,
context: { params: Promise<{ id: string }> },
) {
export async function POST(_request: NextRequest, context: { params: Promise<{ id: string }> }) {
try {
const { id } = await context.params;
const state = getProjectState(id);
@ -337,7 +355,9 @@ export async function POST(
return NextResponse.json({ error: "Project does not need repair." }, { status: 400 });
}
const isWrappedConfigError = state.degradedProject.resolveError.includes("wrapped projects: format");
const isWrappedConfigError = state.degradedProject.resolveError.includes(
"wrapped projects: format",
);
if (!isWrappedConfigError) {
return NextResponse.json(
{ error: "Automatic repair is not available for this degraded config." },
@ -349,6 +369,13 @@ export async function POST(
invalidatePortfolioServicesCache();
revalidateProjectPaths(id);
recordActivityEvent({
projectId: id,
source: "api",
kind: "api.project_repaired",
summary: `project repaired: ${id}`,
});
return NextResponse.json({ ok: true, repaired: true, projectId: id });
} catch (error) {
return NextResponse.json(

20
packages/web/src/app/api/projects/reload/route.ts
View File

@ -1,5 +1,10 @@
import { NextResponse } from "next/server";
import { ConfigNotFoundError, getGlobalConfigPath, loadConfig } from "@aoagents/ao-core";
import {
ConfigNotFoundError,
getGlobalConfigPath,
loadConfig,
recordActivityEvent,
} from "@aoagents/ao-core";
import { invalidatePortfolioServicesCache } from "@/lib/services";
export const dynamic = "force-dynamic";
@ -25,10 +30,19 @@ export async function POST() {
invalidatePortfolioServicesCache();
const config = loadReloadConfig();
const projectCount = Object.keys(config.projects).length;
const degradedCount = Object.keys(config.degradedProjects).length;
recordActivityEvent({
source: "api",
kind: "api.config_reloaded",
summary: `config reloaded: ${projectCount} projects, ${degradedCount} degraded`,
data: { projectCount, degradedCount },
});
return NextResponse.json({
reloaded: true,
projectCount: Object.keys(config.projects).length,
degradedCount: Object.keys(config.degradedProjects).length,
projectCount,
degradedCount,
});
} catch (error) {
return NextResponse.json(

23
packages/web/src/app/api/projects/route.ts
View File

@ -8,6 +8,7 @@ import {
getGlobalConfigPath,
loadConfig,
migrateToGlobalConfig,
recordActivityEvent,
registerProjectInGlobalConfig,
} from "@aoagents/ao-core";
import { revalidatePath } from "next/cache";
@ -108,6 +109,12 @@ export async function POST(request: NextRequest) {
);
invalidatePortfolioServicesCache();
revalidatePortfolioPaths(registeredProjectId);
recordActivityEvent({
projectId: registeredProjectId,
source: "api",
kind: "api.project_added",
summary: `project added: ${registeredProjectId}`,
});
return NextResponse.json({ ok: true, projectId: registeredProjectId }, { status: 201 });
} catch (err) {
const message = err instanceof Error ? err.message : "Failed to add project";
@ -124,6 +131,14 @@ export async function POST(request: NextRequest) {
if (pathAlreadyRegistered) {
const existingProjectId = pathAlreadyRegistered[1];
const suggestedProjectId = generateExternalId(resolvedPath);
recordActivityEvent({
projectId,
source: "api",
kind: "api.project_add_rejected",
level: "warn",
summary: `project add rejected: path already registered`,
data: { reason: "path_already_registered", existingProjectId, statusCode: 409 },
});
return NextResponse.json(
{
error: message,
@ -138,6 +153,14 @@ export async function POST(request: NextRequest) {
if (idAlreadyRegistered) {
const existingProjectId = idAlreadyRegistered[1];
const suggestedProjectId = generateExternalId(resolvedPath);
recordActivityEvent({
projectId,
source: "api",
kind: "api.project_add_rejected",
level: "warn",
summary: `project add rejected: id already registered`,
data: { reason: "id_already_registered", existingProjectId, statusCode: 409 },
});
return NextResponse.json(
{
error: message,

48
packages/web/src/app/api/prs/[id]/merge/route.ts
View File

@ -1,4 +1,5 @@
import { type NextRequest } from "next/server";
import { recordActivityEvent, type OrchestratorConfig } from "@aoagents/ao-core";
import { getServices, getSCM } from "@/lib/services";
import { getCorrelationId, jsonWithCorrelation, recordApiObservation } from "@/lib/observability";
@ -11,15 +12,21 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
return jsonWithCorrelation({ error: "Invalid PR number" }, { status: 400 }, correlationId);
}
const prNumber = Number(id);
let configForObservation: OrchestratorConfig | undefined;
let projectId: string | undefined;
let sessionId: string | undefined;
try {
const { config, registry, sessionManager } = await getServices();
configForObservation = config;
const sessions = await sessionManager.list();
const session = sessions.find((s) => s.pr?.number === prNumber);
if (!session?.pr) {
return jsonWithCorrelation({ error: "PR not found" }, { status: 404 }, correlationId);
}
projectId = session.projectId;
sessionId = session.id;
const project = config.projects[session.projectId];
const scm = getSCM(registry, project);
@ -34,6 +41,15 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
// Validate PR is in a mergeable state
const state = await scm.getPRState(session.pr);
if (state !== "open") {
recordActivityEvent({
projectId: session.projectId,
sessionId: session.id,
source: "api",
kind: "api.pr_merge_rejected",
level: "warn",
summary: `PR ${prNumber} merge rejected: state is ${state}`,
data: { prNumber, prState: state, statusCode: 409 },
});
return jsonWithCorrelation(
{ error: `PR is ${state}, not open` },
{ status: 409 },
@ -43,6 +59,15 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
const mergeability = await scm.getMergeability(session.pr);
if (!mergeability.mergeable) {
recordActivityEvent({
projectId: session.projectId,
sessionId: session.id,
source: "api",
kind: "api.pr_merge_rejected",
level: "warn",
summary: `PR ${prNumber} merge rejected: not mergeable`,
data: { prNumber, blockers: mergeability.blockers, statusCode: 422 },
});
return jsonWithCorrelation(
{ error: "PR is not mergeable", blockers: mergeability.blockers },
{ status: 422 },
@ -63,13 +88,22 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
sessionId: session.id,
data: { prNumber },
});
recordActivityEvent({
projectId: session.projectId,
sessionId: session.id,
source: "api",
kind: "api.pr_merge_requested",
summary: `PR ${prNumber} merge requested`,
data: { prNumber, method: "squash" },
});
return jsonWithCorrelation(
{ ok: true, prNumber, method: "squash" },
{ status: 200 },
correlationId,
);
} catch (err) {
const { config } = await getServices().catch(() => ({ config: undefined }));
const config =
configForObservation ?? (await getServices().catch(() => ({ config: undefined }))).config;
if (config) {
recordApiObservation({
config,
@ -79,10 +113,22 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
startedAt,
outcome: "failure",
statusCode: 500,
projectId,
sessionId,
reason: err instanceof Error ? err.message : "Failed to merge PR",
data: { prNumber },
});
}
const reason = err instanceof Error ? err.message : "Failed to merge PR";
recordActivityEvent({
projectId,
sessionId,
source: "api",
kind: "api.pr_merge_failed",
level: "error",
summary: `PR ${prNumber} merge failed: ${reason}`,
data: { prNumber, reason },
});
return jsonWithCorrelation(
{ error: err instanceof Error ? err.message : "Failed to merge PR" },
{ status: 500 },

18
packages/web/src/app/api/sessions/[id]/kill/route.ts
View File

@ -1,7 +1,7 @@
import { type NextRequest } from "next/server";
import { validateIdentifier } from "@/lib/validation";
import { getServices } from "@/lib/services";
import { SessionNotFoundError } from "@aoagents/ao-core";
import { SessionNotFoundError, recordActivityEvent } from "@aoagents/ao-core";
import {
getCorrelationId,
jsonWithCorrelation,
@ -34,6 +34,13 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
projectId,
sessionId: id,
});
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_kill_requested",
summary: `session kill requested: ${id}`,
});
return jsonWithCorrelation({ ok: true, sessionId: id }, { status: 200 }, correlationId);
} catch (err) {
if (err instanceof SessionNotFoundError) {
@ -56,6 +63,15 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
});
}
const msg = err instanceof Error ? err.message : "Failed to kill session";
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_kill_failed",
level: "error",
summary: `session kill failed: ${msg}`,
data: { reason: msg },
});
return jsonWithCorrelation({ error: msg }, { status: 500 }, correlationId);
}
}

19
packages/web/src/app/api/sessions/[id]/message/route.ts
View File

@ -1,7 +1,7 @@
import { type NextRequest } from "next/server";
import { getServices } from "@/lib/services";
import { stripControlChars, validateIdentifier, validateString } from "@/lib/validation";
import { SessionNotFoundError } from "@aoagents/ao-core";
import { SessionNotFoundError, recordActivityEvent } from "@aoagents/ao-core";
import {
getCorrelationId,
jsonWithCorrelation,
@ -79,6 +79,14 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
sessionId: id,
data: { messageLength: message.length },
});
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_message_sent",
summary: `message sent to session ${id}`,
data: { messageLength: message.length },
});
return jsonWithCorrelation({ success: true }, { status: 200 }, correlationId);
} catch (err) {
const errorMsg = err instanceof Error ? err.message : String(err);
@ -98,6 +106,15 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
if (err instanceof SessionNotFoundError) {
return jsonWithCorrelation({ error: err.message }, { status: 404 }, correlationId);
}
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_message_failed",
level: "error",
summary: `session message failed: ${errorMsg}`,
data: { messageLength: message.length, reason: errorMsg },
});
console.error("Failed to send message:", errorMsg);
return jsonWithCorrelation(
{ error: `Failed to send message: ${errorMsg}` },

27
packages/web/src/app/api/sessions/[id]/remap/route.ts
View File

@ -1,7 +1,7 @@
import { type NextRequest } from "next/server";
import { validateIdentifier } from "@/lib/validation";
import { getServices } from "@/lib/services";
import { SessionNotFoundError } from "@aoagents/ao-core";
import { SessionNotFoundError, recordActivityEvent } from "@aoagents/ao-core";
import {
getCorrelationId,
jsonWithCorrelation,
@ -33,6 +33,13 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
projectId,
sessionId: id,
});
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_remap_requested",
summary: `session remap requested: ${id}`,
});
return jsonWithCorrelation(
{ ok: true, sessionId: id, opencodeSessionId },
{ status: 200 },
@ -74,6 +81,15 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
reason: msg,
});
}
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_remap_failed",
level: "warn",
summary: `session remap failed: ${msg}`,
data: { reason: msg, statusCode: 422 },
});
return jsonWithCorrelation({ error: msg }, { status: 422 }, correlationId);
}
if (config) {
@ -90,6 +106,15 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
reason: msg,
});
}
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_remap_failed",
level: "error",
summary: `session remap failed: ${msg}`,
data: { reason: msg, statusCode: 500 },
});
return jsonWithCorrelation({ error: msg }, { status: 500 }, correlationId);
}
}

59
packages/web/src/app/api/sessions/[id]/restore/route.ts
View File

@ -6,6 +6,8 @@ import {
SessionNotRestorableError,
WorkspaceMissingError,
SessionNotFoundError,
recordActivityEvent,
type OrchestratorConfig,
} from "@aoagents/ao-core";
import {
getCorrelationId,
@ -24,9 +26,13 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
return jsonWithCorrelation({ error: idErr }, { status: 400 }, correlationId);
}
let configForAttribution: OrchestratorConfig | undefined;
let projectIdForAttribution: string | undefined;
try {
const { config, sessionManager } = await getServices();
const projectId = resolveProjectIdForSessionId(config, id);
configForAttribution = config;
projectIdForAttribution = resolveProjectIdForSessionId(config, id);
const restored = await sessionManager.restore(id);
recordApiObservation({
@ -37,9 +43,16 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
startedAt,
outcome: "success",
statusCode: 200,
projectId: restored.projectId ?? projectId,
projectId: restored.projectId ?? projectIdForAttribution,
sessionId: id,
});
recordActivityEvent({
projectId: restored.projectId ?? projectIdForAttribution,
sessionId: id,
source: "api",
kind: "api.session_restore_requested",
summary: `session restore requested: ${id}`,
});
return jsonWithCorrelation(
{
@ -54,29 +67,61 @@ export async function POST(_request: NextRequest, { params }: { params: Promise<
if (err instanceof SessionNotFoundError) {
return jsonWithCorrelation({ error: err.message }, { status: 404 }, correlationId);
}
if (!configForAttribution) {
const serviceContext = await getServices().catch(() => undefined);
configForAttribution = serviceContext?.config;
projectIdForAttribution = configForAttribution
? resolveProjectIdForSessionId(configForAttribution, id)
: undefined;
}
if (err instanceof SessionNotRestorableError) {
recordActivityEvent({
projectId: projectIdForAttribution,
sessionId: id,
source: "api",
kind: "api.session_restore_failed",
level: "warn",
summary: `session restore failed: ${err.message}`,
data: { reason: err.message, statusCode: 409 },
});
return jsonWithCorrelation({ error: err.message }, { status: 409 }, correlationId);
}
if (err instanceof WorkspaceMissingError) {
recordActivityEvent({
projectId: projectIdForAttribution,
sessionId: id,
source: "api",
kind: "api.session_restore_failed",
level: "warn",
summary: `session restore failed: ${err.message}`,
data: { reason: err.message, statusCode: 422 },
});
return jsonWithCorrelation({ error: err.message }, { status: 422 }, correlationId);
}
const { config } = await getServices().catch(() => ({ config: undefined }));
const projectId = config ? resolveProjectIdForSessionId(config, id) : undefined;
if (config) {
if (configForAttribution) {
recordApiObservation({
config,
config: configForAttribution,
method: "POST",
path: "/api/sessions/[id]/restore",
correlationId,
startedAt,
outcome: "failure",
statusCode: 500,
projectId,
projectId: projectIdForAttribution,
sessionId: id,
reason: err instanceof Error ? err.message : "Failed to restore session",
});
}
const msg = err instanceof Error ? err.message : "Failed to restore session";
recordActivityEvent({
projectId: projectIdForAttribution,
sessionId: id,
source: "api",
kind: "api.session_restore_failed",
level: "error",
summary: `session restore failed: ${msg}`,
data: { reason: msg, statusCode: 500 },
});
return jsonWithCorrelation({ error: msg }, { status: 500 }, correlationId);
}
}

19
packages/web/src/app/api/sessions/[id]/send/route.ts
View File

@ -1,7 +1,7 @@
import { type NextRequest } from "next/server";
import { validateIdentifier, validateString, stripControlChars } from "@/lib/validation";
import { getServices } from "@/lib/services";
import { SessionNotFoundError } from "@aoagents/ao-core";
import { SessionNotFoundError, recordActivityEvent } from "@aoagents/ao-core";
import {
getCorrelationId,
jsonWithCorrelation,
@ -55,6 +55,14 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
sessionId: id,
data: { messageLength: message.length },
});
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_message_sent",
summary: `message sent to session ${id}`,
data: { messageLength: message.length },
});
return jsonWithCorrelation(
{ ok: true, sessionId: id, message },
{ status: 200 },
@ -82,6 +90,15 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
});
}
const msg = err instanceof Error ? err.message : "Failed to send message";
recordActivityEvent({
projectId,
sessionId: id,
source: "api",
kind: "api.session_message_failed",
level: "error",
summary: `session message failed: ${msg}`,
data: { messageLength: message.length, reason: msg },
});
return jsonWithCorrelation({ error: msg }, { status: 500 }, correlationId);
}
}

10
packages/web/src/app/api/setup-labels/route.ts
View File

@ -1,4 +1,5 @@
import { NextResponse } from "next/server";
import { recordActivityEvent } from "@aoagents/ao-core";
import { getServices } from "@/lib/services";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
@ -42,6 +43,15 @@ export async function POST() {
}
}
const created = results.filter((r) => r.status === "created").length;
const exists = results.filter((r) => r.status === "exists").length;
recordActivityEvent({
source: "api",
kind: "api.labels_setup",
summary: `labels setup complete: ${created} created, ${exists} exists`,
data: { created, exists, total: results.length },
});
return NextResponse.json({ results });
} catch (err) {
return NextResponse.json(

20
packages/web/src/app/api/spawn/route.ts
View File

@ -1,4 +1,5 @@
import { type NextRequest } from "next/server";
import { recordActivityEvent } from "@aoagents/ao-core";
import { validateIdentifier, validateString, validateConfiguredProject } from "@/lib/validation";
import { getServices } from "@/lib/services";
import { sessionToDashboard } from "@/lib/serialize";
@ -50,6 +51,14 @@ export async function POST(request: NextRequest) {
reason: projectErr,
data: { issueId: body.issueId },
});
recordActivityEvent({
projectId,
source: "api",
kind: "api.session_spawn_rejected",
level: "warn",
summary: `session spawn rejected: ${projectErr}`,
data: { reason: "project_not_configured" },
});
return jsonWithCorrelation({ error: projectErr }, { status: 404 }, correlationId);
}
@ -75,6 +84,17 @@ export async function POST(request: NextRequest) {
sessionId: session.id,
data: { issueId: session.issueId },
});
recordActivityEvent({
projectId: session.projectId,
sessionId: session.id,
source: "api",
kind: "api.session_spawn_requested",
summary: `session spawn requested for ${session.projectId}`,
data: {
issueId: session.issueId ?? undefined,
hasPrompt: Boolean(prompt),
},
});
return jsonWithCorrelation(
{ session: sessionToDashboard(session) },

32
packages/web/src/app/api/verify/route.ts
View File

@ -1,7 +1,7 @@
import { type NextRequest, NextResponse } from "next/server";
import { getVerifyIssues, getServices } from "@/lib/services";
import { validateConfiguredProject } from "@/lib/validation";
import type { Tracker } from "@aoagents/ao-core";
import { recordActivityEvent, type Tracker } from "@aoagents/ao-core";
export const dynamic = "force-dynamic";
@ -25,6 +25,9 @@ export async function GET() {
* Body: { issueId: string, projectId: string, action: "verify" | "fail", comment?: string }
*/
export async function POST(req: NextRequest) {
let issueId: string | undefined;
let projectId: string | undefined;
let action: "verify" | "fail" | undefined;
try {
const body = (await req.json().catch(() => null)) as
| {
@ -37,12 +40,13 @@ export async function POST(req: NextRequest) {
if (!body) {
return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
}
const { issueId, projectId, action, comment } = body as {
let comment: string | undefined;
({ issueId, projectId, action, comment } = body as {
issueId: string;
projectId: string;
action: "verify" | "fail";
comment?: string;
};
});
if (!issueId || !projectId || !action) {
return NextResponse.json(
@ -96,11 +100,25 @@ export async function POST(req: NextRequest) {
);
}
recordActivityEvent({
projectId,
source: "api",
kind: "api.issue_verified",
summary: `issue ${issueId} ${action}`,
data: { issueId, action },
});
return NextResponse.json({ ok: true });
} catch (err) {
return NextResponse.json(
{ error: err instanceof Error ? err.message : "Failed to update issue" },
{ status: 500 },
);
const reason = err instanceof Error ? err.message : "Failed to update issue";
recordActivityEvent({
projectId,
source: "api",
kind: "api.issue_verify_failed",
level: "error",
summary: `issue verify failed: ${reason}`,
data: { issueId, action, reason },
});
return NextResponse.json({ error: reason }, { status: 500 });
}
}

108
packages/web/src/app/api/webhooks/[...slug]/route.ts
View File

@ -1,4 +1,5 @@
import { NextResponse } from "next/server";
import { recordActivityEvent } from "@aoagents/ao-core";
import { getServices } from "@/lib/services";
import {
buildWebhookRequest,
@ -9,11 +10,35 @@ import {
export const dynamic = "force-dynamic";
const WEBHOOK_PATH_PREFIX = "/api/webhooks/";
function deriveSlug(pathname: string): string {
return pathname.startsWith(WEBHOOK_PATH_PREFIX)
? pathname.slice(WEBHOOK_PATH_PREFIX.length)
: pathname;
}
function deriveRemoteAddr(request: Request): string | undefined {
// Next.js does not expose the socket peer address on Request. The standard
// proxy headers are the only signal — first hop in x-forwarded-for is the
// original client. Sanitizer in recordActivityEvent does not redact IPs;
// they are intentionally retained for security audit (per issue #1656).
const xff = request.headers.get("x-forwarded-for");
if (xff) {
const first = xff.split(",")[0]?.trim();
if (first) return first;
}
return request.headers.get("x-real-ip") ?? undefined;
}
export async function POST(request: Request): Promise<Response> {
const pathname = new URL(request.url).pathname;
const slug = deriveSlug(pathname);
const remoteAddr = deriveRemoteAddr(request);
try {
const services = await getServices();
const path = new URL(request.url).pathname;
const candidates = findWebhookProjects(services.config, services.registry, path);
const candidates = findWebhookProjects(services.config, services.registry, pathname);
if (candidates.length === 0) {
return NextResponse.json(
@ -36,6 +61,19 @@ export async function POST(request: Request): Promise<Response> {
Number.isFinite(contentLength) &&
contentLength > maxBodyBytes
) {
recordActivityEvent({
source: "api",
kind: "api.webhook_rejected",
level: "warn",
summary: `webhook payload exceeded ${maxBodyBytes} bytes for ${slug}`,
data: {
slug,
remoteAddr,
contentLength,
maxBodyBytes,
reason: "payload_too_large",
},
});
return NextResponse.json(
{ error: "Webhook payload exceeds configured maxBodyBytes" },
{ status: 413 },
@ -50,12 +88,21 @@ export async function POST(request: Request): Promise<Response> {
const sessionIds = new Set<string>();
const projectIds = new Set<string>();
let verified = false;
let verificationSupported = false;
let unsupportedVerificationCount = 0;
const errors: string[] = [];
const parseErrors: string[] = [];
const lifecycleErrors: string[] = [];
for (const candidate of candidates) {
const verification = await candidate.scm.verifyWebhook?.(webhookRequest, candidate.project);
if (!candidate.scm.verifyWebhook) {
unsupportedVerificationCount += 1;
errors.push("Webhook verification not supported by SCM plugin");
continue;
}
verificationSupported = true;
const verification = await candidate.scm.verifyWebhook(webhookRequest, candidate.project);
if (!verification?.ok) {
if (verification?.reason) errors.push(verification.reason);
continue;
@ -93,12 +140,52 @@ export async function POST(request: Request): Promise<Response> {
}
if (!verified) {
const unsupportedOnly = !verificationSupported && unsupportedVerificationCount > 0;
recordActivityEvent({
source: "api",
kind: "api.webhook_unverified",
level: "warn",
summary: unsupportedOnly
? `webhook verification unsupported for ${slug}`
: `webhook signature verification failed for ${slug}`,
data: {
slug,
remoteAddr,
candidateCount: candidates.length,
verificationSupported,
unsupportedVerificationCount,
reason: unsupportedOnly
? "verification_unsupported"
: (errors[0] ?? "verification_failed"),
},
});
return NextResponse.json(
{ error: errors[0] ?? "Webhook verification failed", ok: false },
{ status: 401 },
{
error: unsupportedOnly
? "No SCM webhook configured for this path"
: (errors[0] ?? "Webhook verification failed"),
ok: false,
verificationSupported,
},
{ status: unsupportedOnly ? 404 : 401 },
);
}
recordActivityEvent({
source: "api",
kind: "api.webhook_received",
level: parseErrors.length > 0 || lifecycleErrors.length > 0 ? "warn" : "info",
summary: `webhook accepted for ${slug}: ${sessionIds.size} session(s) matched`,
data: {
slug,
remoteAddr,
projectIds: [...projectIds],
matchedSessions: sessionIds.size,
parseErrorCount: parseErrors.length,
lifecycleErrorCount: lifecycleErrors.length,
},
});
return NextResponse.json(
{
ok: true,
@ -111,6 +198,17 @@ export async function POST(request: Request): Promise<Response> {
{ status: 202 },
);
} catch (err) {
recordActivityEvent({
source: "api",
kind: "api.webhook_failed",
level: "error",
summary: `webhook pipeline crashed for ${slug}`,
data: {
slug,
remoteAddr,
errorMessage: err instanceof Error ? err.message : String(err),
},
});
return NextResponse.json(
{ error: err instanceof Error ? err.message : "Failed to process SCM webhook" },
{ status: 500 },

2
packages/web/src/lib/scm-webhooks.ts
View File

@ -40,7 +40,7 @@ export function findWebhookProjects(
const webhookPath = getProjectWebhookPath(project);
if (!webhookPath || webhookPath !== pathname) return [];
const scm = registry.get<SCM>("scm", project.scm.plugin);
if (!scm?.parseWebhook || !scm.verifyWebhook) return [];
if (!scm?.parseWebhook) return [];
return [{ projectId, project, scm }];
});
}