diff --git a/.changeset/cli-activity-events.md b/.changeset/cli-activity-events.md new file mode 100644 index 000000000..7528033c4 --- /dev/null +++ b/.changeset/cli-activity-events.md @@ -0,0 +1,6 @@ +--- +"@aoagents/ao-core": minor +"@aoagents/ao-cli": minor +--- + +Wire CLI activity events into `ao start`, `ao stop`, `ao spawn`, `ao update`, `ao setup`, `ao migrate-storage`, and shared CLI helpers. `ao events list --source cli` now answers RCA questions like "did AO start cleanly?", "was AO killed or did it crash?", and "did `ao spawn`/`ao stop` fail and why?". Adds `"cli"` to the `ActivityEventSource` union and 30+ event-emit sites covering startup, graceful and forced shutdown, restore, project resolution, config recovery, and migration paths. diff --git a/.changeset/linear-transient-retry.md b/.changeset/linear-transient-retry.md new file mode 100644 index 000000000..cb1ab2764 --- /dev/null +++ b/.changeset/linear-transient-retry.md @@ -0,0 +1,5 @@ +--- +"@aoagents/ao-plugin-tracker-linear": patch +--- + +Retry transient Linear API HTTP failures in the direct transport to reduce flakes from brief 5xx/429 responses. diff --git a/packages/cli/__tests__/commands/migrate-storage.test.ts b/packages/cli/__tests__/commands/migrate-storage.test.ts new file mode 100644 index 000000000..cf071ea30 --- /dev/null +++ b/packages/cli/__tests__/commands/migrate-storage.test.ts @@ -0,0 +1,114 @@ +/** + * Tests for migrate-storage activity-event instrumentation (issue #1654). + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { Command } from "commander"; +import * as AoCore from "@aoagents/ao-core"; + +const { mockMigrateStorage, mockRollbackStorage } = vi.hoisted(() => ({ + mockMigrateStorage: vi.fn(), + mockRollbackStorage: vi.fn(), +})); + +vi.mock("@aoagents/ao-core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + migrateStorage: (...args: unknown[]) => mockMigrateStorage(...args), + rollbackStorage: (...args: unknown[]) => mockRollbackStorage(...args), + recordActivityEvent: vi.fn(), + }; +}); + +import { registerMigrateStorage } from "../../src/commands/migrate-storage.js"; + +const recordedEvents = (): Array> => + vi.mocked(AoCore.recordActivityEvent).mock.calls.map((c) => c[0] as Record); + +describe("ao migrate-storage — activity events", () => { + let program: Command; + let exitSpy: ReturnType; + let consoleErrSpy: ReturnType; + let consoleLogSpy: ReturnType; + + beforeEach(() => { + vi.mocked(AoCore.recordActivityEvent).mockClear(); + mockMigrateStorage.mockReset(); + mockRollbackStorage.mockReset(); + + program = new Command(); + program.exitOverride(); + registerMigrateStorage(program); + + exitSpy = vi.spyOn(process, "exit").mockImplementation((() => undefined) as never); + consoleErrSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + consoleLogSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + }); + + afterEach(() => { + exitSpy.mockRestore(); + consoleErrSpy.mockRestore(); + consoleLogSpy.mockRestore(); + }); + + it("emits cli.migration_invoked before migration work starts", async () => { + mockMigrateStorage.mockImplementation(async () => { + expect(recordedEvents()).toContainEqual( + expect.objectContaining({ + kind: "cli.migration_invoked", + source: "cli", + level: "info", + data: expect.objectContaining({ + rollback: false, + dryRun: true, + force: true, + }), + }), + ); + return { projects: 1 }; + }); + + await program.parseAsync(["node", "ao", "migrate-storage", "--dry-run", "--force"]); + + expect(mockMigrateStorage).toHaveBeenCalledOnce(); + }); + + it("emits cli.migration_failed when migrateStorage throws", async () => { + mockMigrateStorage.mockRejectedValue(new Error("disk full")); + + await program.parseAsync(["node", "ao", "migrate-storage"]); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.migration_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + rollback: false, + errorMessage: "disk full", + }), + }), + ); + }); + + it("emits cli.migration_failed when rollbackStorage throws", async () => { + mockRollbackStorage.mockRejectedValue(new Error("rollback boom")); + + await program.parseAsync(["node", "ao", "migrate-storage", "--rollback"]); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.migration_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + rollback: true, + errorMessage: "rollback boom", + }), + }), + ); + }); +}); diff --git a/packages/cli/__tests__/commands/project.test.ts b/packages/cli/__tests__/commands/project.test.ts index 264b64f03..1a322a070 100644 --- a/packages/cli/__tests__/commands/project.test.ts +++ b/packages/cli/__tests__/commands/project.test.ts @@ -23,6 +23,7 @@ vi.mock("@aoagents/ao-core", () => ({ isPortfolioEnabled: () => true, getPortfolio: mockGetPortfolio, getPortfolioSessionCounts: mockGetPortfolioSessionCounts, + recordActivityEvent: vi.fn(), registerProject: mockRegisterProject, unregisterProject: mockUnregisterProject, loadPreferences: mockLoadPreferences, diff --git a/packages/cli/__tests__/commands/setup.test.ts b/packages/cli/__tests__/commands/setup.test.ts index 7914a00ac..3b043e88e 100644 --- a/packages/cli/__tests__/commands/setup.test.ts +++ b/packages/cli/__tests__/commands/setup.test.ts @@ -31,6 +31,7 @@ vi.mock("@aoagents/ao-core", () => ({ findConfigFile: (...args: unknown[]) => mockFindConfigFile(...args), isCanonicalGlobalConfigPath: (configPath: string | undefined) => configPath === join(homedir(), ".agent-orchestrator", "config.yaml"), + recordActivityEvent: vi.fn(), })); vi.mock("node:fs", async (importOriginal) => { @@ -52,6 +53,7 @@ vi.mock("../../src/lib/openclaw-probe.js", () => ({ HOOKS_PATH: "/hooks/agent", })); +import { recordActivityEvent } from "@aoagents/ao-core"; import { registerSetup } from "../../src/commands/setup.js"; // --------------------------------------------------------------------------- @@ -90,6 +92,9 @@ function createProgram(): Command { return program; } +const recordedEvents = (): Array> => + vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0] as Record); + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -99,6 +104,7 @@ describe("setup openclaw command", () => { beforeEach(() => { vi.restoreAllMocks(); + vi.mocked(recordActivityEvent).mockClear(); mockFindConfigFile.mockReturnValue("/tmp/agent-orchestrator.yaml"); mockReadFileSync.mockReturnValue(MINIMAL_CONFIG); mockWriteFileSync.mockImplementation(() => {}); @@ -484,6 +490,44 @@ projects: expect(mockWriteFileSync.mock.calls[0][0]).toBe("/custom/path/agent-orchestrator.yaml"); }); + + it("emits setup_degraded instead of setup_failed when OpenClaw JSON write falls back to manual instructions", async () => { + const openclawConfigPath = join(homedir(), ".openclaw", "openclaw.json"); + mockWriteFileSync.mockImplementation((path: string) => { + if (path === openclawConfigPath) { + throw new Error("permission denied"); + } + }); + const program = createProgram(); + + await program.parseAsync([ + "node", + "test", + "setup", + "openclaw", + "--url", + "http://127.0.0.1:18789/hooks/agent", + "--token", + "tok", + "--non-interactive", + ]); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.setup_degraded", + source: "cli", + level: "warn", + data: expect.objectContaining({ reason: "openclaw_json_write_failed" }), + }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ + kind: "cli.setup_failed", + data: expect.objectContaining({ reason: "openclaw_json_write_failed" }), + }), + ); + }); }); describe("error handling", () => { diff --git a/packages/cli/__tests__/commands/spawn.test.ts b/packages/cli/__tests__/commands/spawn.test.ts index 60ffeb6c7..84e076ce9 100644 --- a/packages/cli/__tests__/commands/spawn.test.ts +++ b/packages/cli/__tests__/commands/spawn.test.ts @@ -2,25 +2,28 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { type Session, type SessionManager, getProjectBaseDir } from "@aoagents/ao-core"; +import { + recordActivityEvent, + type Session, + type SessionManager, + getProjectBaseDir, +} from "@aoagents/ao-core"; -const { mockExec, mockConfigRef, mockSessionManager, mockGetRunning } = vi.hoisted( - () => ({ - mockExec: vi.fn(), - mockConfigRef: { current: null as Record | null }, - mockSessionManager: { - list: vi.fn(), - kill: vi.fn(), - cleanup: vi.fn(), - get: vi.fn(), - spawn: vi.fn(), - spawnOrchestrator: vi.fn(), - send: vi.fn(), - claimPR: vi.fn(), - }, - mockGetRunning: vi.fn(), - }), -); +const { mockExec, mockConfigRef, mockSessionManager, mockGetRunning } = vi.hoisted(() => ({ + mockExec: vi.fn(), + mockConfigRef: { current: null as Record | null }, + mockSessionManager: { + list: vi.fn(), + kill: vi.fn(), + cleanup: vi.fn(), + get: vi.fn(), + spawn: vi.fn(), + spawnOrchestrator: vi.fn(), + send: vi.fn(), + claimPR: vi.fn(), + }, + mockGetRunning: vi.fn(), +})); vi.mock("../../src/lib/shell.js", () => ({ tmux: vi.fn(), @@ -49,6 +52,7 @@ vi.mock("@aoagents/ao-core", async (importOriginal) => { return { ...actual, loadConfig: () => mockConfigRef.current, + recordActivityEvent: vi.fn(), }; }); @@ -86,6 +90,9 @@ import { registerSpawn, registerBatchSpawn } from "../../src/commands/spawn.js"; let program: Command; let consoleSpy: ReturnType; +const recordedEvents = (): Array> => + vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0] as Record); + beforeEach(() => { tmpDir = mkdtempSync(join(tmpdir(), "ao-spawn-test-")); configPath = join(tmpDir, "agent-orchestrator.yaml"); @@ -134,6 +141,7 @@ beforeEach(() => { mockSessionManager.claimPR.mockReset(); mockExec.mockReset(); mockGetRunning.mockReset(); + vi.mocked(recordActivityEvent).mockClear(); mockRegistryGet.mockReset().mockReturnValue(null); mockGetRunning.mockResolvedValue({ pid: 1234, port: 3000, startedAt: "", projects: ["my-app"] }); }); @@ -534,9 +542,7 @@ describe("spawn command", () => { it("reports error when spawn fails", async () => { mockSessionManager.spawn.mockRejectedValue(new Error("worktree creation failed")); - await expect(program.parseAsync(["node", "test", "spawn"])).rejects.toThrow( - "process.exit(1)", - ); + await expect(program.parseAsync(["node", "test", "spawn"])).rejects.toThrow("process.exit(1)"); }); it("claims a PR for the spawned session when --claim-pr is provided", async () => { @@ -628,14 +634,7 @@ describe("spawn command", () => { takenOverFrom: ["app-9"], }); - await program.parseAsync([ - "node", - "test", - "spawn", - "--claim-pr", - "123", - "--assign-on-github", - ]); + await program.parseAsync(["node", "test", "spawn", "--claim-pr", "123", "--assign-on-github"]); expect(mockSessionManager.claimPR).toHaveBeenCalledWith("app-1", "123", { assignOnGithub: true, @@ -736,6 +735,19 @@ describe("spawn pre-flight checks", () => { .mock.calls.map((c) => String(c[0])) .join("\n"); expect(errors).toContain("tmux is not installed"); + expect(recordedEvents()).toContainEqual( + expect.objectContaining({ + kind: "cli.spawn_failed", + source: "cli", + projectId: "my-app", + level: "error", + data: expect.objectContaining({ + issueId: null, + agent: null, + errorMessage: "tmux is not installed. Install it: brew install tmux", + }), + }), + ); expect(mockSessionManager.spawn).not.toHaveBeenCalled(); }); @@ -860,7 +872,9 @@ describe("batch-spawn command", () => { return cmd; } - function makeFakeSession(overrides: Partial & Pick): Session { + function makeFakeSession( + overrides: Partial & Pick, + ): Session { return { status: "spawning", activity: null, diff --git a/packages/cli/__tests__/commands/start-stop-instrumentation.test.ts b/packages/cli/__tests__/commands/start-stop-instrumentation.test.ts new file mode 100644 index 000000000..ed0dfd3e1 --- /dev/null +++ b/packages/cli/__tests__/commands/start-stop-instrumentation.test.ts @@ -0,0 +1,571 @@ +/** + * Tests for start.ts activity-event instrumentation (issue #1654). + * + * Covers MUST emits in registerStop and the start action that don't + * require running the full startup pipeline: + * - cli.stop_invoked (start of ao stop action) + * - cli.stop_failed (outer catch of ao stop action) + * - cli.stop_session_failed (per-session kill failure during ao stop) + * - cli.last_stop_write_failed (last-stop persistence failure during ao stop) + * - cli.daemon_killed (SIGTERM sent to parent ao start) + * - cli.start_invoked (true start action entry) + * - cli.start_failed (outer) (outer catch of ao start action) + * - cli.restore_session_failed (per-session restore failure) + * + * cli.start_failed (orchestrator_setup / supervisor_start) is exercised by + * the existing start.test.ts infrastructure; this file + * focuses on emits that are reachable with a small deps surface. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { Command } from "commander"; + +// --------------------------------------------------------------------------- +// Hoisted mocks +// --------------------------------------------------------------------------- + +const { + mockSessionManager, + mockGetRunning, + mockUnregister, + mockWriteLastStop, + mockReadLastStop, + mockClearLastStop, + mockAcquireStartupLock, + mockIsAlreadyRunning, + mockFindPidByPort, + mockKillProcessTree, + mockIsWindows, +} = vi.hoisted(() => ({ + mockSessionManager: { + list: vi.fn(), + kill: vi.fn(), + restore: vi.fn(), + ensureOrchestrator: vi.fn(), + get: vi.fn(), + }, + mockGetRunning: vi.fn(), + mockUnregister: vi.fn(), + mockWriteLastStop: vi.fn(), + mockReadLastStop: vi.fn(), + mockClearLastStop: vi.fn(), + mockAcquireStartupLock: vi.fn(), + mockIsAlreadyRunning: vi.fn(), + mockFindPidByPort: vi.fn(), + mockKillProcessTree: vi.fn(), + mockIsWindows: vi.fn(), +})); + +vi.mock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + findPidByPort: (...args: unknown[]) => mockFindPidByPort(...args), + isWindows: (...args: unknown[]) => mockIsWindows(...args), + killProcessTree: (...args: unknown[]) => mockKillProcessTree(...args), + recordActivityEvent: vi.fn(), + }; +}); + +vi.mock("../../src/lib/create-session-manager.js", () => ({ + getSessionManager: async () => mockSessionManager, + getPluginRegistry: async () => ({ register: vi.fn(), get: () => null }), +})); + +vi.mock("../../src/lib/running-state.js", () => ({ + acquireStartupLock: (...args: unknown[]) => mockAcquireStartupLock(...args), + isAlreadyRunning: (...args: unknown[]) => mockIsAlreadyRunning(...args), + getRunning: (...args: unknown[]) => mockGetRunning(...args), + register: vi.fn(), + unregister: (...args: unknown[]) => mockUnregister(...args), + removeProjectFromRunning: vi.fn(), + addProjectToRunning: vi.fn(), + writeLastStop: (...args: unknown[]) => mockWriteLastStop(...args), + readLastStop: (...args: unknown[]) => mockReadLastStop(...args), + clearLastStop: (...args: unknown[]) => mockClearLastStop(...args), +})); + +vi.mock("../../src/lib/lifecycle-service.js", () => ({ + stopAllLifecycleWorkers: vi.fn(), + listLifecycleWorkers: () => [], +})); + +vi.mock("../../src/lib/project-supervisor.js", () => ({ + startProjectSupervisor: vi.fn(), + stopProjectSupervisor: vi.fn(), +})); + +vi.mock("../../src/lib/preflight.js", () => ({ + preflight: { checkPort: vi.fn(), checkBuilt: vi.fn() }, +})); + +vi.mock("../../src/lib/web-dir.js", () => ({ + findWebDir: vi.fn().mockReturnValue("/fake/web"), + buildDashboardEnv: vi.fn().mockResolvedValue({}), + waitForPortAndOpen: vi.fn(), + openUrl: vi.fn(), + isPortAvailable: vi.fn().mockResolvedValue(true), + findFreePort: vi.fn().mockResolvedValue(3000), + MAX_PORT_SCAN: 100, +})); + +vi.mock("../../src/lib/dashboard-rebuild.js", () => ({ + clearStaleCacheIfNeeded: vi.fn(), + rebuildDashboardProductionArtifacts: vi.fn(), +})); + +vi.mock("../../src/lib/shell.js", () => ({ + exec: vi.fn().mockResolvedValue({ stdout: "" }), + execSilent: vi.fn().mockResolvedValue({ stdout: "" }), + git: vi.fn(), +})); + +vi.mock("../../src/lib/bun-tmp-janitor.js", () => ({ + startBunTmpJanitor: vi.fn(), +})); + +vi.mock("../../src/lib/daemon.js", () => ({ + attachToDaemon: vi.fn(), + killExistingDaemon: vi.fn(), +})); + +vi.mock("../../src/lib/caller-context.js", () => ({ + isHumanCaller: () => false, + getCallerType: () => "automation", +})); + +vi.mock("../../src/lib/detect-env.js", () => ({ + detectEnvironment: vi.fn().mockResolvedValue({}), +})); + +vi.mock("../../src/lib/detect-agent.js", () => ({ + detectAgentRuntime: vi.fn(), + detectAvailableAgents: vi.fn().mockResolvedValue([]), +})); + +vi.mock("../../src/lib/git-utils.js", () => ({ + detectDefaultBranch: vi.fn().mockResolvedValue("main"), +})); + +vi.mock("../../src/lib/prompts.js", () => ({ + promptConfirm: vi.fn().mockResolvedValue(false), + promptSelect: vi.fn(), + promptText: vi.fn(), +})); + +vi.mock("../../src/lib/install-helpers.js", () => ({ + canPromptForInstall: vi.fn().mockReturnValue(false), + genericInstallHints: vi.fn().mockReturnValue([]), + askYesNo: vi.fn().mockResolvedValue(false), + runInteractiveCommand: vi.fn(), + tryInstallWithAttempts: vi.fn(), +})); + +vi.mock("../../src/lib/startup-preflight.js", () => ({ + ensureGit: vi.fn(), + runtimePreflight: vi.fn(), +})); + +vi.mock("../../src/lib/shutdown.js", () => ({ + installShutdownHandlers: vi.fn(), +})); + +vi.mock("../../src/lib/resolve-project.js", () => ({ + resolveOrCreateProject: vi.fn(), +})); + +vi.mock("../../src/lib/project-resolution.js", () => ({ + findProjectForDirectory: vi.fn(), +})); + +vi.mock("../../src/lib/repo-utils.js", () => ({ + extractOwnerRepo: vi.fn(), + isValidRepoString: vi.fn(), +})); + +vi.mock("../../src/lib/project-detection.js", () => ({ + detectProjectType: vi.fn(), + generateRulesFromTemplates: vi.fn(), + formatProjectTypeForDisplay: vi.fn(), +})); + +vi.mock("../../src/lib/cli-errors.js", () => ({ + formatCommandError: vi.fn((err: unknown) => String(err)), +})); + +import { recordActivityEvent } from "@aoagents/ao-core"; +import { registerStart, registerStop } from "../../src/commands/start.js"; + +const recordedEvents = (): Array> => + vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0] as Record); + +function buildProgram(): Command { + const program = new Command(); + program.exitOverride(); + registerStart(program); + registerStop(program); + return program; +} + +describe("ao stop — activity events", () => { + let exitSpy: ReturnType; + + beforeEach(() => { + vi.mocked(recordActivityEvent).mockClear(); + mockGetRunning.mockReset(); + mockSessionManager.list.mockReset(); + mockSessionManager.kill.mockReset(); + mockUnregister.mockReset(); + mockWriteLastStop.mockReset(); + mockGetRunning.mockResolvedValue(null); + mockFindPidByPort.mockReset(); + mockFindPidByPort.mockResolvedValue(null); + mockKillProcessTree.mockReset(); + mockKillProcessTree.mockResolvedValue(undefined); + mockIsWindows.mockReset(); + mockIsWindows.mockReturnValue(false); + + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + exitSpy = vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`process.exit(${code})`); + }); + }); + + afterEach(() => { + exitSpy.mockRestore(); + vi.restoreAllMocks(); + }); + + it("emits cli.stop_invoked at the start of the action", async () => { + const projectArg = "https://token@example.com/org/repo.git"; + // Force a fast failure so the action exits quickly after emitting stop_invoked. + mockGetRunning.mockResolvedValue(null); + // Make loadConfig throw so we hit the outer catch + vi.doMock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + findPidByPort: (...args: unknown[]) => mockFindPidByPort(...args), + isWindows: (...args: unknown[]) => mockIsWindows(...args), + killProcessTree: (...args: unknown[]) => mockKillProcessTree(...args), + recordActivityEvent: vi.mocked(recordActivityEvent), + loadConfig: () => { + throw new Error("config not found"); + }, + }; + }); + + vi.resetModules(); + const reloaded = await import("../../src/commands/start.js"); + const program = new Command(); + program.exitOverride(); + reloaded.registerStop(program); + + await expect(program.parseAsync(["node", "ao", "stop", projectArg])).rejects.toThrow(); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.stop_invoked", + source: "cli", + summary: "ao stop invoked", + data: expect.objectContaining({ + projectArg, + }), + }), + ); + + vi.doUnmock("@aoagents/ao-core"); + }); + + it("emits cli.stop_failed when loadConfig throws", async () => { + vi.doMock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + findPidByPort: (...args: unknown[]) => mockFindPidByPort(...args), + isWindows: (...args: unknown[]) => mockIsWindows(...args), + killProcessTree: (...args: unknown[]) => mockKillProcessTree(...args), + recordActivityEvent: vi.mocked(recordActivityEvent), + loadConfig: () => { + throw new Error("config blew up"); + }, + }; + }); + + vi.resetModules(); + const reloaded = await import("../../src/commands/start.js"); + const program = new Command(); + program.exitOverride(); + reloaded.registerStop(program); + + await expect(program.parseAsync(["node", "ao", "stop"])).rejects.toThrow(); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.stop_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ errorMessage: "config blew up" }), + }), + ); + + vi.doUnmock("@aoagents/ao-core"); + }); + + it("emits cli.daemon_killed when SIGTERM is sent to a running daemon", async () => { + mockGetRunning.mockResolvedValue({ + pid: 99999, + configPath: "/tmp/x.yaml", + port: 3000, + startedAt: new Date().toISOString(), + projects: ["my-app"], + }); + mockSessionManager.list.mockResolvedValue([]); + + vi.doMock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + findPidByPort: (...args: unknown[]) => mockFindPidByPort(...args), + isWindows: (...args: unknown[]) => mockIsWindows(...args), + killProcessTree: (...args: unknown[]) => mockKillProcessTree(...args), + recordActivityEvent: vi.mocked(recordActivityEvent), + loadConfig: () => ({ + configPath: "/tmp/x.yaml", + port: 3000, + projects: { "my-app": { name: "my-app", path: "/tmp/my-app" } }, + defaults: {}, + }), + }; + }); + + vi.resetModules(); + const reloaded = await import("../../src/commands/start.js"); + const program = new Command(); + program.exitOverride(); + reloaded.registerStop(program); + + try { + await program.parseAsync(["node", "ao", "stop"]); + } catch { + // ao stop may exit; we just want the events + } + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.daemon_killed", + source: "cli", + data: expect.objectContaining({ pid: 99999 }), + }), + ); + expect(mockKillProcessTree).toHaveBeenCalledWith(99999, "SIGTERM"); + + vi.doUnmock("@aoagents/ao-core"); + }); + + it("emits cli.stop_session_failed when sm.kill throws during ao stop", async () => { + mockGetRunning.mockResolvedValue({ + pid: 99999, + configPath: "/tmp/x.yaml", + port: 3000, + startedAt: new Date().toISOString(), + projects: ["my-app"], + }); + mockSessionManager.list.mockResolvedValue([ + { + id: "sess-1", + projectId: "my-app", + status: "working", + }, + ]); + mockSessionManager.kill.mockRejectedValue(new Error("kill timeout")); + vi.spyOn(process, "kill").mockImplementation(() => true); + + vi.doMock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + findPidByPort: (...args: unknown[]) => mockFindPidByPort(...args), + isWindows: (...args: unknown[]) => mockIsWindows(...args), + killProcessTree: (...args: unknown[]) => mockKillProcessTree(...args), + recordActivityEvent: vi.mocked(recordActivityEvent), + loadConfig: () => ({ + configPath: "/tmp/x.yaml", + port: 3000, + projects: { "my-app": { name: "my-app", path: "/tmp/my-app" } }, + defaults: {}, + }), + // isTerminalSession returns false so the session is treated as active + isTerminalSession: () => false, + }; + }); + + vi.resetModules(); + const reloaded = await import("../../src/commands/start.js"); + const program = new Command(); + program.exitOverride(); + reloaded.registerStop(program); + + try { + await program.parseAsync(["node", "ao", "stop"]); + } catch { + // ignored + } + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.stop_session_failed", + source: "cli", + level: "warn", + sessionId: "sess-1", + data: expect.objectContaining({ errorMessage: "kill timeout" }), + }), + ); + + vi.doUnmock("@aoagents/ao-core"); + }); + + it("emits cli.last_stop_write_failed when ao stop cannot persist restore state", async () => { + mockGetRunning.mockResolvedValue(null); + mockSessionManager.list.mockResolvedValue([ + { + id: "sess-1", + projectId: "my-app", + status: "working", + }, + ]); + mockSessionManager.kill.mockResolvedValue({ cleaned: true, alreadyTerminated: false }); + mockWriteLastStop.mockRejectedValue(new Error("last-stop lock busy")); + + vi.doMock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + findPidByPort: (...args: unknown[]) => mockFindPidByPort(...args), + isWindows: (...args: unknown[]) => mockIsWindows(...args), + killProcessTree: (...args: unknown[]) => mockKillProcessTree(...args), + recordActivityEvent: vi.mocked(recordActivityEvent), + loadConfig: () => ({ + configPath: "/tmp/x.yaml", + port: 3000, + projects: { "my-app": { name: "my-app", path: "/tmp/my-app" } }, + defaults: {}, + }), + // isTerminalSession returns false so the session is treated as active + isTerminalSession: () => false, + }; + }); + + vi.resetModules(); + const reloaded = await import("../../src/commands/start.js"); + const program = new Command(); + program.exitOverride(); + reloaded.registerStop(program); + + await program.parseAsync(["node", "ao", "stop", "my-app"]); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.last_stop_write_failed", + source: "cli", + level: "error", + projectId: "my-app", + data: expect.objectContaining({ + targetSessionCount: 1, + totalKilled: 1, + errorMessage: "last-stop lock busy", + }), + }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ + kind: "cli.last_stop_written", + }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ + kind: "cli.stop_failed", + }), + ); + const logs = vi.mocked(console.log).mock.calls.map((c) => String(c[0])); + expect(logs.some((line) => line.includes("Could not list sessions"))).toBe(false); + expect(logs.some((line) => line.includes("Could not write last-stop state"))).toBe(true); + + vi.doUnmock("@aoagents/ao-core"); + }); +}); + +describe("ao start — activity events (failure paths)", () => { + let exitSpy: ReturnType; + + beforeEach(() => { + vi.mocked(recordActivityEvent).mockClear(); + mockAcquireStartupLock.mockReset(); + mockIsAlreadyRunning.mockReset(); + mockAcquireStartupLock.mockResolvedValue(() => undefined); + mockIsAlreadyRunning.mockResolvedValue(null); + + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + exitSpy = vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`process.exit(${code})`); + }); + }); + + afterEach(() => { + exitSpy.mockRestore(); + vi.restoreAllMocks(); + }); + + it("emits cli.start_failed with reason 'outer' when resolveOrCreateProject throws", async () => { + const projectArg = "https://token@example.com/org/repo.git"; + const resolveProjectMod = await import("../../src/lib/resolve-project.js"); + vi.mocked(resolveProjectMod.resolveOrCreateProject).mockRejectedValue( + new Error("project resolution exploded"), + ); + + const program = buildProgram(); + + try { + await program.parseAsync(["node", "ao", "start", projectArg]); + } catch { + // process.exit(1) throws in the spy + } + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.start_invoked", + source: "cli", + level: "info", + summary: "ao start invoked", + data: expect.objectContaining({ + projectArg, + }), + }), + ); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.start_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + reason: "outer", + errorMessage: "project resolution exploded", + }), + }), + ); + }); +}); diff --git a/packages/cli/__tests__/commands/start.test.ts b/packages/cli/__tests__/commands/start.test.ts index 60a247c8e..b821d3637 100644 --- a/packages/cli/__tests__/commands/start.test.ts +++ b/packages/cli/__tests__/commands/start.test.ts @@ -19,7 +19,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { parse as parseYaml } from "yaml"; import { EventEmitter } from "node:events"; -import type { SessionManager } from "@aoagents/ao-core"; +import { recordActivityEvent, type SessionManager } from "@aoagents/ao-core"; // --------------------------------------------------------------------------- // Hoisted mocks @@ -153,6 +153,7 @@ vi.mock("@aoagents/ao-core", async (importOriginal) => { sweepDaemonChildren: mockSweepDaemonChildren, scanAoOrphans: mockScanAoOrphans, reapAoOrphans: mockReapAoOrphans, + recordActivityEvent: vi.fn(), }; }); @@ -336,6 +337,7 @@ beforeEach(async () => { program.exitOverride(); registerStart(program); registerStop(program); + vi.mocked(recordActivityEvent).mockClear(); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); @@ -505,6 +507,9 @@ function makeProject(overrides: Record = {}): Record> => + vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0] as Record); + /** Mock process.cwd() to return a specific directory (avoids process.chdir in workers). */ function mockCwd(dir: string): void { cwdSpy = vi.spyOn(process, "cwd").mockReturnValue(dir); @@ -1571,6 +1576,19 @@ describe("start command — orchestrator session strategy display", () => { await expect(program.parseAsync(["node", "test", "start"])).rejects.toThrow("process.exit(1)"); expect(releaseStartupLock).toHaveBeenCalledTimes(1); + const startFailedEvents = recordedEvents().filter((e) => e.kind === "cli.start_failed"); + expect(startFailedEvents).toHaveLength(1); + expect(startFailedEvents[0]).toEqual( + expect.objectContaining({ + projectId: "my-app", + source: "cli", + level: "error", + data: expect.objectContaining({ + reason: "orchestrator_setup", + errorMessage: "Spawn failed", + }), + }), + ); }); it("fails and cleans up dashboard when sm.restore throws on a killed orchestrator", async () => { @@ -1651,6 +1669,56 @@ describe("start command — orchestrator session strategy display", () => { expect(written.stoppedAt).toBe("2026-04-28T10:00:00.000Z"); }); + it("attributes other-project restore failures to the owning project", async () => { + mockReadLastStop.mockResolvedValue({ + stoppedAt: "2026-04-28T10:00:00.000Z", + projectId: "my-app", + sessionIds: ["app-1"], + otherProjects: [{ projectId: "other-app", sessionIds: ["other-1"] }], + }); + + mockConfigRef.current = makeConfig({ + "my-app": makeProject(), + "other-app": makeProject({ name: "Other App", sessionPrefix: "other" }), + }); + const { findWebDir } = await import("../../src/lib/web-dir.js"); + vi.mocked(findWebDir).mockReturnValue(tmpDir); + writeFileSync(join(tmpDir, "package.json"), "{}"); + + const fakeDashboard = { on: vi.fn(), kill: vi.fn(), emit: vi.fn() }; + mockSpawn.mockReturnValue(fakeDashboard); + + mockSessionManager.restore.mockImplementation((id: string) => { + if (id === "other-1") return Promise.reject(new Error("workspace gone")); + return Promise.resolve(undefined); + }); + + await program.parseAsync(["node", "test", "start", "my-app", "--no-orchestrator"]); + + const restoreFailedEvents = recordedEvents().filter( + (e) => e.kind === "cli.restore_session_failed", + ); + expect(restoreFailedEvents).toHaveLength(1); + expect(restoreFailedEvents[0]).toEqual( + expect.objectContaining({ + projectId: "other-app", + sessionId: "other-1", + source: "cli", + level: "warn", + data: expect.objectContaining({ errorMessage: "workspace gone" }), + }), + ); + + const written = mockWriteLastStop.mock.calls[0][0]; + expect(written).toEqual( + expect.objectContaining({ + projectId: "my-app", + sessionIds: [], + otherProjects: [{ projectId: "other-app", sessionIds: ["other-1"] }], + }), + ); + }); + it("clears last-stop record when every session restored successfully", async () => { mockReadLastStop.mockResolvedValue({ stoppedAt: "2026-04-28T10:00:00.000Z", diff --git a/packages/cli/__tests__/commands/update-instrumentation.test.ts b/packages/cli/__tests__/commands/update-instrumentation.test.ts new file mode 100644 index 000000000..30fb22156 --- /dev/null +++ b/packages/cli/__tests__/commands/update-instrumentation.test.ts @@ -0,0 +1,233 @@ +/** + * Tests for update.ts activity-event instrumentation (issue #1654). + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { Command } from "commander"; +import { EventEmitter } from "node:events"; +import * as AoCore from "@aoagents/ao-core"; + +const { mockRunRepoScript } = vi.hoisted(() => ({ + mockRunRepoScript: vi.fn(), +})); + +vi.mock("../../src/lib/script-runner.js", () => ({ + runRepoScript: (...args: unknown[]) => mockRunRepoScript(...args), +})); + +const { + mockDetectInstallMethod, + mockCheckForUpdate, + mockInvalidateCache, + mockGetCurrentVersion, + mockGetUpdateCommand, +} = vi.hoisted(() => ({ + mockDetectInstallMethod: vi.fn(() => "git" as const), + mockCheckForUpdate: vi.fn(), + mockInvalidateCache: vi.fn(), + mockGetCurrentVersion: vi.fn(() => "0.2.2"), + mockGetUpdateCommand: vi.fn(() => "npm install -g @aoagents/ao@latest"), +})); + +vi.mock("../../src/lib/update-check.js", () => ({ + detectInstallMethod: () => mockDetectInstallMethod(), + checkForUpdate: (...args: unknown[]) => mockCheckForUpdate(...args), + invalidateCache: () => mockInvalidateCache(), + getCurrentVersion: () => mockGetCurrentVersion(), + getUpdateCommand: (...args: unknown[]) => mockGetUpdateCommand(...args), + readCachedUpdateInfo: vi.fn(() => undefined), + resolveUpdateChannel: vi.fn(() => "stable"), +})); + +const { mockPromptConfirm } = vi.hoisted(() => ({ + mockPromptConfirm: vi.fn(async () => true), +})); + +vi.mock("../../src/lib/prompts.js", () => ({ + promptConfirm: (...args: unknown[]) => mockPromptConfirm(...args), +})); + +vi.mock("../../src/lib/running-state.js", () => ({ + getRunning: vi.fn().mockResolvedValue(null), +})); + +vi.mock("../../src/lib/create-session-manager.js", () => ({ + getSessionManager: vi.fn(), +})); + +const { mockSpawn } = vi.hoisted(() => ({ mockSpawn: vi.fn() })); + +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { ...actual, spawn: (...args: unknown[]) => mockSpawn(...args) }; +}); + +vi.mock("@aoagents/ao-core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getGlobalConfigPath: () => "/tmp/__ao_update_instrumentation_no_global_config__", + recordActivityEvent: vi.fn(), + }; +}); + +import { registerUpdate } from "../../src/commands/update.js"; + +const recordedEvents = (): Array> => + vi.mocked(AoCore.recordActivityEvent).mock.calls.map((c) => c[0] as Record); + +function createMockChild(exitCode: number | null, signal?: NodeJS.Signals): EventEmitter { + const child = new EventEmitter(); + setTimeout(() => child.emit("exit", exitCode, signal ?? null), 0); + return child; +} + +describe("ao update — activity events", () => { + let program: Command; + let origStdinTTY: boolean | undefined; + let origStdoutTTY: boolean | undefined; + + beforeEach(() => { + vi.mocked(AoCore.recordActivityEvent).mockClear(); + program = new Command(); + program.exitOverride(); + registerUpdate(program); + mockRunRepoScript.mockReset(); + mockDetectInstallMethod.mockReturnValue("git"); + mockCheckForUpdate.mockReset(); + mockInvalidateCache.mockReset(); + mockPromptConfirm.mockReset(); + mockPromptConfirm.mockResolvedValue(true); + mockSpawn.mockReset(); + origStdinTTY = process.stdin.isTTY; + origStdoutTTY = process.stdout.isTTY; + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`process.exit(${code})`); + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + Object.defineProperty(process.stdin, "isTTY", { value: origStdinTTY, configurable: true }); + Object.defineProperty(process.stdout, "isTTY", { value: origStdoutTTY, configurable: true }); + }); + + it("emits cli.update_failed when ao-update.sh exits non-zero (git path)", async () => { + mockDetectInstallMethod.mockReturnValue("git"); + mockRunRepoScript.mockResolvedValue(2); + + // process.exit is mocked to throw — the first `process.exit(2)` triggers + // the throw, which is then re-caught and emits a second event before the + // final exit. The instrumentation event for the non-zero exit is what + // matters; whichever final exit code propagates is incidental. + await expect(program.parseAsync(["node", "ao", "update"])).rejects.toThrow(/process\.exit/); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.update_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ method: "git", exitCode: 2 }), + }), + ); + }); + + it("emits cli.update_failed when ao-update.sh script is missing (git path)", async () => { + mockDetectInstallMethod.mockReturnValue("git"); + mockRunRepoScript.mockRejectedValue(new Error("Script not found: ao-update.sh")); + + await expect(program.parseAsync(["node", "ao", "update"])).rejects.toThrow("process.exit(1)"); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.update_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ method: "git", reason: "script_missing" }), + }), + ); + }); + + it("emits cli.update_failed when npm install exits non-zero (npm path)", async () => { + mockDetectInstallMethod.mockReturnValue("npm-global"); + mockCheckForUpdate.mockResolvedValue({ + currentVersion: "0.2.2", + latestVersion: "0.3.0", + isOutdated: true, + installMethod: "npm-global" as const, + recommendedCommand: "npm install -g @aoagents/ao@latest", + checkedAt: new Date().toISOString(), + }); + Object.defineProperty(process.stdin, "isTTY", { value: true, configurable: true }); + Object.defineProperty(process.stdout, "isTTY", { value: true, configurable: true }); + mockSpawn.mockReturnValue(createMockChild(1)); + + await expect(program.parseAsync(["node", "ao", "update"])).rejects.toThrow("process.exit(1)"); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.update_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ method: "npm-global", exitCode: 1 }), + }), + ); + }); + + it("emits cli.update_failed when npm registry lookup returns no version", async () => { + mockDetectInstallMethod.mockReturnValue("npm-global"); + mockCheckForUpdate.mockResolvedValue({ + currentVersion: "0.2.2", + latestVersion: null, + isOutdated: false, + installMethod: "npm-global" as const, + recommendedCommand: "npm install -g @aoagents/ao@latest", + checkedAt: null, + }); + + await expect(program.parseAsync(["node", "ao", "update"])).rejects.toThrow( + "process.exit(1)", + ); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.update_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + method: "npm-global", + reason: "registry_unreachable", + }), + }), + ); + }); + + it("emits cli.update_failed when npm registry lookup throws", async () => { + mockDetectInstallMethod.mockReturnValue("npm-global"); + mockCheckForUpdate.mockRejectedValue(new Error("registry timeout")); + + await expect(program.parseAsync(["node", "ao", "update"])).rejects.toThrow( + "process.exit(1)", + ); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.update_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + method: "npm-global", + reason: "registry_lookup_threw", + errorMessage: "registry timeout", + }), + }), + ); + }); +}); diff --git a/packages/cli/__tests__/lib/resolve-project-instrumentation.test.ts b/packages/cli/__tests__/lib/resolve-project-instrumentation.test.ts new file mode 100644 index 000000000..149eb9135 --- /dev/null +++ b/packages/cli/__tests__/lib/resolve-project-instrumentation.test.ts @@ -0,0 +1,159 @@ +/** + * Tests for resolve-project.ts activity-event instrumentation (issue #1654). + * + * Covers MUST emits: + * - cli.project_resolve_failed (clone failure inside fromUrl) + * - cli.config_recovery_failed (registerFlatConfig returns null) + */ + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import * as AoCore from "@aoagents/ao-core"; + +vi.mock("@aoagents/ao-core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + recordActivityEvent: vi.fn(), + // resolveCloneTarget points to a tmp dir; isRepoAlreadyCloned forces the + // clone path so cloneRepo is invoked (and can throw). + resolveCloneTarget: () => "/tmp/__ao_test_clone_target__", + isRepoAlreadyCloned: () => false, + loadConfig: () => ({ + configPath: "/tmp/__ao_test_global_config__", + projects: {}, + }), + }; +}); + +vi.mock("../../src/lib/startup-preflight.js", () => ({ + ensureGit: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("../../src/lib/web-dir.js", () => ({ + findFreePort: vi.fn().mockResolvedValue(3000), +})); + +vi.mock("../../src/lib/shell.js", () => ({ + git: vi.fn().mockResolvedValue({ stdout: "" }), +})); + +import { resolveOrCreateProject } from "../../src/lib/resolve-project.js"; + +const recordedEvents = (): Array> => + vi.mocked(AoCore.recordActivityEvent).mock.calls.map((c) => c[0] as Record); + +describe("resolve-project — activity events", () => { + beforeEach(() => { + vi.mocked(AoCore.recordActivityEvent).mockClear(); + }); + + it("emits cli.project_resolve_failed when cloneRepo throws (URL into running daemon)", async () => { + const cloneRepo = vi.fn( + async (_parsed: AoCore.ParsedRepoUrl, _target: string, _cwd: string) => { + throw new Error("network down"); + }, + ); + + await expect( + resolveOrCreateProject( + "https://github.com/foo/bar", + { + addProjectToConfig: vi.fn(), + autoCreateConfig: vi.fn(), + resolveProject: vi.fn(), + resolveProjectByRepo: vi.fn(), + registerFlatConfig: vi.fn().mockResolvedValue(null), + cloneRepo, + }, + // targetGlobalRegistry: true → exercises fromUrlIntoGlobal + { targetGlobalRegistry: true }, + ), + ).rejects.toThrow(/Failed to clone/); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.project_resolve_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + ownerRepo: "foo/bar", + errorMessage: "network down", + }), + }), + ); + }); + + it("emits cli.config_recovery_failed when registerFlatConfig returns null", async () => { + // Trigger fromCwdOrId via undefined arg; if loadConfig() throws something + // other than ConfigNotFoundError, the recovery path runs and asks + // registerFlatConfig to fix it. + // + // Here we force the recovery path to fail by stubbing the deps so that: + // 1. autoCreateConfig is not invoked (fromCwdOrId only calls it when + // loadConfig throws ConfigNotFoundError — we trigger a different + // error so the registerFlatConfig branch runs). + // 2. registerFlatConfig returns null (recovery fails). + // + // We can't easily make the real loadConfig() throw a non-ConfigNotFoundError + // synchronously, so instead we patch findConfigFile via the mocked module + // surface. The simplest, robust approach: simulate the public call shape + // and assert that whenever `registerFlatConfig` returns null, the event + // fires at the call site. To do that we drive the function with a + // controlled cwd that lacks a parseable config but has a config file + // present — replicated by stubbing findConfigFile in @aoagents/ao-core. + + // Reach into the same module mock by re-mocking findConfigFile + loadConfig. + vi.doMock("@aoagents/ao-core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + recordActivityEvent: vi.mocked(AoCore.recordActivityEvent), + // findConfigFile returns a path so the recovery branch runs. + findConfigFile: () => "/tmp/__ao_test_flat_config__", + // loadConfig throws a generic Error (not ConfigNotFoundError) so the + // catch block falls into the registerFlatConfig recovery branch. + loadConfig: () => { + throw new Error("malformed config"); + }, + }; + }); + + vi.resetModules(); + const { resolveOrCreateProject: resolveOrCreateProjectReloaded } = + await import("../../src/lib/resolve-project.js"); + // Re-grab the mock so cleared calls inside the doMock factory don't get lost. + const { recordActivityEvent: reloadedRecord } = await import("@aoagents/ao-core"); + vi.mocked(reloadedRecord).mockClear(); + + await expect( + resolveOrCreateProjectReloaded( + undefined, + { + addProjectToConfig: vi.fn(), + autoCreateConfig: vi.fn(), + resolveProject: vi.fn(), + resolveProjectByRepo: vi.fn(), + registerFlatConfig: vi.fn().mockResolvedValue(null), + cloneRepo: vi.fn(), + }, + {}, + ), + ).rejects.toThrow(/malformed config/); + + const events = vi.mocked(reloadedRecord).mock.calls.map((c) => c[0] as Record); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.config_recovery_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + configPath: "/tmp/__ao_test_flat_config__", + errorMessage: "malformed config", + }), + }), + ); + + vi.doUnmock("@aoagents/ao-core"); + }); +}); diff --git a/packages/cli/__tests__/lib/shutdown.test.ts b/packages/cli/__tests__/lib/shutdown.test.ts new file mode 100644 index 000000000..6ff859cc5 --- /dev/null +++ b/packages/cli/__tests__/lib/shutdown.test.ts @@ -0,0 +1,259 @@ +/** + * Tests for shutdown.ts activity-event instrumentation (issue #1654). + * + * Each test mounts handlers via `installShutdownHandlers`, fires a signal, + * and asserts the expected `cli.*` activity events are emitted. Real + * `process.exit` is stubbed so the test process is not terminated by the + * shutdown handler. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { recordActivityEvent } from "@aoagents/ao-core"; + +const { + mockListSessions, + mockKillSession, + mockGetSessionManager, + mockUnregister, + mockWriteLastStop, + mockStopBunTmpJanitor, + mockStopProjectSupervisor, + mockStopAllLifecycleWorkers, + mockLoadConfig, + mockIsTerminalSession, +} = vi.hoisted(() => ({ + mockListSessions: vi.fn(), + mockKillSession: vi.fn(), + mockGetSessionManager: vi.fn(), + mockUnregister: vi.fn(), + mockWriteLastStop: vi.fn(), + mockStopBunTmpJanitor: vi.fn(), + mockStopProjectSupervisor: vi.fn(), + mockStopAllLifecycleWorkers: vi.fn(), + mockLoadConfig: vi.fn(), + mockIsTerminalSession: vi.fn(), +})); + +vi.mock("@aoagents/ao-core", async (importOriginal) => { + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const actual = await importOriginal(); + return { + ...actual, + loadConfig: (...args: unknown[]) => mockLoadConfig(...args), + isTerminalSession: (...args: unknown[]) => mockIsTerminalSession(...args), + recordActivityEvent: vi.fn(), + }; +}); + +vi.mock("../../src/lib/create-session-manager.js", () => ({ + getSessionManager: (...args: unknown[]) => mockGetSessionManager(...args), +})); + +vi.mock("../../src/lib/lifecycle-service.js", () => ({ + stopAllLifecycleWorkers: (...args: unknown[]) => mockStopAllLifecycleWorkers(...args), +})); + +vi.mock("../../src/lib/project-supervisor.js", () => ({ + stopProjectSupervisor: (...args: unknown[]) => mockStopProjectSupervisor(...args), +})); + +vi.mock("../../src/lib/running-state.js", () => ({ + unregister: (...args: unknown[]) => mockUnregister(...args), + writeLastStop: (...args: unknown[]) => mockWriteLastStop(...args), +})); + +vi.mock("../../src/lib/bun-tmp-janitor.js", () => ({ + stopBunTmpJanitor: (...args: unknown[]) => mockStopBunTmpJanitor(...args), +})); + +const recordedEvents = (): Array> => + vi.mocked(recordActivityEvent).mock.calls.map((c) => c[0] as Record); + +const flushAsync = async (): Promise => { + // The shutdown handler launches an async IIFE; allow it to settle. + for (let i = 0; i < 10; i++) { + await new Promise((r) => setImmediate(r)); + } +}; + +describe("shutdown handlers — activity events", () => { + let exitSpy: ReturnType; + let originalListenersSigint: NodeJS.SignalsListener[]; + let originalListenersSigterm: NodeJS.SignalsListener[]; + + beforeEach(async () => { + vi.resetModules(); + vi.mocked(recordActivityEvent).mockClear(); + mockListSessions.mockReset(); + mockKillSession.mockReset(); + mockGetSessionManager.mockReset(); + mockUnregister.mockReset(); + mockWriteLastStop.mockReset(); + mockStopBunTmpJanitor.mockReset(); + mockStopProjectSupervisor.mockReset(); + mockStopAllLifecycleWorkers.mockReset(); + mockLoadConfig.mockReset(); + mockIsTerminalSession.mockReset(); + + mockLoadConfig.mockReturnValue({ projects: {} }); + mockIsTerminalSession.mockReturnValue(false); + mockGetSessionManager.mockResolvedValue({ + list: mockListSessions, + kill: mockKillSession, + }); + mockListSessions.mockResolvedValue([]); + mockKillSession.mockResolvedValue({ cleaned: true }); + mockUnregister.mockResolvedValue(undefined); + mockWriteLastStop.mockResolvedValue(undefined); + mockStopBunTmpJanitor.mockResolvedValue(undefined); + + // Snapshot existing signal listeners so we can restore them after each + // test and avoid leaking handlers across tests in the same process. + originalListenersSigint = process.listeners("SIGINT") as NodeJS.SignalsListener[]; + originalListenersSigterm = process.listeners("SIGTERM") as NodeJS.SignalsListener[]; + + exitSpy = vi.spyOn(process, "exit").mockImplementation((() => { + // Throw a sentinel to short-circuit the async IIFE without leaving + // the test process in an unknown state. + return undefined as never; + }) as never); + }); + + afterEach(() => { + exitSpy.mockRestore(); + + // Restore signal listeners + process.removeAllListeners("SIGINT"); + process.removeAllListeners("SIGTERM"); + for (const l of originalListenersSigint) process.on("SIGINT", l); + for (const l of originalListenersSigterm) process.on("SIGTERM", l); + }); + + it("emits cli.shutdown_signal when SIGINT is received", async () => { + const { installShutdownHandlers } = await import("../../src/lib/shutdown.js"); + installShutdownHandlers({ configPath: "/tmp/cfg.yaml", projectId: "p1" }); + + process.emit("SIGINT", "SIGINT"); + await flushAsync(); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.shutdown_signal", + source: "cli", + projectId: "p1", + data: expect.objectContaining({ signal: "SIGINT", exitCode: 130 }), + }), + ); + }); + + it("emits cli.shutdown_completed after clean shutdown", async () => { + const { installShutdownHandlers } = await import("../../src/lib/shutdown.js"); + installShutdownHandlers({ configPath: "/tmp/cfg.yaml", projectId: "p1" }); + + process.emit("SIGTERM", "SIGTERM"); + await flushAsync(); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.shutdown_completed", + source: "cli", + projectId: "p1", + }), + ); + }); + + it("emits cli.shutdown_failed when shutdown body throws before completion", async () => { + const { installShutdownHandlers } = await import("../../src/lib/shutdown.js"); + mockGetSessionManager.mockRejectedValue(new Error("getSessionManager boom")); + + installShutdownHandlers({ configPath: "/tmp/cfg.yaml", projectId: "p1" }); + + process.emit("SIGTERM", "SIGTERM"); + await flushAsync(); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.shutdown_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ errorMessage: "getSessionManager boom" }), + }), + ); + // Failure path should NOT emit shutdown_completed + const completedEvents = events.filter((e) => e.kind === "cli.shutdown_completed"); + expect(completedEvents).toHaveLength(0); + }); + + it("still unregisters running state when writing last-stop state fails", async () => { + const { installShutdownHandlers } = await import("../../src/lib/shutdown.js"); + mockListSessions.mockResolvedValue([ + { + id: "s1", + projectId: "p1", + status: "working", + }, + ]); + mockWriteLastStop.mockRejectedValue(new Error("disk full")); + + installShutdownHandlers({ configPath: "/tmp/cfg.yaml", projectId: "p1" }); + + process.emit("SIGTERM", "SIGTERM"); + await flushAsync(); + + expect(mockWriteLastStop).toHaveBeenCalled(); + expect(mockUnregister).toHaveBeenCalled(); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.last_stop_write_failed", + source: "cli", + level: "error", + data: expect.objectContaining({ + targetSessionCount: 1, + otherProjectCount: 0, + totalKilled: 1, + errorMessage: "disk full", + }), + }), + ); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.shutdown_completed", + source: "cli", + projectId: "p1", + }), + ); + expect(events.filter((e) => e.kind === "cli.shutdown_failed")).toHaveLength(0); + }); + + it("emits cli.shutdown_force_exit when the 10s timer fires", async () => { + vi.useFakeTimers(); + try { + const { installShutdownHandlers } = await import("../../src/lib/shutdown.js"); + // Hang the async cleanup so the force-exit timer wins. + mockGetSessionManager.mockReturnValue(new Promise(() => {})); + + installShutdownHandlers({ configPath: "/tmp/cfg.yaml", projectId: "p1" }); + + process.emit("SIGINT", "SIGINT"); + // Advance past the 10s timeout + await vi.advanceTimersByTimeAsync(10_000); + + const events = recordedEvents(); + expect(events).toContainEqual( + expect.objectContaining({ + kind: "cli.shutdown_force_exit", + source: "cli", + level: "warn", + data: expect.objectContaining({ timeoutMs: 10_000, exitCode: 130 }), + }), + ); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/cli/src/commands/migrate-storage.ts b/packages/cli/src/commands/migrate-storage.ts index 04f450439..4e90bbd28 100644 --- a/packages/cli/src/commands/migrate-storage.ts +++ b/packages/cli/src/commands/migrate-storage.ts @@ -1,6 +1,6 @@ import type { Command } from "commander"; import chalk from "chalk"; -import { migrateStorage, rollbackStorage } from "@aoagents/ao-core"; +import { migrateStorage, recordActivityEvent, rollbackStorage } from "@aoagents/ao-core"; export function registerMigrateStorage(program: Command): void { program @@ -13,12 +13,31 @@ export function registerMigrateStorage(program: Command): void { .option("--rollback", "Reverse a previous migration (restores .migrated directories)") .action( async (opts: { dryRun?: boolean; force?: boolean; rollback?: boolean }) => { + recordActivityEvent({ + source: "cli", + kind: "cli.migration_invoked", + level: "info", + summary: `storage ${opts.rollback ? "rollback" : "migration"} invoked`, + data: { + rollback: opts.rollback === true, + dryRun: opts.dryRun === true, + force: opts.force === true, + }, + }); + try { if (opts.rollback) { await rollbackStorage({ dryRun: opts.dryRun, log: (msg) => console.log(msg), }); + recordActivityEvent({ + source: "cli", + kind: "cli.migration_completed", + level: "info", + summary: `storage rollback completed`, + data: { rollback: true, dryRun: opts.dryRun === true }, + }); } else { const result = await migrateStorage({ dryRun: opts.dryRun, @@ -31,8 +50,30 @@ export function registerMigrateStorage(program: Command): void { } else { console.log(chalk.green("\nMigration complete.")); } + recordActivityEvent({ + source: "cli", + kind: "cli.migration_completed", + level: "info", + summary: `storage migration completed (${result.projects} project(s))`, + data: { + rollback: false, + dryRun: opts.dryRun === true, + force: opts.force === true, + projects: result.projects, + }, + }); } } catch (err) { + recordActivityEvent({ + source: "cli", + kind: "cli.migration_failed", + level: "error", + summary: `storage migration failed`, + data: { + rollback: opts.rollback === true, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); console.error( chalk.red(err instanceof Error ? err.message : String(err)), ); diff --git a/packages/cli/src/commands/project.ts b/packages/cli/src/commands/project.ts index 5dfc35c21..9c0a97228 100644 --- a/packages/cli/src/commands/project.ts +++ b/packages/cli/src/commands/project.ts @@ -6,6 +6,7 @@ import { getPortfolio, getPortfolioSessionCounts, isPortfolioEnabled, + recordActivityEvent, registerProject, unregisterProject, loadPreferences, @@ -94,18 +95,48 @@ export function registerProjectCommand(program: Command): void { const existingConfigPath = candidatePaths.find((candidate) => existsSync(candidate)); if (!existingConfigPath) { + recordActivityEvent({ + source: "cli", + kind: "cli.project_register_failed", + level: "warn", + summary: `ao project add: no agent-orchestrator config found`, + data: { resolvedPath, reason: "no_config_found" }, + }); console.error(chalk.red(`No agent-orchestrator.yaml found at ${resolvedPath}`)); process.exit(1); } try { loadConfig(existingConfigPath); + recordActivityEvent({ + source: "cli", + kind: "cli.project_register_failed", + level: "warn", + summary: `ao project add: found old-format config requiring migration`, + data: { + resolvedPath, + configPath: existingConfigPath, + reason: "old_format", + }, + }); console.error( chalk.red( `Found old-format config at ${existingConfigPath}. Run \`ao start\` in that project to migrate it before using \`ao project add\`.`, ), ); } catch (error) { + recordActivityEvent({ + source: "cli", + kind: "cli.project_register_failed", + level: "error", + summary: `ao project add: config load failed`, + data: { + resolvedPath, + configPath: existingConfigPath, + reason: "load_error", + errorMessage: error instanceof Error ? error.message : String(error), + }, + }); console.error( chalk.red( `Found agent-orchestrator config at ${existingConfigPath}, but it could not be loaded: ${error instanceof Error ? error.message : String(error)}`, diff --git a/packages/cli/src/commands/setup.ts b/packages/cli/src/commands/setup.ts index fa03f465d..0ae91e5e9 100644 --- a/packages/cli/src/commands/setup.ts +++ b/packages/cli/src/commands/setup.ts @@ -14,7 +14,12 @@ import { join } from "node:path"; import chalk from "chalk"; import type { Command } from "commander"; import { parse as yamlParse, parseDocument } from "yaml"; -import { CONFIG_SCHEMA_URL, findConfigFile, isCanonicalGlobalConfigPath } from "@aoagents/ao-core"; +import { + CONFIG_SCHEMA_URL, + findConfigFile, + isCanonicalGlobalConfigPath, + recordActivityEvent, +} from "@aoagents/ao-core"; import { probeGateway, validateToken, @@ -512,6 +517,16 @@ export function registerSetup(program: Command): void { try { await runSetupAction(opts); } catch (err) { + recordActivityEvent({ + source: "cli", + kind: "cli.setup_failed", + level: "error", + summary: `ao setup openclaw failed`, + data: { + aborted: err instanceof SetupAbortedError, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); if (err instanceof SetupAbortedError) { console.error(err.message); process.exit(err.exitCode); @@ -574,6 +589,14 @@ export async function runSetupAction(opts: SetupOptions): Promise { const openclawConfigWritten = writeOpenClawJsonConfig(resolved.token); if (openclawConfigWritten && nonInteractive) { console.log(chalk.green("✓ Wrote hooks config to ~/.openclaw/openclaw.json")); + } else if (!openclawConfigWritten) { + recordActivityEvent({ + source: "cli", + kind: "cli.setup_degraded", + level: "warn", + summary: `ao setup openclaw completed without writing ~/.openclaw/openclaw.json`, + data: { reason: "openclaw_json_write_failed" }, + }); } // --- Write shell export -------------------------------------------------- diff --git a/packages/cli/src/commands/spawn.ts b/packages/cli/src/commands/spawn.ts index fed2f0585..cb2730b43 100644 --- a/packages/cli/src/commands/spawn.ts +++ b/packages/cli/src/commands/spawn.ts @@ -4,6 +4,7 @@ import type { Command } from "commander"; import { resolve } from "node:path"; import { loadConfig, + recordActivityEvent, resolveSpawnTarget, TERMINAL_STATUSES, type OrchestratorConfig, @@ -216,6 +217,20 @@ async function spawnSession( throw new Error("Prompt must be at most 4096 characters"); } + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.spawn_invoked", + level: "info", + summary: `ao spawn invoked${issueId ? ` for issue ${issueId}` : ""}`, + data: { + issueId: issueId ?? null, + agent: agent ?? null, + hasPrompt: !!sanitizedPrompt, + claimPr: claimOptions?.claimPr ?? null, + }, + }); + const session = await sm.spawn({ projectId, issueId, @@ -243,9 +258,7 @@ async function spawnSession( const issueLabel = issueId ? ` for issue #${issueId}` : ""; const claimLabel = claimedPrUrl ? ` (claimed ${claimedPrUrl})` : ""; const port = config.port ?? DEFAULT_PORT; - spinner.succeed( - `Session ${chalk.green(session.id)} spawned${issueLabel}${claimLabel}`, - ); + spinner.succeed(`Session ${chalk.green(session.id)} spawned${issueLabel}${claimLabel}`); console.log(` View: ${chalk.dim(projectSessionUrl(port, projectId, session.id))}`); // Open terminal tab if requested @@ -262,6 +275,18 @@ async function spawnSession( console.log(`SESSION=${session.id}`); } catch (err) { spinner.fail("Failed to create or initialize session"); + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.spawn_failed", + level: "error", + summary: `ao spawn failed${issueId ? ` for issue ${issueId}` : ""}`, + data: { + issueId: issueId ?? null, + agent: agent ?? null, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); throw err; } } @@ -279,7 +304,10 @@ export function registerSpawn(program: Command): void { .option("--agent ", "Override the agent plugin (e.g. codex, claude-code)") .option("--claim-pr ", "Immediately claim an existing PR for the spawned session") .option("--assign-on-github", "Assign the claimed PR to the authenticated GitHub user") - .option("--prompt ", "Initial prompt/instructions for the agent (use instead of an issue)") + .option( + "--prompt ", + "Initial prompt/instructions for the agent (use instead of an issue)", + ) .action( async ( issue: string | undefined, @@ -326,8 +354,34 @@ export function registerSpawn(program: Command): void { try { await runSpawnPreflight(config, projectId, claimOptions); await ensureAOPollingProject(projectId); + } catch (err) { + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.spawn_failed", + level: "error", + summary: `ao spawn preflight failed${issueId ? ` for issue ${issueId}` : ""}`, + data: { + issueId: issueId ?? null, + agent: opts.agent ?? null, + claimPr: claimOptions.claimPr ?? null, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); + console.error(chalk.red(`✗ ${err instanceof Error ? err.message : String(err)}`)); + process.exit(1); + } - await spawnSession(config, projectId, issueId, opts.open, opts.agent, claimOptions, opts.prompt); + try { + await spawnSession( + config, + projectId, + issueId, + opts.open, + opts.agent, + claimOptions, + opts.prompt, + ); } catch (err) { console.error(chalk.red(`✗ ${err instanceof Error ? err.message : String(err)}`)); process.exit(1); @@ -386,9 +440,7 @@ export function registerBatchSpawn(program: Command): void { console.log(banner("BATCH SESSION SPAWNER")); console.log(); for (const [pid, items] of groups) { - console.log( - ` ${chalk.bold(pid)}: ${items.map((i) => i.original).join(", ")}`, - ); + console.log(` ${chalk.bold(pid)}: ${items.map((i) => i.original).join(", ")}`); } console.log(); @@ -404,6 +456,17 @@ export function registerBatchSpawn(program: Command): void { await runSpawnPreflight(config, groupProjectId); await ensureAOPollingProject(groupProjectId); } catch (err) { + recordActivityEvent({ + projectId: groupProjectId, + source: "cli", + kind: "cli.spawn_failed", + level: "error", + summary: `batch-spawn preflight failed for group`, + data: { + batchSize: items.length, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); console.error(chalk.red(`✗ ${err instanceof Error ? err.message : String(err)}`)); process.exit(1); } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 7f657c399..0602bba8b 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -32,6 +32,7 @@ import { findPidByPort, killProcessTree, loadLocalProjectConfigDetailed, + recordActivityEvent, registerProjectInGlobalConfig, getGlobalConfigPath, type OrchestratorConfig, @@ -116,6 +117,17 @@ import { projectSessionUrl } from "../lib/routes.js"; // HELPERS // ============================================================================= +class CliFailureEventRecordedError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "CliFailureEventRecordedError"; + } +} + +function isCliFailureEventRecordedError(err: unknown): boolean { + return err instanceof CliFailureEventRecordedError; +} + function readProjectBehaviorConfig(projectPath: string): LocalProjectConfig { const localConfig = loadLocalProjectConfigDetailed(projectPath); if (localConfig.kind === "loaded") { @@ -162,6 +174,15 @@ async function registerFlatConfig(configPath: string): Promise { ...(repo ? { repo } : {}), }); + recordActivityEvent({ + projectId: registeredProjectId, + source: "cli", + kind: "cli.config_migrated", + level: "info", + summary: `flat config registered into global config`, + data: { projectPath, configPath }, + }); + console.log(chalk.green(` ✓ Registered "${registeredProjectId}"\n`)); return registeredProjectId; } @@ -940,10 +961,21 @@ async function runStartup( } } catch (err) { spinner.fail("Orchestrator setup failed"); + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.start_failed", + level: "error", + summary: `orchestrator setup failed`, + data: { + reason: "orchestrator_setup", + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); if (dashboardProcess) { dashboardProcess.kill(); } - throw new Error( + throw new CliFailureEventRecordedError( `Failed to setup orchestrator: ${err instanceof Error ? err.message : String(err)}`, { cause: err }, ); @@ -957,10 +989,21 @@ async function runStartup( spinner.succeed("Lifecycle project supervisor started"); } catch (err) { spinner.fail("Project supervisor failed to start"); + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.start_failed", + level: "error", + summary: `project supervisor failed to start`, + data: { + reason: "supervisor_start", + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); if (dashboardProcess) { dashboardProcess.kill(); } - throw new Error( + throw new CliFailureEventRecordedError( `Failed to start project supervisor: ${err instanceof Error ? err.message : String(err)}`, { cause: err }, ); @@ -974,12 +1017,21 @@ async function runStartup( if (lastStop && lastStop.sessionIds.length > 0) { const stoppedAgo = `stopped at ${new Date(lastStop.stoppedAt).toLocaleString()}`; const otherProjects = lastStop.otherProjects ?? []; + const restoreProjectBySessionId = new Map(); // Build flat list of all sessions to restore, grouped for display const allRestoreSessions: string[] = [ ...(lastStop.projectId === projectId ? lastStop.sessionIds : []), ...otherProjects.flatMap((p) => p.sessionIds), ]; + for (const sessionId of lastStop.sessionIds) { + restoreProjectBySessionId.set(sessionId, lastStop.projectId); + } + for (const otherProject of otherProjects) { + for (const sessionId of otherProject.sessionIds) { + restoreProjectBySessionId.set(sessionId, otherProject.projectId); + } + } // Display grouped by project const currentProjectSessions = lastStop.projectId === projectId ? lastStop.sessionIds : []; @@ -1005,6 +1057,17 @@ async function runStartup( if (allRestoreSessions.length > 0) { const shouldRestore = await promptConfirm("Restore these sessions?", true); if (shouldRestore) { + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.restore_started", + level: "info", + summary: `restoring ${allRestoreSessions.length} session(s) from last-stop`, + data: { + sessionCount: allRestoreSessions.length, + stoppedAt: lastStop.stoppedAt, + }, + }); // Use global config so the session manager can see all projects let restoreConfig = config; if (otherProjects.length > 0) { @@ -1029,11 +1092,33 @@ async function runStartup( restoredCount++; } catch (err) { failedSessionIds.add(sessionId); + const restoreProjectId = restoreProjectBySessionId.get(sessionId) ?? projectId; + recordActivityEvent({ + projectId: restoreProjectId, + sessionId, + source: "cli", + kind: "cli.restore_session_failed", + level: "warn", + summary: `failed to restore session`, + data: { errorMessage: err instanceof Error ? err.message : String(err) }, + }); warnings.push( ` Warning: could not restore ${sessionId}: ${err instanceof Error ? err.message : String(err)}`, ); } } + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.restore_completed", + level: "info", + summary: `restored ${restoredCount}/${allRestoreSessions.length} session(s)`, + data: { + requested: allRestoreSessions.length, + restored: restoredCount, + failed: failedSessionIds.size, + }, + }); if (restoredCount === allRestoreSessions.length) { restoreSpinner.succeed( `Restored ${restoredCount}/${allRestoreSessions.length} session(s)`, @@ -1086,7 +1171,15 @@ async function runStartup( await clearLastStop(); } } - } catch { + } catch (err) { + recordActivityEvent({ + projectId, + source: "cli", + kind: "cli.last_stop_read_failed", + level: "warn", + summary: `failed to read or process last-stop state during startup`, + data: { errorMessage: err instanceof Error ? err.message : String(err) }, + }); // Non-fatal: don't block startup if last-stop handling fails } } @@ -1359,6 +1452,21 @@ export function registerStart(program: Command): void { reapOrphans?: boolean; }, ) => { + recordActivityEvent({ + source: "cli", + kind: "cli.start_invoked", + level: "info", + summary: "ao start invoked", + data: { + projectArg: projectArg ?? null, + dashboard: opts?.dashboard !== false, + orchestrator: opts?.orchestrator !== false, + rebuild: opts?.rebuild === true, + dev: opts?.dev === true, + interactive: opts?.interactive === true, + }, + }); + let releaseStartupLock: (() => void) | undefined; let startupLockReleased = false; const unlockStartup = (): void => { @@ -1491,6 +1599,13 @@ export function registerStart(program: Command): void { // Resolve happens below; the suffix mutation runs after. startNewOrchestrator = true; } else if (choice === "restart") { + recordActivityEvent({ + source: "cli", + kind: "cli.daemon_restart", + level: "info", + summary: `user chose restart, killing existing daemon`, + data: { existingPid: running.pid, existingPort: running.port }, + }); await killExistingDaemon(running); console.log(chalk.yellow("\n Stopped existing instance. Restarting...\n")); running = null; @@ -1649,6 +1764,18 @@ export function registerStart(program: Command): void { // Ctrl+C and `ao stop` (which sends SIGTERM) perform a full // graceful shutdown via the handler installed inside runStartup(). } catch (err) { + if (!isCliFailureEventRecordedError(err)) { + recordActivityEvent({ + source: "cli", + kind: "cli.start_failed", + level: "error", + summary: `ao start action failed`, + data: { + reason: "outer", + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); + } if (err instanceof Error) { console.error(chalk.red("\nError:"), err.message); } else { @@ -1720,6 +1847,17 @@ export function registerStop(program: Command): void { .option("--purge-session", "Delete mapped OpenCode session when stopping") .option("--all", "Stop all running AO instances") .action(async (projectArg?: string, opts: { purgeSession?: boolean; all?: boolean } = {}) => { + recordActivityEvent({ + source: "cli", + kind: "cli.stop_invoked", + level: "info", + summary: "ao stop invoked", + data: { + projectArg: projectArg ?? null, + all: opts.all === true, + purgeSession: opts.purgeSession === true, + }, + }); try { // Check running.json first const running = await getRunning(); @@ -1796,6 +1934,15 @@ export function registerStop(program: Command): void { killedSessionIds.push(session.id); } } catch (err) { + recordActivityEvent({ + projectId: session.projectId ?? _projectId, + sessionId: session.id, + source: "cli", + kind: "cli.stop_session_failed", + level: "warn", + summary: `failed to kill session during ao stop`, + data: { errorMessage: err instanceof Error ? err.message : String(err) }, + }); warnings.push( ` Warning: failed to stop ${session.id}: ${err instanceof Error ? err.message : String(err)}`, ); @@ -1840,12 +1987,48 @@ export function registerStop(program: Command): void { otherProjects.push({ projectId: pid, sessionIds: ids }); } - await writeLastStop({ - stoppedAt: new Date().toISOString(), - projectId: _projectId, - sessionIds: killedSessionIds.filter((id) => targetActive.some((s) => s.id === id)), - otherProjects: otherProjects.length > 0 ? otherProjects : undefined, - }); + const targetSessionIds = killedSessionIds.filter((id) => + targetActive.some((s) => s.id === id), + ); + try { + await writeLastStop({ + stoppedAt: new Date().toISOString(), + projectId: _projectId, + sessionIds: targetSessionIds, + otherProjects: otherProjects.length > 0 ? otherProjects : undefined, + }); + recordActivityEvent({ + projectId: _projectId, + source: "cli", + kind: "cli.last_stop_written", + level: "info", + summary: `last-stop state written with ${killedSessionIds.length} session(s)`, + data: { + targetSessionCount: targetSessionIds.length, + otherProjectCount: otherProjects.length, + totalKilled: killedSessionIds.length, + }, + }); + } catch (err) { + recordActivityEvent({ + projectId: _projectId, + source: "cli", + kind: "cli.last_stop_write_failed", + level: "error", + summary: `failed to write last-stop state during ao stop`, + data: { + targetSessionCount: targetSessionIds.length, + otherProjectCount: otherProjects.length, + totalKilled: killedSessionIds.length, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); + console.log( + chalk.yellow( + ` Could not write last-stop state: ${err instanceof Error ? err.message : String(err)}`, + ), + ); + } } } catch (err) { console.log( @@ -1872,7 +2055,29 @@ export function registerStop(program: Command): void { // 0x800700e8). No-op on non-Windows. await sweepWindowsPtyHostsBeforeParentKill(); await sweepRegisteredDaemonChildren(running.pid); - await killProcessTree(running.pid, "SIGTERM"); + try { + await killProcessTree(running.pid, "SIGTERM"); + recordActivityEvent({ + projectId: _projectId, + source: "cli", + kind: "cli.daemon_killed", + level: "info", + summary: `SIGTERM sent to parent ao start`, + data: { pid: running.pid, port: running.port }, + }); + } catch (err) { + recordActivityEvent({ + projectId: _projectId, + source: "cli", + kind: "cli.daemon_killed", + level: "warn", + summary: `parent ao start was already dead`, + data: { + pid: running.pid, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); + } await unregister(); } else { await sweepRegisteredDaemonChildren(); @@ -1892,6 +2097,16 @@ export function registerStop(program: Command): void { console.log(chalk.dim(` Projects: ${Object.keys(config.projects).join(", ")}\n`)); } } catch (err) { + recordActivityEvent({ + source: "cli", + kind: "cli.stop_failed", + level: "error", + summary: `ao stop action failed`, + data: { + projectArg: projectArg ?? null, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); if (err instanceof Error) { console.error(chalk.red("\nError:"), err.message); } else { diff --git a/packages/cli/src/commands/update.ts b/packages/cli/src/commands/update.ts index 8364d613d..8e3d37dd9 100644 --- a/packages/cli/src/commands/update.ts +++ b/packages/cli/src/commands/update.ts @@ -8,6 +8,7 @@ import { isWindows, loadConfig, loadGlobalConfig, + recordActivityEvent, type Session, } from "@aoagents/ao-core"; import { runRepoScript } from "../lib/script-runner.js"; @@ -83,6 +84,14 @@ export function registerUpdate(program: Command): void { const method = detectInstallMethod(); + recordActivityEvent({ + source: "cli", + kind: "cli.update_invoked", + level: "info", + summary: `ao update invoked (method: ${method})`, + data: { method, options: opts }, + }); + // Reject git-only flags up front when the install isn't a git source. // Without this, users copy/pasting `ao update --skip-smoke` from older // docs would silently no-op on npm/pnpm/bun installs (the flag would be @@ -225,6 +234,13 @@ async function handleGitUpdate(opts: { try { const exitCode = await runRepoScript("ao-update.sh", args); if (exitCode !== 0) { + recordActivityEvent({ + source: "cli", + kind: "cli.update_failed", + level: "error", + summary: `ao update (git) failed: ao-update.sh exited non-zero`, + data: { method: "git", exitCode }, + }); process.exit(exitCode); } invalidateCache(); @@ -233,6 +249,13 @@ async function handleGitUpdate(opts: { error instanceof Error && error.message.includes("Script not found: ao-update.sh") ) { + recordActivityEvent({ + source: "cli", + kind: "cli.update_failed", + level: "error", + summary: `ao update (git) failed: ao-update.sh missing from bundled assets`, + data: { method: "git", reason: "script_missing" }, + }); console.error( chalk.red( "ao-update.sh is missing from the bundled assets. " + @@ -243,6 +266,16 @@ async function handleGitUpdate(opts: { process.exit(1); } + recordActivityEvent({ + source: "cli", + kind: "cli.update_failed", + level: "error", + summary: `ao update (git) failed`, + data: { + method: "git", + errorMessage: error instanceof Error ? error.message : String(error), + }, + }); console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } @@ -260,9 +293,34 @@ async function handleNpmUpdate(method: InstallMethod): Promise { // would overwrite cache.channel before we can read it. const previousChannel = readCachedUpdateInfo(method)?.channel; - const info = await checkForUpdate({ force: true, channel }); + let info: Awaited>; + try { + info = await checkForUpdate({ force: true, channel }); + } catch (error) { + recordActivityEvent({ + source: "cli", + kind: "cli.update_failed", + level: "error", + summary: `ao update (${method}) failed: npm registry lookup threw`, + data: { + method, + channel, + reason: "registry_lookup_threw", + errorMessage: error instanceof Error ? error.message : String(error), + }, + }); + console.error(chalk.red("Could not reach npm registry. Check your network and try again.")); + process.exit(1); + } if (!info.latestVersion) { + recordActivityEvent({ + source: "cli", + kind: "cli.update_failed", + level: "error", + summary: `ao update (${method}) failed: npm registry lookup returned no version`, + data: { method, channel, reason: "registry_unreachable" }, + }); console.error(chalk.red("Could not reach npm registry. Check your network and try again.")); process.exit(1); } @@ -369,6 +427,13 @@ async function handleNpmUpdate(method: InstallMethod): Promise { invalidateCache(); console.log(chalk.green("\nUpdate complete.")); } else { + recordActivityEvent({ + source: "cli", + kind: "cli.update_failed", + level: "error", + summary: `ao update (${method}) failed: install command exited non-zero`, + data: { method, command, exitCode }, + }); process.exit(exitCode); } } diff --git a/packages/cli/src/lib/credential-resolver.ts b/packages/cli/src/lib/credential-resolver.ts index 133ab6224..75fb6a6ff 100644 --- a/packages/cli/src/lib/credential-resolver.ts +++ b/packages/cli/src/lib/credential-resolver.ts @@ -12,6 +12,7 @@ import { existsSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; +import { recordActivityEvent } from "@aoagents/ao-core"; /** Keys that AO agents commonly need and OpenClaw may already store. */ const RESOLVABLE_KEYS = [ @@ -66,8 +67,18 @@ function readOpenClawKeys(): Record { } } } - } catch { - // Malformed config — silently skip. + } catch (err) { + recordActivityEvent({ + source: "cli", + kind: "cli.credential_load_failed", + level: "warn", + summary: `failed to read or parse ~/.openclaw/openclaw.json`, + data: { + configPath, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); + // Malformed config — silently skip (event surfaces it). } return keys; diff --git a/packages/cli/src/lib/resolve-project.ts b/packages/cli/src/lib/resolve-project.ts index b6bb2bc69..8d1c34de2 100644 --- a/packages/cli/src/lib/resolve-project.ts +++ b/packages/cli/src/lib/resolve-project.ts @@ -27,6 +27,7 @@ import { isRepoUrl, loadConfig, parseRepoUrl, + recordActivityEvent, registerProjectInGlobalConfig, resolveCloneTarget, isRepoAlreadyCloned, @@ -196,6 +197,18 @@ async function fromUrlIntoGlobal(arg: string, deps: ResolveDeps): Promise timeoutMs) { + recordActivityEvent({ + source: "cli", + kind: "cli.lock_timeout", + level: "warn", + summary: `lock acquisition timed out`, + data: { + resourceName, + lockFile, + timeoutMs, + attempts: attempt, + }, + }); throw new Error(`Could not acquire ${resourceName} (${lockFile})`); } @@ -246,6 +258,18 @@ export async function getRunning(): Promise { if (!isProcessAlive(state.pid)) { // Stale entry — process is dead, clean up writeState(null); + recordActivityEvent({ + source: "cli", + kind: "cli.stale_running_pruned", + level: "warn", + summary: `pruned stale running.json entry (dead pid ${state.pid})`, + data: { + pid: state.pid, + port: state.port, + startedAt: state.startedAt, + projects: state.projects, + }, + }); return null; } diff --git a/packages/cli/src/lib/shutdown.ts b/packages/cli/src/lib/shutdown.ts index 228f5f88d..8e024e4f8 100644 --- a/packages/cli/src/lib/shutdown.ts +++ b/packages/cli/src/lib/shutdown.ts @@ -16,6 +16,7 @@ import { isTerminalSession, loadConfig, markDaemonShutdownHandlerInstalled, + recordActivityEvent, sweepDaemonChildren, } from "@aoagents/ao-core"; import { stopBunTmpJanitor } from "./bun-tmp-janitor.js"; @@ -62,6 +63,15 @@ export function installShutdownHandlers(ctx: ShutdownContext): void { const exitCode = signal === "SIGINT" ? 130 : 0; + recordActivityEvent({ + projectId: ctx.projectId, + source: "cli", + kind: "cli.shutdown_signal", + level: "info", + summary: `received ${signal}, beginning graceful shutdown`, + data: { signal, exitCode }, + }); + try { stopProjectSupervisor(); stopAllLifecycleWorkers(); @@ -69,7 +79,17 @@ export function installShutdownHandlers(ctx: ShutdownContext): void { // Best-effort — never block shutdown on observability. } - const forceExit = setTimeout(() => process.exit(exitCode), SHUTDOWN_TIMEOUT_MS); + const forceExit = setTimeout(() => { + recordActivityEvent({ + projectId: ctx.projectId, + source: "cli", + kind: "cli.shutdown_force_exit", + level: "warn", + summary: `force-exit after ${SHUTDOWN_TIMEOUT_MS}ms timeout`, + data: { signal, timeoutMs: SHUTDOWN_TIMEOUT_MS, exitCode }, + }); + process.exit(exitCode); + }, SHUTDOWN_TIMEOUT_MS); forceExit.unref(); void (async () => { @@ -86,8 +106,16 @@ export function installShutdownHandlers(ctx: ShutdownContext): void { if (result.cleaned || result.alreadyTerminated) { killedSessionIds.push(session.id); } - } catch { - // Best-effort per session + } catch (err) { + recordActivityEvent({ + projectId: session.projectId ?? ctx.projectId, + sessionId: session.id, + source: "cli", + kind: "cli.shutdown_session_kill_failed", + level: "warn", + summary: `failed to kill session during shutdown`, + data: { errorMessage: err instanceof Error ? err.message : String(err) }, + }); } } @@ -107,18 +135,49 @@ export function installShutdownHandlers(ctx: ShutdownContext): void { for (const [pid, ids] of otherByProject) { otherProjects.push({ projectId: pid, sessionIds: ids }); } - await writeLastStop({ - stoppedAt: new Date().toISOString(), - projectId: ctx.projectId, - sessionIds: targetIds, - otherProjects: otherProjects.length > 0 ? otherProjects : undefined, - }); + try { + await writeLastStop({ + stoppedAt: new Date().toISOString(), + projectId: ctx.projectId, + sessionIds: targetIds, + otherProjects: otherProjects.length > 0 ? otherProjects : undefined, + }); + } catch (err) { + recordActivityEvent({ + projectId: ctx.projectId, + source: "cli", + kind: "cli.last_stop_write_failed", + level: "error", + summary: `failed to write last-stop state during shutdown`, + data: { + targetSessionCount: targetIds.length, + otherProjectCount: otherProjects.length, + totalKilled: killedSessionIds.length, + errorMessage: err instanceof Error ? err.message : String(err), + }, + }); + } } await sweepDaemonChildren({ ownerPid: process.pid }); await unregister(); - } catch { - // Best-effort — always exit even if cleanup fails + recordActivityEvent({ + projectId: ctx.projectId, + source: "cli", + kind: "cli.shutdown_completed", + level: "info", + summary: `clean shutdown completed`, + data: { signal, killedSessionCount: killedSessionIds.length, exitCode }, + }); + } catch (err) { + recordActivityEvent({ + projectId: ctx.projectId, + source: "cli", + kind: "cli.shutdown_failed", + level: "error", + summary: `shutdown body threw before cleanup completed`, + data: { signal, errorMessage: err instanceof Error ? err.message : String(err) }, + }); } try { // Await any in-flight sweep so shutdown does not exit while diff --git a/packages/core/src/activity-events.ts b/packages/core/src/activity-events.ts index ed9d79841..d7a9313d9 100644 --- a/packages/core/src/activity-events.ts +++ b/packages/core/src/activity-events.ts @@ -25,6 +25,7 @@ export type ActivityEventSource = | "notifier" | "reaction" | "report-watcher" + | "cli" | "config" | "plugin-registry" | "migration" diff --git a/packages/integration-tests/src/tracker-linear.integration.test.ts b/packages/integration-tests/src/tracker-linear.integration.test.ts index 6749761ea..7a8583cdc 100644 --- a/packages/integration-tests/src/tracker-linear.integration.test.ts +++ b/packages/integration-tests/src/tracker-linear.integration.test.ts @@ -38,15 +38,28 @@ const canRun = hasCredentials && Boolean(LINEAR_TEAM_ID); // Helpers // --------------------------------------------------------------------------- +interface LinearGraphQLOptions { + maxAttempts?: number; + retryDelayMs?: number; + timeoutMs?: number; +} + /** * Direct GraphQL call for test setup/cleanup. * Only available when LINEAR_API_KEY is set. */ -function linearGraphQL(query: string, variables: Record): Promise { +function linearGraphQL( + query: string, + variables: Record, + options: LinearGraphQLOptions = {}, +): Promise { if (!LINEAR_API_KEY) { throw new Error("linearGraphQL requires LINEAR_API_KEY"); } const body = JSON.stringify({ query, variables }); + const maxAttempts = options.maxAttempts ?? 3; + const retryDelayMs = options.retryDelayMs ?? 1_000; + const timeoutMs = options.timeoutMs ?? 30_000; async function executeWithRetry(attempt = 1): Promise { try { @@ -95,9 +108,9 @@ function linearGraphQL(query: string, variables: Record): Pr }, ); - req.setTimeout(30_000, () => { + req.setTimeout(timeoutMs, () => { req.destroy(); - reject(new Error("Linear API request timed out")); + reject(new Error(`Linear API request timed out after ${timeoutMs}ms`)); }); req.on("error", (err) => reject(err)); @@ -105,8 +118,8 @@ function linearGraphQL(query: string, variables: Record): Pr req.end(); }); } catch (err) { - if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, 1_000)); + if (attempt < maxAttempts) { + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); return executeWithRetry(attempt + 1); } throw err; @@ -179,7 +192,18 @@ describe.skipIf(!canRun)("tracker-linear (integration)", () => { if (!issueIdentifier) return; try { - if (issueUuid && LINEAR_API_KEY) { + if (LINEAR_API_KEY) { + const cleanupRequest = { maxAttempts: 1, timeoutMs: 5_000 }; + if (!issueUuid) { + const data = await linearGraphQL<{ issue: { id: string } }>( + `query($id: String!) { issue(id: $id) { id } }`, + { id: issueIdentifier }, + cleanupRequest, + ); + issueUuid = data.issue.id; + } + + if (!issueUuid) return; await linearGraphQL( `mutation($id: String!) { issueUpdate(id: $id, input: { trashed: true }) { @@ -187,6 +211,7 @@ describe.skipIf(!canRun)("tracker-linear (integration)", () => { } }`, { id: issueUuid }, + cleanupRequest, ); } else { // Composio-only: best-effort close via plugin diff --git a/packages/plugins/tracker-linear/src/index.ts b/packages/plugins/tracker-linear/src/index.ts index 6101ac1c3..850431e30 100644 --- a/packages/plugins/tracker-linear/src/index.ts +++ b/packages/plugins/tracker-linear/src/index.ts @@ -53,6 +53,43 @@ type GraphQLTransport = (query: string, variables?: Record) // --------------------------------------------------------------------------- const LINEAR_API_URL = "https://api.linear.app/graphql"; +const DIRECT_TRANSPORT_MAX_ATTEMPTS = 3; +const DIRECT_TRANSPORT_RETRY_DELAY_MS = 500; + +class LinearHttpError extends Error { + constructor( + readonly status: number, + body: string, + ) { + super(`Linear API returned HTTP ${status}: ${body.slice(0, 200)}`); + } + + get transient(): boolean { + return this.status === 408 || this.status === 429 || this.status >= 500; + } +} + +class LinearNetworkError extends Error { + constructor(message: string) { + super(`Linear API network error: ${message}`); + } +} + +class LinearTimeoutError extends Error { + constructor() { + super("Linear API request timed out after 30s"); + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isRetryableDirectTransportError(err: unknown): boolean { + if (err instanceof LinearHttpError) return err.transient; + if (err instanceof LinearTimeoutError) return true; + return err instanceof LinearNetworkError; +} function getApiKey(): string { const key = process.env["LINEAR_API_KEY"]; @@ -70,84 +107,100 @@ interface LinearResponse { } function createDirectTransport(): GraphQLTransport { - return (query: string, variables?: Record): Promise => { + return async (query: string, variables?: Record): Promise => { const apiKey = getApiKey(); const body = JSON.stringify({ query, variables }); - return new Promise((resolve, reject) => { - const url = new URL(LINEAR_API_URL); - let settled = false; - const settle = (fn: () => void) => { - if (!settled) { - settled = true; - fn(); - } - }; + const execute = (): Promise => + new Promise((resolve, reject) => { + const url = new URL(LINEAR_API_URL); + let settled = false; + const settle = (fn: () => void) => { + if (!settled) { + settled = true; + fn(); + } + }; - const req = request( - { - hostname: url.hostname, - path: url.pathname, - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: apiKey, - "Content-Length": Buffer.byteLength(body), - }, - }, - (res) => { - const chunks: Buffer[] = []; - res.on("error", (err: Error) => settle(() => reject(err))); - res.on("data", (chunk: Buffer) => chunks.push(chunk)); - res.on("end", () => { - settle(() => { - try { - const text = Buffer.concat(chunks).toString("utf-8"); - const status = res.statusCode ?? 0; - if (status < 200 || status >= 300) { - reject(new Error(`Linear API returned HTTP ${status}: ${text.slice(0, 200)}`)); - return; - } - const json: LinearResponse = JSON.parse(text); - if (json.errors && json.errors.length > 0) { - reject(new Error(`Linear API error: ${json.errors[0].message}`)); - return; - } - if (!json.data) { - reject(new Error("Linear API returned no data")); - return; - } - resolve(json.data); - } catch (err) { - reject(err); - } - }); - }); - }, - ); - - req.setTimeout(30_000, () => { - settle(() => { - req.destroy(); - recordTransportActivityEvent({ - source: "tracker", - kind: "tracker.api_timeout", - level: "warn", - summary: "Linear API request timed out after 30s", - data: { - plugin: "tracker-linear", - transport: "direct", - timeoutMs: 30_000, + const req = request( + { + hostname: url.hostname, + path: url.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: apiKey, + "Content-Length": Buffer.byteLength(body), }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("error", (err: Error) => + settle(() => reject(new LinearNetworkError(err.message))), + ); + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => { + settle(() => { + try { + const text = Buffer.concat(chunks).toString("utf-8"); + const status = res.statusCode ?? 0; + if (status < 200 || status >= 300) { + reject(new LinearHttpError(status, text)); + return; + } + const json: LinearResponse = JSON.parse(text); + if (json.errors && json.errors.length > 0) { + reject(new Error(`Linear API error: ${json.errors[0].message}`)); + return; + } + if (!json.data) { + reject(new Error("Linear API returned no data")); + return; + } + resolve(json.data); + } catch (err) { + reject(err); + } + }); + }); + }, + ); + + req.setTimeout(30_000, () => { + settle(() => { + req.destroy(); + recordTransportActivityEvent({ + source: "tracker", + kind: "tracker.api_timeout", + level: "warn", + summary: "Linear API request timed out after 30s", + data: { + plugin: "tracker-linear", + transport: "direct", + timeoutMs: 30_000, + }, + }); + reject(new LinearTimeoutError()); }); - reject(new Error("Linear API request timed out after 30s")); }); + + req.on("error", (err) => settle(() => reject(new LinearNetworkError(err.message)))); + req.write(body); + req.end(); }); - req.on("error", (err) => settle(() => reject(err))); - req.write(body); - req.end(); - }); + for (let attempt = 1; attempt <= DIRECT_TRANSPORT_MAX_ATTEMPTS; attempt++) { + try { + return await execute(); + } catch (err) { + const shouldRetry = + isRetryableDirectTransportError(err) && attempt < DIRECT_TRANSPORT_MAX_ATTEMPTS; + if (!shouldRetry) throw err; + await sleep(DIRECT_TRANSPORT_RETRY_DELAY_MS * attempt); + } + } + + throw new Error("unreachable"); }; } diff --git a/packages/plugins/tracker-linear/test/index.test.ts b/packages/plugins/tracker-linear/test/index.test.ts index d96d21e55..d31f562ac 100644 --- a/packages/plugins/tracker-linear/test/index.test.ts +++ b/packages/plugins/tracker-linear/test/index.test.ts @@ -136,6 +136,40 @@ function mockHTTPError(statusCode: number, body: string) { ); } +/** Queue a request-level network error before Linear returns a response. */ +function mockRequestError(message: string) { + requestMock.mockImplementationOnce(() => { + const req = Object.assign(new EventEmitter(), { + write: vi.fn(), + end: vi.fn(() => { + process.nextTick(() => req.emit("error", new Error(message))); + }), + destroy: vi.fn(), + setTimeout: vi.fn(), + }); + return req; + }); +} + +/** Queue a client-side timeout before Linear returns a response. */ +function mockRequestTimeout() { + requestMock.mockImplementationOnce(() => { + let timeoutHandler: (() => void) | undefined; + const req = Object.assign(new EventEmitter(), { + write: vi.fn(), + end: vi.fn(() => { + process.nextTick(() => timeoutHandler?.()); + }), + destroy: vi.fn(), + setTimeout: vi.fn((_ms: number, handler: () => void) => { + timeoutHandler = handler; + return req; + }), + }); + return req; + }); +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -271,11 +305,41 @@ describe("tracker-linear plugin", () => { }); it("throws on HTTP errors", async () => { - mockHTTPError(500, "Internal Server Error"); + mockHTTPError(400, "Bad Request"); await expect(tracker.getIssue("INT-123", project)).rejects.toThrow( - "Linear API returned HTTP 500", + "Linear API returned HTTP 400", ); }); + + it("retries transient HTTP errors", async () => { + mockHTTPError(502, "Bad Gateway"); + mockLinearAPI({ issue: sampleIssueNode }); + + const issue = await tracker.getIssue("INT-123", project); + + expect(issue.id).toBe("INT-123"); + expect(requestMock).toHaveBeenCalledTimes(2); + }); + + it("retries request-level network errors", async () => { + mockRequestError("ECONNRESET"); + mockLinearAPI({ issue: sampleIssueNode }); + + const issue = await tracker.getIssue("INT-123", project); + + expect(issue.id).toBe("INT-123"); + expect(requestMock).toHaveBeenCalledTimes(2); + }); + + it("retries client-side request timeouts", async () => { + mockRequestTimeout(); + mockLinearAPI({ issue: sampleIssueNode }); + + const issue = await tracker.getIssue("INT-123", project); + + expect(issue.id).toBe("INT-123"); + expect(requestMock).toHaveBeenCalledTimes(2); + }); }); // ---- isCompleted -------------------------------------------------------