agent-orchestrator/backend/internal/session_manager/manager.go

1485 lines
58 KiB
Go

// Package sessionmanager drives internal session command operations over runtime,
// agent, workspace, storage, messenger, and lifecycle dependencies.
package sessionmanager
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/aoagents/agent-orchestrator/backend/internal/domain"
"github.com/aoagents/agent-orchestrator/backend/internal/ports"
aoprocess "github.com/aoagents/agent-orchestrator/backend/internal/process"
"github.com/aoagents/agent-orchestrator/backend/internal/skillassets"
)
// Sentinel errors returned by the Session Manager; callers match them with
// errors.Is.
var (
ErrNotFound = errors.New("session: not found")
ErrNotRestorable = errors.New("session: not restorable (not terminal)")
ErrTerminated = errors.New("session: terminated")
ErrIncompleteHandle = errors.New("session: incomplete teardown handle")
// ErrProjectNotResolvable means the spawn's project has no usable repo
// (unregistered, archived, or missing a path). The API maps it to a 400.
ErrProjectNotResolvable = errors.New("session: project repo not resolvable")
// ErrUnknownHarness means the requested agent harness has no registered
// adapter. The API maps it to a 400 so a typo'd `--harness` is a validation
// error, not an opaque 500.
ErrUnknownHarness = errors.New("session: unknown agent harness")
// ErrMissingHarness means neither the spawn request nor the project's role
// config selected an agent. Worker/orchestrator spawns must be explicit.
ErrMissingHarness = errors.New("session: agent harness required")
// ErrNotResumable means a terminated session cannot be relaunched: its adapter
// cannot natively resume it AND it has no prompt to fresh-launch from, and it is
// not an orchestrator (orchestrators are promptless by design and relaunch fresh
// with the system prompt only). Workers without a task and without a native
// session id have nothing meaningful to restore.
ErrNotResumable = errors.New("session: nothing to resume from")
)
// Env vars a spawned process reads to learn who it is.
const (
EnvSessionID = "AO_SESSION_ID"
EnvProjectID = "AO_PROJECT_ID"
EnvIssueID = "AO_ISSUE_ID"
// EnvDataDir tells a spawned agent's AO hook commands where the store lives.
EnvDataDir = "AO_DATA_DIR"
)
// hookBinaryName is the executable name the workspace hook commands invoke:
// every agent adapter installs a bare `ao hooks <agent> <event>`. The session
// PATH pin (hookPATH) only works when the daemon's own executable carries this
// name, since prepending its directory must change what `ao` resolves to.
const hookBinaryName = "ao"
type lifecycleRecorder interface {
MarkSpawned(ctx context.Context, id domain.SessionID, metadata domain.SessionMetadata) error
MarkTerminated(ctx context.Context, id domain.SessionID) error
}
type runtimeController interface {
Create(ctx context.Context, cfg ports.RuntimeConfig) (ports.RuntimeHandle, error)
Destroy(ctx context.Context, handle ports.RuntimeHandle) error
// IsAlive reports whether the handle's runtime session still exists. Used by
// Reconcile on boot to adopt crash-surviving sessions and reap leaked ones.
IsAlive(ctx context.Context, handle ports.RuntimeHandle) (bool, error)
}
// Store is the persistence surface needed by the internal session Manager.
type Store interface {
// GetProject loads a project row so spawn can resolve its per-project agent
// config into the launch command. ok=false means the project is unknown.
GetProject(ctx context.Context, id string) (domain.ProjectRecord, bool, error)
CreateSession(ctx context.Context, rec domain.SessionRecord) (domain.SessionRecord, error)
GetSession(ctx context.Context, id domain.SessionID) (domain.SessionRecord, bool, error)
ListSessions(ctx context.Context, project domain.ProjectID) ([]domain.SessionRecord, error)
ListAllSessions(ctx context.Context) ([]domain.SessionRecord, error)
// DeleteSession removes a session row only if it is still in seed state
// (no workspace, runtime handle, agent session id, or prompt; not
// terminated). Returns deleted=true when removal happened; deleted=false
// when the row had already progressed past seed state — preserving the
// no-resurrection guarantee for live sessions.
DeleteSession(ctx context.Context, id domain.SessionID) (bool, error)
// UpsertSessionWorktree records or updates the worktree row for a session.
// SaveAndTeardownAll writes the preserved_ref here (even when empty) as the
// "shutdown-saved" marker before ForceDestroying the worktree.
UpsertSessionWorktree(ctx context.Context, row domain.SessionWorktreeRecord) error
// ListSessionWorktrees returns every worktree row for a session. RestoreAll
// uses this to identify sessions saved by the last SaveAndTeardownAll: the
// presence of any row is the marker; preserved_ref may be empty for clean
// worktrees.
ListSessionWorktrees(ctx context.Context, id domain.SessionID) ([]domain.SessionWorktreeRecord, error)
// DeleteSessionWorktrees consumes stale shutdown-restore markers. Explicit
// Kill and successful RestoreAll must remove these rows to prevent
// resurrecting sessions the user intentionally terminated.
DeleteSessionWorktrees(ctx context.Context, id domain.SessionID) error
}
// Manager coordinates internal session spawn, restore, kill, and cleanup over
// the outbound ports. User-facing read-model assembly lives in the service package.
type Manager struct {
runtime runtimeController
agents ports.AgentResolver
workspace ports.Workspace
store Store
messenger ports.AgentMessenger
lcm lifecycleRecorder
dataDir string
clock func() time.Time
// lookPath is exec.LookPath in production; tests substitute a stub so
// they don't need real binaries on PATH. Returns ports.ErrAgentBinaryNotFound
// when the binary is missing so the sentinel propagates through toAPIError.
lookPath func(string) (string, error)
// executable resolves the daemon's own binary (os.Executable in
// production); its directory is prepended to spawned sessions' PATH so the
// workspace hook commands resolve back to this daemon. Tests inject a stub.
executable func() (string, error)
logger *slog.Logger
}
// Deps are the collaborators a Session Manager needs; New wires them together.
type Deps struct {
Runtime runtimeController
Agents ports.AgentResolver
Workspace ports.Workspace
Store Store
Messenger ports.AgentMessenger
Lifecycle lifecycleRecorder
// DataDir is exported to spawned agents as AO_DATA_DIR so their hook
// commands can open the same store.
DataDir string
Clock func() time.Time
// LookPath overrides exec.LookPath for the pre-launch agent-binary check.
// Production wiring leaves this nil and the manager defaults to
// exec.LookPath; tests inject a stub so they need not seed real binaries.
LookPath func(string) (string, error)
// Executable overrides os.Executable for the session PATH pin (see
// hookPATH). Production wiring leaves this nil; tests inject a stub so they
// control what the test binary appears to be.
Executable func() (string, error)
// Logger receives spawn-time diagnostics (e.g. when the session PATH
// cannot be pinned to the daemon binary). Nil defaults to slog.Default().
Logger *slog.Logger
}
// New builds a Session Manager from its dependencies, defaulting the clock to
// time.Now when Deps.Clock is nil.
func New(d Deps) *Manager {
m := &Manager{
runtime: d.Runtime,
agents: d.Agents,
workspace: d.Workspace,
store: d.Store,
messenger: d.Messenger,
lcm: d.Lifecycle,
dataDir: d.DataDir,
clock: d.Clock,
lookPath: d.LookPath,
executable: d.Executable,
logger: d.Logger,
}
if m.clock == nil {
// UTC so spawn-stamped CreatedAt/UpdatedAt match every other session
// write (rename, activity) — all of which use time.Now().UTC(). A local
// default produced mixed-timezone timestamps in `ao session get`.
m.clock = func() time.Time { return time.Now().UTC() }
}
if m.lookPath == nil {
m.lookPath = exec.LookPath
}
if m.executable == nil {
m.executable = os.Executable
}
if m.logger == nil {
m.logger = slog.Default()
}
return m
}
// Spawn creates the session row (which assigns the "{project}-{n}" id), then the
// workspace and runtime, then reports completion to the LCM. If workspace
// materialization fails the still-seed row is deleted outright; a later failure
// parks the row as terminated and rolls back what was built.
func (m *Manager) Spawn(ctx context.Context, cfg ports.SpawnConfig) (domain.SessionRecord, error) {
project, err := m.loadProject(ctx, cfg.ProjectID)
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("spawn: %w", err)
}
// A per-project role override picks the harness when the spawn names none,
// so a project can default workers to one agent and orchestrators to another.
cfg.Harness = effectiveHarness(cfg.Harness, cfg.Kind, project.Config)
if cfg.Harness == "" {
return domain.SessionRecord{}, fmt.Errorf("spawn: %w: configure project %s.agent or pass --harness", ErrMissingHarness, roleConfigName(cfg.Kind))
}
// Reject an unknown harness before any durable state is created. Doing this
// after CreateSession would leave a terminated orphan row and waste a
// worktree on a spawn that can never launch.
if _, ok := m.agents.Agent(cfg.Harness); !ok {
return domain.SessionRecord{}, fmt.Errorf("spawn: %w: %q", ErrUnknownHarness, cfg.Harness)
}
if err := m.validateRuntimePrerequisites(); err != nil {
return domain.SessionRecord{}, fmt.Errorf("spawn: %w", err)
}
prompt, systemPrompt, err := m.buildSpawnTexts(ctx, cfg)
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("spawn: prompt: %w", err)
}
rec, err := m.store.CreateSession(ctx, seedRecord(cfg, m.clock()))
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("spawn: create: %w", err)
}
id := rec.ID
systemPromptFile, err := m.prepareSystemPromptFile(id, cfg.Harness, systemPrompt)
if err != nil {
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: system prompt file: %w", id, err)
}
branch := cfg.Branch
if branch == "" {
branch = defaultSessionBranch(id, cfg.Kind, sessionPrefix(project))
}
ws, err := m.workspace.Create(ctx, ports.WorkspaceConfig{
ProjectID: cfg.ProjectID,
SessionID: id,
Kind: cfg.Kind,
SessionPrefix: sessionPrefix(project),
Branch: branch,
BaseBranch: project.Config.WithDefaults().DefaultBranch,
})
if err != nil {
// Nothing observable exists yet — no worktree, no runtime — so the seed
// row is deleted outright instead of accumulating as a terminated orphan
// in session lists (e.g. when gitworktree refuses the branch).
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: workspace: %w", id, err)
}
// Per-project workspace provisioning: symlink shared files, then run any
// post-create commands (e.g. `pnpm install`) before the agent launches.
if err := m.provisionWorkspace(ctx, project, ws.Path); err != nil {
_ = m.workspace.Destroy(ctx, ws)
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: provision: %w", id, err)
}
agent, ok := m.agents.Agent(cfg.Harness)
if !ok {
_ = m.workspace.Destroy(ctx, ws)
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: no agent adapter for harness %q", id, cfg.Harness)
}
if err := m.prepareWorkspace(ctx, agent, id, ws.Path); err != nil {
_ = m.workspace.Destroy(ctx, ws)
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: %w", id, err)
}
agentConfig := effectiveAgentConfig(cfg.Kind, project.Config)
argv, err := agent.GetLaunchCommand(ctx, ports.LaunchConfig{
SessionID: string(id),
WorkspacePath: ws.Path,
Prompt: prompt,
SystemPrompt: systemPrompt,
SystemPromptFile: systemPromptFile,
IssueID: string(cfg.IssueID),
Config: agentConfig,
Permissions: agentConfig.Permissions,
})
if err != nil {
_ = m.workspace.Destroy(ctx, ws)
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: launch command: %w", id, err)
}
// Pre-flight: confirm argv[0] actually exists on PATH (or as an absolute
// path the adapter returned) BEFORE handing the launch to the runtime.
// tmux happily creates a session+pane around a missing command, so an
// unresolved binary would leak through as a "live" session that never ran.
if err := m.validateAgentBinary(argv); err != nil {
_ = m.workspace.Destroy(ctx, ws)
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: %w", id, err)
}
handle, err := m.runtime.Create(ctx, ports.RuntimeConfig{
SessionID: id,
WorkspacePath: ws.Path,
Argv: argv,
Env: m.runtimeEnv(id, cfg.ProjectID, cfg.IssueID, project.Config.Env),
})
if err != nil {
_ = m.workspace.Destroy(ctx, ws)
m.rollbackSpawnSeedRow(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: runtime: %w", id, err)
}
metadata := domain.SessionMetadata{Branch: ws.Branch, WorkspacePath: ws.Path, RuntimeHandleID: handle.ID, Prompt: prompt}
if err := m.lcm.MarkSpawned(ctx, id, metadata); err != nil {
_ = m.runtime.Destroy(ctx, handle)
_ = m.workspace.Destroy(ctx, ws)
m.markSpawnFailedTerminated(ctx, id)
return domain.SessionRecord{}, fmt.Errorf("spawn %s: completed: %w", id, err)
}
return m.getRecord(ctx, id)
}
// loadProject loads the project record so spawn can resolve its per-project
// config (harness/agent overrides, env, branch, rules, provisioning). A missing
// project yields a zero record rather than an error: the project may be
// unregistered yet still have live sessions, and an empty config simply means
// every field falls back to its default.
func (m *Manager) loadProject(ctx context.Context, projectID domain.ProjectID) (domain.ProjectRecord, error) {
row, ok, err := m.store.GetProject(ctx, string(projectID))
if err != nil {
return domain.ProjectRecord{}, fmt.Errorf("load project: %w", err)
}
if !ok {
return domain.ProjectRecord{}, nil
}
return row, nil
}
// effectiveHarness resolves the harness for a spawn: an explicit harness wins;
// otherwise the project's role override for the session kind applies. Empty is
// invalid for new worker/orchestrator launches and is rejected by Spawn.
func effectiveHarness(explicit domain.AgentHarness, kind domain.SessionKind, cfg domain.ProjectConfig) domain.AgentHarness {
if explicit != "" {
return explicit
}
if role := roleOverride(kind, cfg).Harness; role != "" {
return role
}
return ""
}
func roleConfigName(kind domain.SessionKind) string {
if kind == domain.KindOrchestrator {
return "orchestrator"
}
return "worker"
}
// effectiveAgentConfig merges the role override's agent config over the
// project's base agent config; set override fields win.
func effectiveAgentConfig(kind domain.SessionKind, cfg domain.ProjectConfig) ports.AgentConfig {
merged := cfg.AgentConfig
override := roleOverride(kind, cfg).AgentConfig
if override.Model != "" {
merged.Model = override.Model
}
if override.Permissions != "" {
merged.Permissions = override.Permissions
}
return merged
}
func roleOverride(kind domain.SessionKind, cfg domain.ProjectConfig) domain.RoleOverride {
if kind == domain.KindOrchestrator {
return cfg.Orchestrator
}
return cfg.Worker
}
// sessionPrefix returns the display prefix for a project: the explicit
// SessionPrefix when set, otherwise the first 12 characters of the project ID.
func sessionPrefix(project domain.ProjectRecord) string {
if p := strings.TrimSpace(project.Config.SessionPrefix); p != "" {
return p
}
if len(project.ID) <= 12 {
return project.ID
}
return project.ID[:12]
}
// markSpawnFailedTerminated best-effort parks an orphaned spawn as terminated.
// A phantom half-spawned row is worse than a terminal one; we only delete the
// row when nothing observable has landed yet (seed state) via rollbackSpawn or
// rollbackSpawnSeedRow.
func (m *Manager) markSpawnFailedTerminated(ctx context.Context, id domain.SessionID) {
_ = m.lcm.MarkTerminated(ctx, id)
m.cleanupSystemPromptDir(id)
}
// rollbackSpawnSeedRow best-effort removes the row of a spawn that failed
// before anything observable (worktree, runtime) was built, so failed spawns
// don't accumulate terminated rows in session lists. DeleteSession only removes
// rows still in seed state; if the row has progressed or the delete itself
// fails, fall back to parking it terminated so a phantom row never looks live.
func (m *Manager) rollbackSpawnSeedRow(ctx context.Context, id domain.SessionID) {
if deleted, err := m.store.DeleteSession(ctx, id); err == nil && deleted {
m.cleanupSystemPromptDir(id)
return
}
m.markSpawnFailedTerminated(ctx, id)
}
// rollbackSpawn deletes a session row when it is still in seed state — used
// when an out-of-band step that happens AFTER `Spawn` returns (e.g. PR claim
// over HTTP) has failed and the caller wants the partially-spawned session
// gone without leaving a terminated orphan visible under `--include-terminated`.
//
// If the row has progressed past seed state (workspace exists, runtime created,
// etc.), DeleteSession is a no-op and rollbackSpawn falls back to a Kill so the
// runtime/workspace are torn down. Returns (deleted, killed):
// - deleted=true: the row was a seed row and has been removed
// - killed=true: the row had spawn output and was torn down + terminated
// - both false: the row was already terminated or absent — benign no-op
func (m *Manager) rollbackSpawn(ctx context.Context, id domain.SessionID) (deleted, killed bool, err error) {
deleted, err = m.store.DeleteSession(ctx, id)
if err != nil {
return false, false, fmt.Errorf("rollback %s: %w", id, err)
}
if deleted {
m.cleanupSystemPromptDir(id)
return true, false, nil
}
killed, err = m.Kill(ctx, id)
if err != nil {
return false, false, err
}
return false, killed, nil
}
// RollbackSpawn is the public surface of rollbackSpawn for service-layer callers.
func (m *Manager) RollbackSpawn(ctx context.Context, id domain.SessionID) (deleted, killed bool, err error) {
return m.rollbackSpawn(ctx, id)
}
// Kill records terminal intent with the LCM, then tears down the runtime and
// workspace. A workspace teardown refused by the worktree-remove safety
// (uncommitted work) is never forced: the session still terminates and Kill
// succeeds with freed=false, signalling the workspace was preserved.
//
// A session whose runtime handle or workspace path is missing (e.g. spawn
// failed partway, handle lost after a crash) is still terminated — the destroy
// steps are skipped for whatever is absent, but the session record always
// moves to terminal state so it can be cleaned up from the dashboard.
func (m *Manager) Kill(ctx context.Context, id domain.SessionID) (bool, error) {
rec, ok, err := m.store.GetSession(ctx, id)
if err != nil {
return false, fmt.Errorf("kill %s: %w", id, err)
}
if !ok {
return false, nil // already gone: benign race
}
handle := runtimeHandle(rec.Metadata)
ws := workspaceInfo(rec)
// Always record terminal intent so the session is marked terminated even
// when the runtime/workspace handle is missing.
if err := m.lcm.MarkTerminated(ctx, id); err != nil {
return false, fmt.Errorf("kill %s: %w", id, err)
}
defer m.cleanupSystemPromptDir(id)
// Clear the restore marker so the next boot's RestoreAll cannot resurrect a
// killed session (#2319). Best-effort: teardown below still matters.
if err := m.store.DeleteSessionWorktrees(ctx, id); err != nil {
m.logger.Warn("kill: delete restore marker failed", "sessionID", id, "error", err)
}
// Only tear down what exists. A session may have lost its handle after a
// crash or never acquired one if spawn failed partway.
if handle.ID != "" {
if err := m.runtime.Destroy(ctx, handle); err != nil {
return false, fmt.Errorf("kill %s: runtime: %w", id, err)
}
}
freed := false
if ws.Path != "" {
if err := m.workspace.Destroy(ctx, ws); err != nil {
if errors.Is(err, ports.ErrWorkspaceDirty) {
return false, nil
}
return false, fmt.Errorf("kill %s: workspace: %w", id, err)
}
freed = true
}
return freed, nil
}
// RetireForReplacement terminates a live orchestrator and releases its branch
// for a replacement session. Unlike Kill, this captures uncommitted work before
// force-removing the worktree, so a dirty canonical orchestrator worktree does
// not block the replacement from claiming the canonical branch.
//
// This deliberately does not write a session_worktrees row: those rows are
// boot-restore markers, and a replaced orchestrator must stay terminated.
func (m *Manager) RetireForReplacement(ctx context.Context, id domain.SessionID) error {
rec, ok, err := m.store.GetSession(ctx, id)
if err != nil {
return fmt.Errorf("retire replacement %s: %w", id, err)
}
if !ok || rec.IsTerminated {
return nil
}
if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" {
if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil {
return fmt.Errorf("retire replacement %s: clear restore markers: %w", id, err)
}
handle := runtimeHandle(rec.Metadata)
if handle.ID != "" {
if err := m.runtime.Destroy(ctx, handle); err != nil {
return fmt.Errorf("retire replacement %s: runtime: %w", id, err)
}
}
if err := m.lcm.MarkTerminated(ctx, id); err != nil {
return fmt.Errorf("retire replacement %s: mark terminated: %w", id, err)
}
return nil
}
ws := workspaceInfo(rec)
if _, err := m.workspace.StashUncommitted(ctx, ws); err != nil {
return fmt.Errorf("retire replacement %s: stash: %w", id, err)
}
if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil {
return fmt.Errorf("retire replacement %s: clear restore markers: %w", id, err)
}
handle := runtimeHandle(rec.Metadata)
if handle.ID != "" {
if err := m.runtime.Destroy(ctx, handle); err != nil {
return fmt.Errorf("retire replacement %s: runtime: %w", id, err)
}
}
if err := m.workspace.ForceDestroy(ctx, ws); err != nil {
return fmt.Errorf("retire replacement %s: force destroy: %w", id, err)
}
if err := m.lcm.MarkTerminated(ctx, rec.ID); err != nil {
return fmt.Errorf("retire replacement %s: mark terminated: %w", id, err)
}
return nil
}
// Restore relaunches a torn-down session in its workspace. The fallible I/O runs
// before any durable session write, so a failure never resurrects the row or destroys
// the worktree (it may hold the agent's prior work).
func (m *Manager) Restore(ctx context.Context, id domain.SessionID) (domain.SessionRecord, error) {
rec, ok, err := m.store.GetSession(ctx, id)
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
}
if !ok {
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, ErrNotFound)
}
if !rec.IsTerminated {
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, ErrNotRestorable)
}
meta := rec.Metadata
// Mirror Kill's incomplete-handle guard: a session whose spawn failed before
// the workspace landed has neither WorkspacePath nor Branch, and there is
// nothing meaningful to restore from. Surface this as a typed 409 instead of
// letting workspace.Restore fail with an opaque wrapped error.
if meta.WorkspacePath == "" || meta.Branch == "" {
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, ErrIncompleteHandle)
}
// Resumability is decided inside restoreArgv, not here. A promptless session
// can still be fully resumable when the harness pins a deterministic session id
// (Claude Code). restoreArgv returns ErrNotResumable only for a promptless,
// unresumable non-orchestrator (a worker with no task and no native id to resume).
// Orchestrators always relaunch fresh with the system prompt only.
project, err := m.loadProject(ctx, rec.ProjectID)
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
}
ws, err := m.workspace.Restore(ctx, ports.WorkspaceConfig{
ProjectID: rec.ProjectID,
SessionID: id,
Kind: rec.Kind,
SessionPrefix: sessionPrefix(project),
Branch: meta.Branch,
})
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("restore %s: workspace: %w", id, err)
}
agent, ok := m.agents.Agent(rec.Harness)
if !ok {
return domain.SessionRecord{}, fmt.Errorf("restore %s: no agent adapter for harness %q", id, rec.Harness)
}
if err := m.prepareWorkspace(ctx, agent, id, ws.Path); err != nil {
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
}
// The system prompt is derived, not persisted: recompute it so a restored
// session keeps its standing instructions across the relaunch.
systemPrompt, err := m.buildSystemPrompt(ctx, rec.Kind, rec.ProjectID)
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("restore %s: system prompt: %w", id, err)
}
systemPromptFile, err := m.prepareSystemPromptFile(id, rec.Harness, systemPrompt)
if err != nil {
m.cleanupSystemPromptDir(id)
return domain.SessionRecord{}, fmt.Errorf("restore %s: system prompt file: %w", id, err)
}
// Restore re-applies the project's resolved agent config so a configured
// model/permissions carry across a restore, matching fresh spawn.
argv, err := restoreArgv(ctx, agent, id, ws.Path, meta, systemPrompt, systemPromptFile, effectiveAgentConfig(rec.Kind, project.Config), rec.Kind)
if err != nil {
m.cleanupSystemPromptDir(id)
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
}
handle, err := m.runtime.Create(ctx, ports.RuntimeConfig{
SessionID: id,
WorkspacePath: ws.Path,
Argv: argv,
Env: m.runtimeEnv(id, rec.ProjectID, rec.IssueID, project.Config.Env),
})
if err != nil {
m.cleanupSystemPromptDir(id)
return domain.SessionRecord{}, fmt.Errorf("restore %s: runtime: %w", id, err)
}
metadata := domain.SessionMetadata{Branch: ws.Branch, WorkspacePath: ws.Path, RuntimeHandleID: handle.ID, AgentSessionID: meta.AgentSessionID, Prompt: meta.Prompt}
if err := m.lcm.MarkSpawned(ctx, id, metadata); err != nil {
_ = m.runtime.Destroy(ctx, handle)
m.cleanupSystemPromptDir(id)
return domain.SessionRecord{}, fmt.Errorf("restore %s: completed: %w", id, err)
}
return m.getRecord(ctx, id)
}
func (m *Manager) getRecord(ctx context.Context, id domain.SessionID) (domain.SessionRecord, error) {
rec, ok, err := m.store.GetSession(ctx, id)
if err != nil {
return domain.SessionRecord{}, fmt.Errorf("get %s: %w", id, err)
}
if !ok {
return domain.SessionRecord{}, fmt.Errorf("get %s: %w", id, ErrNotFound)
}
return rec, nil
}
// SaveAndTeardownAll captures uncommitted work and tears down every live
// session that has a workspace path. It is the shutdown path for the daemon:
// each session's uncommitted work is stashed into a preserve ref, the ref is
// written to session_worktrees (the "shutdown-saved" marker) BEFORE the
// worktree is force-removed. The DB write is committed before the worktree is
// destroyed so a crash between the two leaves the ref in place and the row
// present; RestoreAll will replay both.
//
// Failures on individual sessions are logged and do not abort the loop.
// ForceDestroy is never called if capture or the DB write did not succeed.
func (m *Manager) SaveAndTeardownAll(ctx context.Context) error {
recs, err := m.store.ListAllSessions(ctx)
if err != nil {
return fmt.Errorf("save-teardown-all: list sessions: %w", err)
}
for _, rec := range recs {
if rec.IsTerminated {
continue
}
if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" {
continue
}
if err := m.saveAndTeardownOne(ctx, rec); err != nil {
m.logger.Error("save-teardown-all: session failed, skipping", "sessionID", rec.ID, "error", err)
}
}
return nil
}
// saveAndTeardownOne runs the capture-then-destroy sequence for a single
// session. The DB write (UpsertSessionWorktree) is committed before
// ForceDestroy; if either capture or the DB write fails, ForceDestroy is
// not called.
func (m *Manager) saveAndTeardownOne(ctx context.Context, rec domain.SessionRecord) error {
ws := workspaceInfo(rec)
// 1. Capture uncommitted work (ref may be "" for clean worktrees).
ref, err := m.workspace.StashUncommitted(ctx, ws)
if err != nil {
return fmt.Errorf("save %s: stash: %w", rec.ID, err)
}
// 2. Write the shutdown-saved marker to the DB. The row's presence (even
// with an empty preserved_ref) is what RestoreAll uses to identify sessions
// saved by this run. This MUST be committed before ForceDestroy.
row := domain.SessionWorktreeRecord{
SessionID: rec.ID,
RepoName: domain.RootWorkspaceRepoName,
Branch: rec.Metadata.Branch,
WorktreePath: rec.Metadata.WorkspacePath,
PreservedRef: ref,
}
if err := m.store.UpsertSessionWorktree(ctx, row); err != nil {
return fmt.Errorf("save %s: upsert worktree row: %w", rec.ID, err)
}
// 3. Mark terminal via the LCM (same path Kill uses).
if err := m.lcm.MarkTerminated(ctx, rec.ID); err != nil {
return fmt.Errorf("save %s: mark terminated: %w", rec.ID, err)
}
// 4. Runtime teardown (best-effort; same pattern as Kill).
handle := runtimeHandle(rec.Metadata)
if handle.ID != "" {
if err := m.runtime.Destroy(ctx, handle); err != nil {
m.logger.Warn("save-teardown-all: runtime destroy failed", "sessionID", rec.ID, "error", err)
}
}
// 5. Force-remove the worktree (safe: work is captured in step 1 and the
// DB write in step 2 is already committed).
if err := m.workspace.ForceDestroy(ctx, ws); err != nil {
m.logger.Warn("save-teardown-all: force destroy failed", "sessionID", rec.ID, "error", err)
}
return nil
}
// reconcileLive handles a single non-terminated session on boot. If its runtime
// session is still alive (tmux is the persistence layer, so it survives a daemon
// crash) we adopt it: a no-op, the agent keeps running. If the runtime is gone,
// the agent died with the daemon, so we save-and-tear-down to the SAME end state
// a graceful shutdown produces: capture uncommitted work into a preserve ref,
// record the session_worktrees restore marker, mark terminated, and remove the
// worktree. RestoreAll (which Reconcile runs immediately after) then relaunches
// it on this same boot, resuming history. Crash recovery thus matches graceful
// restart instead of silently abandoning the session.
//
// If the work capture fails we mark terminated WITHOUT a marker and leave the
// worktree intact: better to skip the relaunch than to tear down un-preserved
// work or relaunch onto an inconsistent worktree.
func (m *Manager) reconcileLive(ctx context.Context, rec domain.SessionRecord) error {
if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" {
return nil
}
handle := runtimeHandle(rec.Metadata)
if handle.ID != "" {
alive, err := m.runtime.IsAlive(ctx, handle)
if err != nil {
// A failed probe is not proof of death: leave the session as-is.
return fmt.Errorf("reconcile %s: probe: %w", rec.ID, err)
}
if alive {
return nil // adopt: the session survived the crash.
}
}
// Runtime is gone: capture uncommitted work first.
ws := workspaceInfo(rec)
ref, err := m.workspace.StashUncommitted(ctx, ws)
if err != nil {
// Could not capture work: do NOT write a restore marker or tear down the
// worktree (that would risk losing un-preserved work). Mark terminated so
// a dead session is not left looking live; the worktree stays put.
m.logger.Warn("reconcile: stash uncommitted failed; terminating without restore marker", "sessionID", rec.ID, "error", err)
if mErr := m.lcm.MarkTerminated(ctx, rec.ID); mErr != nil {
return fmt.Errorf("reconcile %s: mark terminated: %w", rec.ID, mErr)
}
return nil
}
// Work captured. Record the shutdown-saved marker BEFORE tearing down the
// worktree, mirroring saveAndTeardownOne, so RestoreAll relaunches it.
row := domain.SessionWorktreeRecord{
SessionID: rec.ID,
RepoName: domain.RootWorkspaceRepoName,
Branch: rec.Metadata.Branch,
WorktreePath: rec.Metadata.WorkspacePath,
PreservedRef: ref,
}
if err := m.store.UpsertSessionWorktree(ctx, row); err != nil {
return fmt.Errorf("reconcile %s: upsert worktree marker: %w", rec.ID, err)
}
if err := m.lcm.MarkTerminated(ctx, rec.ID); err != nil {
return fmt.Errorf("reconcile %s: mark terminated: %w", rec.ID, err)
}
// Remove the worktree (work is captured in the ref): RestoreAll re-creates it
// clean and replays the ref. The dead runtime needs no Destroy.
if err := m.workspace.ForceDestroy(ctx, ws); err != nil {
m.logger.Warn("reconcile: force destroy failed after marker", "sessionID", rec.ID, "error", err)
}
return nil
}
// reconcileReap kills the leaked tmux session of a session the DB already marks
// terminated. This covers the teardown that marked the row terminated but failed
// to kill the runtime (e.g. ForceDestroy/Destroy errored after MarkTerminated).
// Destroy is idempotent, so an already-gone session is a no-op.
func (m *Manager) reconcileReap(ctx context.Context, rec domain.SessionRecord) error {
handle := runtimeHandle(rec.Metadata)
if handle.ID == "" {
return nil
}
alive, err := m.runtime.IsAlive(ctx, handle)
if err != nil {
return fmt.Errorf("reconcile reap %s: probe: %w", rec.ID, err)
}
if !alive {
return nil
}
if err := m.runtime.Destroy(ctx, handle); err != nil {
return fmt.Errorf("reconcile reap %s: destroy: %w", rec.ID, err)
}
return nil
}
// Reconcile is the boot-time consistency pass. It replaces the bare RestoreAll
// call so that however the previous daemon died (clean shutdown, SIGKILL, or
// crash), live reality matches the DB:
//
// 1. Live pass: for each non-terminated session, adopt it if its runtime
// survived, else capture work and mark terminated (reconcileLive).
// 2. Reap pass: for each terminated session whose runtime leaked, kill it
// (reconcileReap). Runs before restore so a restored session does not
// collide with a leaked tmux of the same name.
// 3. Restore pass: relaunch shutdown-saved sessions (existing RestoreAll).
//
// Best-effort throughout: a per-session failure is logged and never aborts the
// pass or blocks boot.
func (m *Manager) Reconcile(ctx context.Context) error {
recs, err := m.store.ListAllSessions(ctx)
if err != nil {
return fmt.Errorf("reconcile: list sessions: %w", err)
}
for _, rec := range recs {
if rec.IsTerminated {
continue
}
if err := m.reconcileLive(ctx, rec); err != nil {
m.logger.Error("reconcile: live pass failed, skipping", "sessionID", rec.ID, "error", err)
}
}
for _, rec := range recs {
if !rec.IsTerminated {
continue
}
if err := m.reconcileReap(ctx, rec); err != nil {
m.logger.Error("reconcile: reap pass failed, skipping", "sessionID", rec.ID, "error", err)
}
}
return m.RestoreAll(ctx)
}
// RestoreAll relaunches every terminated session that was saved by the last
// SaveAndTeardownAll. The "shutdown-saved" marker is the presence of a
// session_worktrees row for the session; sessions the user killed before
// shutdown have no such row and are left terminated.
//
// For each saved session:
// 1. Ensure the worktree exists via workspace.Restore.
// 2. If a preserve ref is recorded, replay it via ApplyPreserved; on conflict
// log and continue (still relaunch the agent, never delete the ref).
// 3. Relaunch via the existing Restore method.
//
// Failures on individual sessions are logged and do not abort the loop.
func (m *Manager) RestoreAll(ctx context.Context) error {
recs, err := m.store.ListAllSessions(ctx)
if err != nil {
return fmt.Errorf("restore-all: list sessions: %w", err)
}
for _, rec := range recs {
if !rec.IsTerminated {
continue
}
// Check the shutdown-saved marker: is there a session_worktrees row?
rows, err := m.store.ListSessionWorktrees(ctx, rec.ID)
if err != nil {
m.logger.Error("restore-all: list worktrees failed", "sessionID", rec.ID, "error", err)
continue
}
if len(rows) == 0 {
// No marker: this session was killed by the user before shutdown.
continue
}
// Collect the preserve ref (may be "" for clean worktrees).
var preserveRef string
for _, r := range rows {
if r.PreservedRef != "" {
preserveRef = r.PreservedRef
break
}
}
// Step 1: ensure the worktree exists. workspace.Restore re-creates it
// if it was removed by SaveAndTeardownAll.
project, err := m.loadProject(ctx, rec.ProjectID)
if err != nil {
m.logger.Error("restore-all: load project failed", "sessionID", rec.ID, "error", err)
continue
}
ws, err := m.workspace.Restore(ctx, ports.WorkspaceConfig{
ProjectID: rec.ProjectID,
SessionID: rec.ID,
Kind: rec.Kind,
SessionPrefix: sessionPrefix(project),
Branch: rec.Metadata.Branch,
})
if err != nil {
m.logger.Error("restore-all: workspace restore failed", "sessionID", rec.ID, "error", err)
continue
}
// Step 2: replay preserve ref when one was recorded.
if preserveRef != "" {
if applyErr := m.workspace.ApplyPreserved(ctx, ws, preserveRef); applyErr != nil {
if errors.Is(applyErr, ports.ErrPreservedConflict) {
m.logger.Warn("restore-all: apply preserved produced conflicts; agent relaunched with conflict markers in place",
"sessionID", rec.ID, "ref", preserveRef, "error", applyErr)
} else {
m.logger.Error("restore-all: apply preserved failed", "sessionID", rec.ID, "error", applyErr)
}
// Continue: always relaunch even on conflict (never delete the ref here).
}
}
// Step 3: relaunch via the existing single-session Restore method.
if _, err := m.Restore(ctx, rec.ID); err != nil {
// A promptless, unresumable worker is intentionally left terminated
// (ErrNotResumable): expected, not an operational failure, so log it
// quietly rather than as an error.
if errors.Is(err, ErrNotResumable) {
m.logger.Warn("restore-all: session left terminated (nothing to resume)", "sessionID", rec.ID)
} else {
m.logger.Error("restore-all: relaunch failed", "sessionID", rec.ID, "error", err)
}
continue
}
if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil {
m.logger.Error("restore-all: delete consumed worktree marker failed", "sessionID", rec.ID, "error", err)
}
}
return nil
}
// Send delivers a message to a running session's agent via the messenger.
func (m *Manager) Send(ctx context.Context, id domain.SessionID, message string) error {
if err := m.messenger.Send(ctx, id, message); err != nil {
return fmt.Errorf("send %s: %w", id, err)
}
return nil
}
// CleanupSkip reports one terminal session whose workspace was preserved
// rather than reclaimed, and why.
type CleanupSkip struct {
SessionID domain.SessionID
Reason string
}
// CleanupResult reports what Cleanup reclaimed and what it preserved.
type CleanupResult struct {
Cleaned []domain.SessionID
Skipped []CleanupSkip
}
// Cleanup reclaims the workspaces of terminal sessions in a project. A workspace
// whose teardown is refused (uncommitted work) is never forced; it is reported
// in Skipped with the reason so the refusal is visible instead of silent.
func (m *Manager) Cleanup(ctx context.Context, project domain.ProjectID) (CleanupResult, error) {
recs, err := m.cleanupRecords(ctx, project)
if err != nil {
return CleanupResult{}, fmt.Errorf("cleanup %s: %w", project, err)
}
result := CleanupResult{Cleaned: make([]domain.SessionID, 0, len(recs)), Skipped: []CleanupSkip{}}
for _, rec := range recs {
if !rec.IsTerminated {
continue
}
ws := workspaceInfo(rec)
if ws.Path == "" {
m.cleanupSystemPromptDir(rec.ID)
continue
}
if h := runtimeHandle(rec.Metadata); h.ID != "" {
_ = m.runtime.Destroy(ctx, h) // best effort; usually already gone
}
if err := m.workspace.Destroy(ctx, ws); err != nil {
if !errors.Is(err, ports.ErrWorkspaceDirty) {
// The public reason stays a fixed string (the raw error carries
// internal filesystem paths); the full cause lands here.
m.logger.Warn("cleanup: workspace teardown failed", "sessionID", rec.ID, "path", ws.Path, "error", err)
}
result.Skipped = append(result.Skipped, CleanupSkip{SessionID: rec.ID, Reason: cleanupSkipReason(err)})
continue
}
m.cleanupSystemPromptDir(rec.ID)
result.Cleaned = append(result.Cleaned, rec.ID)
}
return result, nil
}
// cleanupSkipReason renders a workspace teardown refusal as a short
// user-facing reason for the cleanup report. Deliberately not the raw error:
// it flows to the API response and CLI output, and teardown errors embed
// internal filesystem paths.
func cleanupSkipReason(err error) string {
if errors.Is(err, ports.ErrWorkspaceDirty) {
return "workspace has uncommitted changes"
}
return "workspace teardown failed"
}
func (m *Manager) cleanupRecords(ctx context.Context, project domain.ProjectID) ([]domain.SessionRecord, error) {
if project == "" {
return m.store.ListAllSessions(ctx)
}
return m.store.ListSessions(ctx, project)
}
// ---- helpers ----
func seedRecord(cfg ports.SpawnConfig, now time.Time) domain.SessionRecord {
return domain.SessionRecord{
ProjectID: cfg.ProjectID,
IssueID: cfg.IssueID,
Kind: cfg.Kind,
CreatedAt: now,
UpdatedAt: now,
Harness: cfg.Harness,
DisplayName: cfg.DisplayName,
Activity: domain.Activity{State: domain.ActivityIdle, LastActivityAt: now},
}
}
func defaultSessionBranch(id domain.SessionID, kind domain.SessionKind, prefix string) string {
if kind == domain.KindOrchestrator {
return "ao/" + prefix + "-orchestrator"
}
// A fresh, unique branch per worker session: gitworktree can't add a worktree
// on a branch already checked out elsewhere (e.g. main). Put the root work
// branch under a session namespace so sibling PR branches such as
// ao/<session>/<topic> remain valid Git refs.
return "ao/" + string(id) + "/root"
}
func buildPrompt(cfg ports.SpawnConfig) string {
return buildTaskPrompt(taskPromptConfig{
Role: promptRoleForKind(cfg.Kind),
Prompt: cfg.Prompt,
IssueID: string(cfg.IssueID),
IssueContext: cfg.IssueContext,
})
}
func promptRoleForKind(kind domain.SessionKind) sessionPromptRole {
switch kind {
case domain.KindOrchestrator:
return sessionPromptRoleOrchestrator
case domain.KindWorker:
return sessionPromptRoleWorker
default:
return ""
}
}
func promptProjectContext(projectID domain.ProjectID, project domain.ProjectRecord) promptProject {
cfg := project.Config.WithDefaults()
id := project.ID
if strings.TrimSpace(id) == "" {
id = string(projectID)
}
return promptProject{
ID: id,
Name: project.DisplayName,
Repo: project.RepoOriginURL,
DefaultBranch: cfg.DefaultBranch,
Path: project.Path,
}
}
// buildSpawnTexts returns the user-facing prompt and the system prompt to
// deliver separately to the agent. Orchestrator role instructions and worker
// coordination hints are placed in the system prompt so they are treated as
// standing instructions rather than part of the human's task request. A
// promptless spawn delivers no user prompt at all: the agent simply lands at an
// empty input box rather than receiving an auto-generated kickoff turn.
func (m *Manager) buildSpawnTexts(ctx context.Context, cfg ports.SpawnConfig) (prompt, systemPrompt string, err error) {
prompt = buildPrompt(cfg)
systemPrompt, err = m.buildSystemPrompt(ctx, cfg.Kind, cfg.ProjectID)
if err != nil {
return "", "", err
}
return prompt, systemPrompt, nil
}
// buildSystemPrompt derives the standing instructions for a session of the
// given kind from current store state. Restore recomputes them through here
// rather than persisting them, so a restored worker points at the orchestrator
// that is active now, not the one from its original spawn.
func (m *Manager) buildSystemPrompt(ctx context.Context, kind domain.SessionKind, projectID domain.ProjectID) (string, error) {
project, err := m.loadProject(ctx, projectID)
if err != nil {
return "", err
}
role := promptRoleForKind(kind)
cfg := systemPromptConfig{
Role: role,
Project: promptProjectContext(projectID, project),
}
switch kind {
case domain.KindOrchestrator:
cfg.OrchestratorRules = project.Config.OrchestratorRules
case domain.KindWorker:
orchestratorID, ok, err := m.activeOrchestratorSessionID(ctx, projectID)
if err != nil {
return "", err
}
if ok {
cfg.OrchestratorSessionID = string(orchestratorID)
}
rules, err := buildProjectRules(projectRulesConfig{
ProjectPath: project.Path,
AgentRules: project.Config.AgentRules,
AgentRulesFile: project.Config.AgentRulesFile,
})
if err != nil {
return "", err
}
cfg.ProjectRules = rules
}
cfg.AOSkillPointer = m.aoSkillPointer()
return buildSystemPromptText(cfg), nil
}
// aoSkillPointer is appended to every agent system prompt. It points the agent
// at the using-ao skill the daemon installs under the data dir, rather than
// inlining the whole CLI catalog. The path is absolute so it resolves from any
// project's worktree, not just the AO repo (the only place a repo-relative
// skills/ path would exist). The skill file carries exact flags and examples,
// so the standing prompt stays a short pointer rather than a command dump.
func (m *Manager) aoSkillPointer() string {
dir := skillassets.Dir(m.dataDir)
skillFile := filepath.Join(dir, "SKILL.md")
commandsGlob := filepath.Join(dir, "commands", "*.md")
return "## Using the ao CLI\n\n" +
"When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples."
}
func (m *Manager) activeOrchestratorSessionID(ctx context.Context, project domain.ProjectID) (domain.SessionID, bool, error) {
recs, err := m.store.ListSessions(ctx, project)
if err != nil {
return "", false, fmt.Errorf("list sessions for %s: %w", project, err)
}
for _, rec := range recs {
if rec.Kind == domain.KindOrchestrator && !rec.IsTerminated {
return rec.ID, true, nil
}
}
return "", false, nil
}
func (m *Manager) writeSystemPromptFile(id domain.SessionID, systemPrompt string) (string, error) {
if systemPrompt == "" || strings.TrimSpace(m.dataDir) == "" {
return "", nil
}
path := filepath.Join(m.systemPromptDir(id), "system.md")
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return "", err
}
if err := os.WriteFile(path, []byte(strings.TrimRight(systemPrompt, "\n")+"\n"), 0o600); err != nil {
return "", err
}
return path, nil
}
func (m *Manager) prepareSystemPromptFile(id domain.SessionID, harness domain.AgentHarness, systemPrompt string) (string, error) {
path, err := m.writeSystemPromptFile(id, systemPrompt)
if err == nil || path != "" {
return path, err
}
if systemPromptFileRequired(harness) {
return "", err
}
m.logger.Warn("system prompt file unavailable; falling back to inline system prompt", "session", id, "harness", harness, "err", err)
return "", nil
}
func systemPromptFileRequired(harness domain.AgentHarness) bool {
switch harness {
case domain.HarnessAider,
domain.HarnessAuggie,
domain.HarnessOpenCode,
domain.HarnessCopilot:
return true
default:
return false
}
}
func (m *Manager) systemPromptDir(id domain.SessionID) string {
if strings.TrimSpace(m.dataDir) == "" {
return ""
}
return filepath.Join(m.dataDir, "prompts", string(id))
}
func (m *Manager) cleanupSystemPromptDir(id domain.SessionID) {
dir := m.systemPromptDir(id)
if dir == "" {
return
}
if err := os.RemoveAll(dir); err != nil {
m.logger.Warn("system prompt cleanup failed", "session", id, "path", dir, "err", err)
}
}
// spawnEnv builds the runtime environment: the per-project env vars first, then
// the AO-internal vars last so they always win (a project cannot override
// AO_SESSION_ID and friends).
func spawnEnv(id domain.SessionID, project domain.ProjectID, issue domain.IssueID, dataDir string, projectEnv map[string]string) map[string]string {
env := make(map[string]string, len(projectEnv)+4)
for k, v := range projectEnv {
env[k] = v
}
env[EnvSessionID] = string(id)
env[EnvProjectID] = string(project)
env[EnvIssueID] = string(issue)
env[EnvDataDir] = dataDir
return env
}
// runtimeEnv is spawnEnv plus the hook PATH pin: the session's PATH puts the
// running daemon's own directory first, so the bare `ao` in workspace hook
// commands resolves to the daemon that installed them rather than whatever
// `ao` is first on the inherited PATH (e.g. a legacy CLI without the hooks
// command, which fails every callback and silently kills activity tracking).
// When the pin cannot be applied the inherited PATH is kept and a warning is
// logged so the degradation isn't silent.
func (m *Manager) runtimeEnv(id domain.SessionID, project domain.ProjectID, issue domain.IssueID, projectEnv map[string]string) map[string]string {
env := spawnEnv(id, project, issue, m.dataDir, projectEnv)
path, err := HookPATH(m.executable, os.Getenv, projectEnv)
if err != nil {
m.logger.Warn("session PATH not pinned to the daemon binary; `ao hooks` callbacks may resolve to a different ao and activity tracking will stall",
"session", id, "error", err)
return env
}
env["PATH"] = path
return env
}
// HookPATH builds the PATH value pinned into a spawned session: the daemon
// executable's directory prepended to the base PATH (the project's PATH
// override when set, else the daemon's inherited PATH — matching what the
// runtime would have exported anyway). An error means the pin cannot be
// applied: the executable is unresolvable, or is not named "ao", in which case
// prepending its directory would not change what `ao` resolves to. Exported so
// the reviewer launcher can pin its pane's PATH the same way.
func HookPATH(executable func() (string, error), getenv func(string) string, projectEnv map[string]string) (string, error) {
exe, err := executable()
if err != nil {
return "", fmt.Errorf("resolve daemon executable: %w", err)
}
name := filepath.Base(exe)
if runtime.GOOS == "windows" {
name = strings.TrimSuffix(strings.ToLower(name), ".exe")
}
if name != hookBinaryName {
return "", fmt.Errorf("daemon executable %s is not named %q", exe, hookBinaryName)
}
base := projectEnv["PATH"]
if base == "" {
base = getenv("PATH")
}
dir := filepath.Dir(exe)
if base == "" {
return dir, nil
}
return dir + string(os.PathListSeparator) + base, nil
}
// provisionWorkspace applies the project's per-workspace setup after the
// worktree exists: symlink shared files from the project repo, then run any
// post-create commands. Either failing aborts the spawn so a half-provisioned
// workspace never launches an agent.
func (m *Manager) provisionWorkspace(ctx context.Context, project domain.ProjectRecord, workspacePath string) error {
if err := applySymlinks(project.Path, workspacePath, project.Config.Symlinks); err != nil {
return err
}
return runPostCreate(ctx, workspacePath, project.Config.PostCreate)
}
// applySymlinks links each repo-relative path into the workspace. A source that
// does not exist is skipped (symlinks are a convenience for optional files like
// .env); a real link failure aborts. Paths must be repo-relative with no
// parent traversal (no leading "/", no ".." segment) — a bad path is refused
// up front so a project config cannot escape the project or workspace tree.
func applySymlinks(projectPath, workspacePath string, symlinks []string) error {
for _, rel := range symlinks {
rel = strings.TrimSpace(rel)
if rel == "" {
continue
}
clean, err := safeRelPath(rel)
if err != nil {
return fmt.Errorf("symlink %q: %w", rel, err)
}
source := filepath.Join(projectPath, clean)
if _, err := os.Stat(source); err != nil {
continue
}
target := filepath.Join(workspacePath, clean)
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
return fmt.Errorf("symlink %q: %w", rel, err)
}
if _, err := os.Lstat(target); err == nil {
continue
}
if err := os.Symlink(source, target); err != nil {
return fmt.Errorf("symlink %q: %w", rel, err)
}
}
return nil
}
// safeRelPath confines rel to a repo-relative path: no absolute paths and no
// ".." segments (before or after Clean). The cleaned form is returned so
// callers join it against project/workspace roots safely.
func safeRelPath(rel string) (string, error) {
if filepath.IsAbs(rel) || strings.HasPrefix(rel, "/") || strings.HasPrefix(rel, `\`) {
return "", fmt.Errorf("path must be repo-relative")
}
clean := filepath.Clean(rel)
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || clean == "." || clean == "" {
return "", fmt.Errorf("path must be repo-relative")
}
for _, seg := range strings.Split(filepath.ToSlash(clean), "/") {
if seg == ".." {
return "", fmt.Errorf("path must be repo-relative")
}
}
return clean, nil
}
// runPostCreate runs each post-create command in the workspace via the platform
// shell, so OS-agnostic commands like "pnpm install" work. A non-zero exit
// aborts the spawn with the command output.
func runPostCreate(ctx context.Context, workspacePath string, commands []string) error {
for _, command := range commands {
command = strings.TrimSpace(command)
if command == "" {
continue
}
var cmd *exec.Cmd
if runtime.GOOS == "windows" {
cmd = aoprocess.CommandContext(ctx, "cmd", "/c", command)
} else {
cmd = aoprocess.CommandContext(ctx, "sh", "-c", command)
}
cmd.Dir = workspacePath
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("postCreate %q: %w: %s", command, err, strings.TrimSpace(string(out)))
}
}
return nil
}
// preLauncher is an optional Agent capability: a step the manager runs before
// launch. Claude Code implements it to record workspace trust in ~/.claude.json
// so its interactive "do you trust this folder?" dialog can't block the headless
// pane. Adapters that don't need it simply omit the method.
type preLauncher interface {
PreLaunch(ctx context.Context, cfg ports.LaunchConfig) error
}
// prepareWorkspace runs the per-session pre-launch steps before the runtime
// starts the agent: installing the workspace-local activity hooks (so early
// startup hooks can update the already-created session row), then any optional
// PreLaunch step. Shared by Spawn and Restore.
func (m *Manager) prepareWorkspace(ctx context.Context, agent ports.Agent, id domain.SessionID, workspacePath string) error {
if err := agent.GetAgentHooks(ctx, ports.WorkspaceHookConfig{
SessionID: string(id),
WorkspacePath: workspacePath,
DataDir: m.dataDir,
}); err != nil {
return fmt.Errorf("install hooks: %w", err)
}
if pl, ok := agent.(preLauncher); ok {
if err := pl.PreLaunch(ctx, ports.LaunchConfig{SessionID: string(id), WorkspacePath: workspacePath}); err != nil {
return fmt.Errorf("pre-launch: %w", err)
}
}
return nil
}
// restoreArgv builds the argv to relaunch a torn-down session: the agent's
// native resume command when it can continue the session, else a fresh launch.
// The agent signals via ok=false (e.g. no native session id captured yet).
// Returns ErrNotResumable only for a promptless, unresumable non-orchestrator:
// a worker with no prompt and no native session id has nothing to restore from.
// Orchestrators are promptless by design and always relaunch fresh with the
// system prompt only.
func restoreArgv(ctx context.Context, agent ports.Agent, id domain.SessionID, workspacePath string, meta domain.SessionMetadata, systemPrompt, systemPromptFile string, agentConfig ports.AgentConfig, kind domain.SessionKind) ([]string, error) {
ref := ports.SessionRef{
ID: string(id),
WorkspacePath: workspacePath,
Metadata: map[string]string{ports.MetadataKeyAgentSessionID: meta.AgentSessionID},
}
cmd, ok, err := agent.GetRestoreCommand(ctx, ports.RestoreConfig{Session: ref, SystemPrompt: systemPrompt, SystemPromptFile: systemPromptFile, Config: agentConfig, Permissions: agentConfig.Permissions})
if err != nil {
return nil, fmt.Errorf("restore command: %w", err)
}
if ok {
return cmd, nil
}
// Adapter cannot resume. A saved prompt is replayed fresh. An orchestrator is
// promptless by design and relaunches with the system prompt only. A promptless
// WORKER has no task and no session id to restore from: do not blank-relaunch it.
if meta.Prompt == "" && kind != domain.KindOrchestrator {
return nil, ErrNotResumable
}
// Fall through to GetLaunchCommand (replays meta.Prompt; empty for an orchestrator).
argv, err := agent.GetLaunchCommand(ctx, ports.LaunchConfig{
SessionID: string(id),
WorkspacePath: workspacePath,
Prompt: meta.Prompt,
SystemPrompt: systemPrompt,
SystemPromptFile: systemPromptFile,
Config: agentConfig,
Permissions: agentConfig.Permissions,
})
if err != nil {
return nil, fmt.Errorf("launch command: %w", err)
}
return argv, nil
}
// validateAgentBinary checks that argv[0] resolves via the manager's
// lookPath (exec.LookPath in prod) before any runtime work happens. Adapters
// that can't resolve their binary now return ports.ErrAgentBinaryNotFound from
// GetLaunchCommand directly; this guard is a defense-in-depth for adapters
// that return an argv[0] like "claude" without verifying. Some adapters prefix
// their command with `env KEY=value`; in that case validate the first real
// executable after the environment assignments.
func (m *Manager) validateAgentBinary(argv []string) error {
if len(argv) == 0 {
return fmt.Errorf("agent: empty launch argv: %w", ports.ErrAgentBinaryNotFound)
}
bin, ok := launchBinary(argv)
if !ok {
return fmt.Errorf("agent: launch argv missing binary: %w", ports.ErrAgentBinaryNotFound)
}
if _, err := m.lookPath(bin); err != nil {
return fmt.Errorf("agent binary %q: %w", bin, ports.ErrAgentBinaryNotFound)
}
return nil
}
func launchBinary(argv []string) (string, bool) {
if len(argv) == 0 {
return "", false
}
if filepath.Base(argv[0]) != "env" {
return argv[0], true
}
for _, arg := range argv[1:] {
if strings.Contains(arg, "=") {
continue
}
return arg, true
}
return "", false
}
func (m *Manager) validateRuntimePrerequisites() error {
if runtime.GOOS == "windows" {
return nil
}
if path, err := m.lookPath("tmux"); err != nil || path == "" {
return fmt.Errorf("%w: tmux required on macOS/Linux but not in PATH", ports.ErrRuntimePrerequisite)
}
return nil
}
func runtimeHandle(meta domain.SessionMetadata) ports.RuntimeHandle {
return ports.RuntimeHandle{ID: meta.RuntimeHandleID}
}
func workspaceInfo(rec domain.SessionRecord) ports.WorkspaceInfo {
return ports.WorkspaceInfo{
Path: rec.Metadata.WorkspacePath,
Branch: rec.Metadata.Branch,
SessionID: rec.ID,
ProjectID: rec.ProjectID,
}
}