1485 lines
58 KiB
Go
1485 lines
58 KiB
Go
// Package sessionmanager drives internal session command operations over runtime,
|
|
// agent, workspace, storage, messenger, and lifecycle dependencies.
|
|
package sessionmanager
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aoagents/agent-orchestrator/backend/internal/domain"
|
|
"github.com/aoagents/agent-orchestrator/backend/internal/ports"
|
|
aoprocess "github.com/aoagents/agent-orchestrator/backend/internal/process"
|
|
"github.com/aoagents/agent-orchestrator/backend/internal/skillassets"
|
|
)
|
|
|
|
// Sentinel errors returned by the Session Manager; callers match them with
|
|
// errors.Is.
|
|
var (
|
|
ErrNotFound = errors.New("session: not found")
|
|
ErrNotRestorable = errors.New("session: not restorable (not terminal)")
|
|
ErrTerminated = errors.New("session: terminated")
|
|
ErrIncompleteHandle = errors.New("session: incomplete teardown handle")
|
|
// ErrProjectNotResolvable means the spawn's project has no usable repo
|
|
// (unregistered, archived, or missing a path). The API maps it to a 400.
|
|
ErrProjectNotResolvable = errors.New("session: project repo not resolvable")
|
|
// ErrUnknownHarness means the requested agent harness has no registered
|
|
// adapter. The API maps it to a 400 so a typo'd `--harness` is a validation
|
|
// error, not an opaque 500.
|
|
ErrUnknownHarness = errors.New("session: unknown agent harness")
|
|
// ErrMissingHarness means neither the spawn request nor the project's role
|
|
// config selected an agent. Worker/orchestrator spawns must be explicit.
|
|
ErrMissingHarness = errors.New("session: agent harness required")
|
|
// ErrNotResumable means a terminated session cannot be relaunched: its adapter
|
|
// cannot natively resume it AND it has no prompt to fresh-launch from, and it is
|
|
// not an orchestrator (orchestrators are promptless by design and relaunch fresh
|
|
// with the system prompt only). Workers without a task and without a native
|
|
// session id have nothing meaningful to restore.
|
|
ErrNotResumable = errors.New("session: nothing to resume from")
|
|
)
|
|
|
|
// Env vars a spawned process reads to learn who it is.
|
|
const (
|
|
EnvSessionID = "AO_SESSION_ID"
|
|
EnvProjectID = "AO_PROJECT_ID"
|
|
EnvIssueID = "AO_ISSUE_ID"
|
|
// EnvDataDir tells a spawned agent's AO hook commands where the store lives.
|
|
EnvDataDir = "AO_DATA_DIR"
|
|
)
|
|
|
|
// hookBinaryName is the executable name the workspace hook commands invoke:
|
|
// every agent adapter installs a bare `ao hooks <agent> <event>`. The session
|
|
// PATH pin (hookPATH) only works when the daemon's own executable carries this
|
|
// name, since prepending its directory must change what `ao` resolves to.
|
|
const hookBinaryName = "ao"
|
|
|
|
type lifecycleRecorder interface {
|
|
MarkSpawned(ctx context.Context, id domain.SessionID, metadata domain.SessionMetadata) error
|
|
MarkTerminated(ctx context.Context, id domain.SessionID) error
|
|
}
|
|
|
|
type runtimeController interface {
|
|
Create(ctx context.Context, cfg ports.RuntimeConfig) (ports.RuntimeHandle, error)
|
|
Destroy(ctx context.Context, handle ports.RuntimeHandle) error
|
|
// IsAlive reports whether the handle's runtime session still exists. Used by
|
|
// Reconcile on boot to adopt crash-surviving sessions and reap leaked ones.
|
|
IsAlive(ctx context.Context, handle ports.RuntimeHandle) (bool, error)
|
|
}
|
|
|
|
// Store is the persistence surface needed by the internal session Manager.
|
|
type Store interface {
|
|
// GetProject loads a project row so spawn can resolve its per-project agent
|
|
// config into the launch command. ok=false means the project is unknown.
|
|
GetProject(ctx context.Context, id string) (domain.ProjectRecord, bool, error)
|
|
CreateSession(ctx context.Context, rec domain.SessionRecord) (domain.SessionRecord, error)
|
|
GetSession(ctx context.Context, id domain.SessionID) (domain.SessionRecord, bool, error)
|
|
ListSessions(ctx context.Context, project domain.ProjectID) ([]domain.SessionRecord, error)
|
|
ListAllSessions(ctx context.Context) ([]domain.SessionRecord, error)
|
|
// DeleteSession removes a session row only if it is still in seed state
|
|
// (no workspace, runtime handle, agent session id, or prompt; not
|
|
// terminated). Returns deleted=true when removal happened; deleted=false
|
|
// when the row had already progressed past seed state — preserving the
|
|
// no-resurrection guarantee for live sessions.
|
|
DeleteSession(ctx context.Context, id domain.SessionID) (bool, error)
|
|
// UpsertSessionWorktree records or updates the worktree row for a session.
|
|
// SaveAndTeardownAll writes the preserved_ref here (even when empty) as the
|
|
// "shutdown-saved" marker before ForceDestroying the worktree.
|
|
UpsertSessionWorktree(ctx context.Context, row domain.SessionWorktreeRecord) error
|
|
// ListSessionWorktrees returns every worktree row for a session. RestoreAll
|
|
// uses this to identify sessions saved by the last SaveAndTeardownAll: the
|
|
// presence of any row is the marker; preserved_ref may be empty for clean
|
|
// worktrees.
|
|
ListSessionWorktrees(ctx context.Context, id domain.SessionID) ([]domain.SessionWorktreeRecord, error)
|
|
// DeleteSessionWorktrees consumes stale shutdown-restore markers. Explicit
|
|
// Kill and successful RestoreAll must remove these rows to prevent
|
|
// resurrecting sessions the user intentionally terminated.
|
|
DeleteSessionWorktrees(ctx context.Context, id domain.SessionID) error
|
|
}
|
|
|
|
// Manager coordinates internal session spawn, restore, kill, and cleanup over
|
|
// the outbound ports. User-facing read-model assembly lives in the service package.
|
|
type Manager struct {
|
|
runtime runtimeController
|
|
agents ports.AgentResolver
|
|
workspace ports.Workspace
|
|
store Store
|
|
messenger ports.AgentMessenger
|
|
lcm lifecycleRecorder
|
|
dataDir string
|
|
clock func() time.Time
|
|
// lookPath is exec.LookPath in production; tests substitute a stub so
|
|
// they don't need real binaries on PATH. Returns ports.ErrAgentBinaryNotFound
|
|
// when the binary is missing so the sentinel propagates through toAPIError.
|
|
lookPath func(string) (string, error)
|
|
// executable resolves the daemon's own binary (os.Executable in
|
|
// production); its directory is prepended to spawned sessions' PATH so the
|
|
// workspace hook commands resolve back to this daemon. Tests inject a stub.
|
|
executable func() (string, error)
|
|
logger *slog.Logger
|
|
}
|
|
|
|
// Deps are the collaborators a Session Manager needs; New wires them together.
|
|
type Deps struct {
|
|
Runtime runtimeController
|
|
Agents ports.AgentResolver
|
|
Workspace ports.Workspace
|
|
Store Store
|
|
Messenger ports.AgentMessenger
|
|
Lifecycle lifecycleRecorder
|
|
// DataDir is exported to spawned agents as AO_DATA_DIR so their hook
|
|
// commands can open the same store.
|
|
DataDir string
|
|
Clock func() time.Time
|
|
// LookPath overrides exec.LookPath for the pre-launch agent-binary check.
|
|
// Production wiring leaves this nil and the manager defaults to
|
|
// exec.LookPath; tests inject a stub so they need not seed real binaries.
|
|
LookPath func(string) (string, error)
|
|
// Executable overrides os.Executable for the session PATH pin (see
|
|
// hookPATH). Production wiring leaves this nil; tests inject a stub so they
|
|
// control what the test binary appears to be.
|
|
Executable func() (string, error)
|
|
// Logger receives spawn-time diagnostics (e.g. when the session PATH
|
|
// cannot be pinned to the daemon binary). Nil defaults to slog.Default().
|
|
Logger *slog.Logger
|
|
}
|
|
|
|
// New builds a Session Manager from its dependencies, defaulting the clock to
|
|
// time.Now when Deps.Clock is nil.
|
|
func New(d Deps) *Manager {
|
|
m := &Manager{
|
|
runtime: d.Runtime,
|
|
agents: d.Agents,
|
|
workspace: d.Workspace,
|
|
store: d.Store,
|
|
messenger: d.Messenger,
|
|
lcm: d.Lifecycle,
|
|
dataDir: d.DataDir,
|
|
clock: d.Clock,
|
|
lookPath: d.LookPath,
|
|
executable: d.Executable,
|
|
logger: d.Logger,
|
|
}
|
|
if m.clock == nil {
|
|
// UTC so spawn-stamped CreatedAt/UpdatedAt match every other session
|
|
// write (rename, activity) — all of which use time.Now().UTC(). A local
|
|
// default produced mixed-timezone timestamps in `ao session get`.
|
|
m.clock = func() time.Time { return time.Now().UTC() }
|
|
}
|
|
if m.lookPath == nil {
|
|
m.lookPath = exec.LookPath
|
|
}
|
|
if m.executable == nil {
|
|
m.executable = os.Executable
|
|
}
|
|
if m.logger == nil {
|
|
m.logger = slog.Default()
|
|
}
|
|
return m
|
|
}
|
|
|
|
// Spawn creates the session row (which assigns the "{project}-{n}" id), then the
|
|
// workspace and runtime, then reports completion to the LCM. If workspace
|
|
// materialization fails the still-seed row is deleted outright; a later failure
|
|
// parks the row as terminated and rolls back what was built.
|
|
func (m *Manager) Spawn(ctx context.Context, cfg ports.SpawnConfig) (domain.SessionRecord, error) {
|
|
project, err := m.loadProject(ctx, cfg.ProjectID)
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn: %w", err)
|
|
}
|
|
// A per-project role override picks the harness when the spawn names none,
|
|
// so a project can default workers to one agent and orchestrators to another.
|
|
cfg.Harness = effectiveHarness(cfg.Harness, cfg.Kind, project.Config)
|
|
if cfg.Harness == "" {
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn: %w: configure project %s.agent or pass --harness", ErrMissingHarness, roleConfigName(cfg.Kind))
|
|
}
|
|
|
|
// Reject an unknown harness before any durable state is created. Doing this
|
|
// after CreateSession would leave a terminated orphan row and waste a
|
|
// worktree on a spawn that can never launch.
|
|
if _, ok := m.agents.Agent(cfg.Harness); !ok {
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn: %w: %q", ErrUnknownHarness, cfg.Harness)
|
|
}
|
|
|
|
if err := m.validateRuntimePrerequisites(); err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn: %w", err)
|
|
}
|
|
|
|
prompt, systemPrompt, err := m.buildSpawnTexts(ctx, cfg)
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn: prompt: %w", err)
|
|
}
|
|
|
|
rec, err := m.store.CreateSession(ctx, seedRecord(cfg, m.clock()))
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn: create: %w", err)
|
|
}
|
|
id := rec.ID
|
|
systemPromptFile, err := m.prepareSystemPromptFile(id, cfg.Harness, systemPrompt)
|
|
if err != nil {
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: system prompt file: %w", id, err)
|
|
}
|
|
|
|
branch := cfg.Branch
|
|
if branch == "" {
|
|
branch = defaultSessionBranch(id, cfg.Kind, sessionPrefix(project))
|
|
}
|
|
ws, err := m.workspace.Create(ctx, ports.WorkspaceConfig{
|
|
ProjectID: cfg.ProjectID,
|
|
SessionID: id,
|
|
Kind: cfg.Kind,
|
|
SessionPrefix: sessionPrefix(project),
|
|
Branch: branch,
|
|
BaseBranch: project.Config.WithDefaults().DefaultBranch,
|
|
})
|
|
if err != nil {
|
|
// Nothing observable exists yet — no worktree, no runtime — so the seed
|
|
// row is deleted outright instead of accumulating as a terminated orphan
|
|
// in session lists (e.g. when gitworktree refuses the branch).
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: workspace: %w", id, err)
|
|
}
|
|
|
|
// Per-project workspace provisioning: symlink shared files, then run any
|
|
// post-create commands (e.g. `pnpm install`) before the agent launches.
|
|
if err := m.provisionWorkspace(ctx, project, ws.Path); err != nil {
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: provision: %w", id, err)
|
|
}
|
|
|
|
agent, ok := m.agents.Agent(cfg.Harness)
|
|
if !ok {
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: no agent adapter for harness %q", id, cfg.Harness)
|
|
}
|
|
if err := m.prepareWorkspace(ctx, agent, id, ws.Path); err != nil {
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: %w", id, err)
|
|
}
|
|
agentConfig := effectiveAgentConfig(cfg.Kind, project.Config)
|
|
argv, err := agent.GetLaunchCommand(ctx, ports.LaunchConfig{
|
|
SessionID: string(id),
|
|
WorkspacePath: ws.Path,
|
|
Prompt: prompt,
|
|
SystemPrompt: systemPrompt,
|
|
SystemPromptFile: systemPromptFile,
|
|
IssueID: string(cfg.IssueID),
|
|
Config: agentConfig,
|
|
Permissions: agentConfig.Permissions,
|
|
})
|
|
if err != nil {
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: launch command: %w", id, err)
|
|
}
|
|
// Pre-flight: confirm argv[0] actually exists on PATH (or as an absolute
|
|
// path the adapter returned) BEFORE handing the launch to the runtime.
|
|
// tmux happily creates a session+pane around a missing command, so an
|
|
// unresolved binary would leak through as a "live" session that never ran.
|
|
if err := m.validateAgentBinary(argv); err != nil {
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: %w", id, err)
|
|
}
|
|
handle, err := m.runtime.Create(ctx, ports.RuntimeConfig{
|
|
SessionID: id,
|
|
WorkspacePath: ws.Path,
|
|
Argv: argv,
|
|
Env: m.runtimeEnv(id, cfg.ProjectID, cfg.IssueID, project.Config.Env),
|
|
})
|
|
if err != nil {
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.rollbackSpawnSeedRow(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: runtime: %w", id, err)
|
|
}
|
|
|
|
metadata := domain.SessionMetadata{Branch: ws.Branch, WorkspacePath: ws.Path, RuntimeHandleID: handle.ID, Prompt: prompt}
|
|
if err := m.lcm.MarkSpawned(ctx, id, metadata); err != nil {
|
|
_ = m.runtime.Destroy(ctx, handle)
|
|
_ = m.workspace.Destroy(ctx, ws)
|
|
m.markSpawnFailedTerminated(ctx, id)
|
|
return domain.SessionRecord{}, fmt.Errorf("spawn %s: completed: %w", id, err)
|
|
}
|
|
return m.getRecord(ctx, id)
|
|
}
|
|
|
|
// loadProject loads the project record so spawn can resolve its per-project
|
|
// config (harness/agent overrides, env, branch, rules, provisioning). A missing
|
|
// project yields a zero record rather than an error: the project may be
|
|
// unregistered yet still have live sessions, and an empty config simply means
|
|
// every field falls back to its default.
|
|
func (m *Manager) loadProject(ctx context.Context, projectID domain.ProjectID) (domain.ProjectRecord, error) {
|
|
row, ok, err := m.store.GetProject(ctx, string(projectID))
|
|
if err != nil {
|
|
return domain.ProjectRecord{}, fmt.Errorf("load project: %w", err)
|
|
}
|
|
if !ok {
|
|
return domain.ProjectRecord{}, nil
|
|
}
|
|
return row, nil
|
|
}
|
|
|
|
// effectiveHarness resolves the harness for a spawn: an explicit harness wins;
|
|
// otherwise the project's role override for the session kind applies. Empty is
|
|
// invalid for new worker/orchestrator launches and is rejected by Spawn.
|
|
func effectiveHarness(explicit domain.AgentHarness, kind domain.SessionKind, cfg domain.ProjectConfig) domain.AgentHarness {
|
|
if explicit != "" {
|
|
return explicit
|
|
}
|
|
if role := roleOverride(kind, cfg).Harness; role != "" {
|
|
return role
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func roleConfigName(kind domain.SessionKind) string {
|
|
if kind == domain.KindOrchestrator {
|
|
return "orchestrator"
|
|
}
|
|
return "worker"
|
|
}
|
|
|
|
// effectiveAgentConfig merges the role override's agent config over the
|
|
// project's base agent config; set override fields win.
|
|
func effectiveAgentConfig(kind domain.SessionKind, cfg domain.ProjectConfig) ports.AgentConfig {
|
|
merged := cfg.AgentConfig
|
|
override := roleOverride(kind, cfg).AgentConfig
|
|
if override.Model != "" {
|
|
merged.Model = override.Model
|
|
}
|
|
if override.Permissions != "" {
|
|
merged.Permissions = override.Permissions
|
|
}
|
|
return merged
|
|
}
|
|
|
|
func roleOverride(kind domain.SessionKind, cfg domain.ProjectConfig) domain.RoleOverride {
|
|
if kind == domain.KindOrchestrator {
|
|
return cfg.Orchestrator
|
|
}
|
|
return cfg.Worker
|
|
}
|
|
|
|
// sessionPrefix returns the display prefix for a project: the explicit
|
|
// SessionPrefix when set, otherwise the first 12 characters of the project ID.
|
|
func sessionPrefix(project domain.ProjectRecord) string {
|
|
if p := strings.TrimSpace(project.Config.SessionPrefix); p != "" {
|
|
return p
|
|
}
|
|
if len(project.ID) <= 12 {
|
|
return project.ID
|
|
}
|
|
return project.ID[:12]
|
|
}
|
|
|
|
// markSpawnFailedTerminated best-effort parks an orphaned spawn as terminated.
|
|
// A phantom half-spawned row is worse than a terminal one; we only delete the
|
|
// row when nothing observable has landed yet (seed state) via rollbackSpawn or
|
|
// rollbackSpawnSeedRow.
|
|
func (m *Manager) markSpawnFailedTerminated(ctx context.Context, id domain.SessionID) {
|
|
_ = m.lcm.MarkTerminated(ctx, id)
|
|
m.cleanupSystemPromptDir(id)
|
|
}
|
|
|
|
// rollbackSpawnSeedRow best-effort removes the row of a spawn that failed
|
|
// before anything observable (worktree, runtime) was built, so failed spawns
|
|
// don't accumulate terminated rows in session lists. DeleteSession only removes
|
|
// rows still in seed state; if the row has progressed or the delete itself
|
|
// fails, fall back to parking it terminated so a phantom row never looks live.
|
|
func (m *Manager) rollbackSpawnSeedRow(ctx context.Context, id domain.SessionID) {
|
|
if deleted, err := m.store.DeleteSession(ctx, id); err == nil && deleted {
|
|
m.cleanupSystemPromptDir(id)
|
|
return
|
|
}
|
|
m.markSpawnFailedTerminated(ctx, id)
|
|
}
|
|
|
|
// rollbackSpawn deletes a session row when it is still in seed state — used
|
|
// when an out-of-band step that happens AFTER `Spawn` returns (e.g. PR claim
|
|
// over HTTP) has failed and the caller wants the partially-spawned session
|
|
// gone without leaving a terminated orphan visible under `--include-terminated`.
|
|
//
|
|
// If the row has progressed past seed state (workspace exists, runtime created,
|
|
// etc.), DeleteSession is a no-op and rollbackSpawn falls back to a Kill so the
|
|
// runtime/workspace are torn down. Returns (deleted, killed):
|
|
// - deleted=true: the row was a seed row and has been removed
|
|
// - killed=true: the row had spawn output and was torn down + terminated
|
|
// - both false: the row was already terminated or absent — benign no-op
|
|
func (m *Manager) rollbackSpawn(ctx context.Context, id domain.SessionID) (deleted, killed bool, err error) {
|
|
deleted, err = m.store.DeleteSession(ctx, id)
|
|
if err != nil {
|
|
return false, false, fmt.Errorf("rollback %s: %w", id, err)
|
|
}
|
|
if deleted {
|
|
m.cleanupSystemPromptDir(id)
|
|
return true, false, nil
|
|
}
|
|
killed, err = m.Kill(ctx, id)
|
|
if err != nil {
|
|
return false, false, err
|
|
}
|
|
return false, killed, nil
|
|
}
|
|
|
|
// RollbackSpawn is the public surface of rollbackSpawn for service-layer callers.
|
|
func (m *Manager) RollbackSpawn(ctx context.Context, id domain.SessionID) (deleted, killed bool, err error) {
|
|
return m.rollbackSpawn(ctx, id)
|
|
}
|
|
|
|
// Kill records terminal intent with the LCM, then tears down the runtime and
|
|
// workspace. A workspace teardown refused by the worktree-remove safety
|
|
// (uncommitted work) is never forced: the session still terminates and Kill
|
|
// succeeds with freed=false, signalling the workspace was preserved.
|
|
//
|
|
// A session whose runtime handle or workspace path is missing (e.g. spawn
|
|
// failed partway, handle lost after a crash) is still terminated — the destroy
|
|
// steps are skipped for whatever is absent, but the session record always
|
|
// moves to terminal state so it can be cleaned up from the dashboard.
|
|
func (m *Manager) Kill(ctx context.Context, id domain.SessionID) (bool, error) {
|
|
rec, ok, err := m.store.GetSession(ctx, id)
|
|
if err != nil {
|
|
return false, fmt.Errorf("kill %s: %w", id, err)
|
|
}
|
|
if !ok {
|
|
return false, nil // already gone: benign race
|
|
}
|
|
handle := runtimeHandle(rec.Metadata)
|
|
ws := workspaceInfo(rec)
|
|
|
|
// Always record terminal intent so the session is marked terminated even
|
|
// when the runtime/workspace handle is missing.
|
|
if err := m.lcm.MarkTerminated(ctx, id); err != nil {
|
|
return false, fmt.Errorf("kill %s: %w", id, err)
|
|
}
|
|
defer m.cleanupSystemPromptDir(id)
|
|
|
|
// Clear the restore marker so the next boot's RestoreAll cannot resurrect a
|
|
// killed session (#2319). Best-effort: teardown below still matters.
|
|
if err := m.store.DeleteSessionWorktrees(ctx, id); err != nil {
|
|
m.logger.Warn("kill: delete restore marker failed", "sessionID", id, "error", err)
|
|
}
|
|
|
|
// Only tear down what exists. A session may have lost its handle after a
|
|
// crash or never acquired one if spawn failed partway.
|
|
if handle.ID != "" {
|
|
if err := m.runtime.Destroy(ctx, handle); err != nil {
|
|
return false, fmt.Errorf("kill %s: runtime: %w", id, err)
|
|
}
|
|
}
|
|
freed := false
|
|
if ws.Path != "" {
|
|
if err := m.workspace.Destroy(ctx, ws); err != nil {
|
|
if errors.Is(err, ports.ErrWorkspaceDirty) {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("kill %s: workspace: %w", id, err)
|
|
}
|
|
freed = true
|
|
}
|
|
return freed, nil
|
|
}
|
|
|
|
// RetireForReplacement terminates a live orchestrator and releases its branch
|
|
// for a replacement session. Unlike Kill, this captures uncommitted work before
|
|
// force-removing the worktree, so a dirty canonical orchestrator worktree does
|
|
// not block the replacement from claiming the canonical branch.
|
|
//
|
|
// This deliberately does not write a session_worktrees row: those rows are
|
|
// boot-restore markers, and a replaced orchestrator must stay terminated.
|
|
func (m *Manager) RetireForReplacement(ctx context.Context, id domain.SessionID) error {
|
|
rec, ok, err := m.store.GetSession(ctx, id)
|
|
if err != nil {
|
|
return fmt.Errorf("retire replacement %s: %w", id, err)
|
|
}
|
|
if !ok || rec.IsTerminated {
|
|
return nil
|
|
}
|
|
if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" {
|
|
if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil {
|
|
return fmt.Errorf("retire replacement %s: clear restore markers: %w", id, err)
|
|
}
|
|
handle := runtimeHandle(rec.Metadata)
|
|
if handle.ID != "" {
|
|
if err := m.runtime.Destroy(ctx, handle); err != nil {
|
|
return fmt.Errorf("retire replacement %s: runtime: %w", id, err)
|
|
}
|
|
}
|
|
if err := m.lcm.MarkTerminated(ctx, id); err != nil {
|
|
return fmt.Errorf("retire replacement %s: mark terminated: %w", id, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
ws := workspaceInfo(rec)
|
|
if _, err := m.workspace.StashUncommitted(ctx, ws); err != nil {
|
|
return fmt.Errorf("retire replacement %s: stash: %w", id, err)
|
|
}
|
|
if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil {
|
|
return fmt.Errorf("retire replacement %s: clear restore markers: %w", id, err)
|
|
}
|
|
handle := runtimeHandle(rec.Metadata)
|
|
if handle.ID != "" {
|
|
if err := m.runtime.Destroy(ctx, handle); err != nil {
|
|
return fmt.Errorf("retire replacement %s: runtime: %w", id, err)
|
|
}
|
|
}
|
|
if err := m.workspace.ForceDestroy(ctx, ws); err != nil {
|
|
return fmt.Errorf("retire replacement %s: force destroy: %w", id, err)
|
|
}
|
|
if err := m.lcm.MarkTerminated(ctx, rec.ID); err != nil {
|
|
return fmt.Errorf("retire replacement %s: mark terminated: %w", id, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Restore relaunches a torn-down session in its workspace. The fallible I/O runs
|
|
// before any durable session write, so a failure never resurrects the row or destroys
|
|
// the worktree (it may hold the agent's prior work).
|
|
func (m *Manager) Restore(ctx context.Context, id domain.SessionID) (domain.SessionRecord, error) {
|
|
rec, ok, err := m.store.GetSession(ctx, id)
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
|
|
}
|
|
if !ok {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, ErrNotFound)
|
|
}
|
|
if !rec.IsTerminated {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, ErrNotRestorable)
|
|
}
|
|
meta := rec.Metadata
|
|
// Mirror Kill's incomplete-handle guard: a session whose spawn failed before
|
|
// the workspace landed has neither WorkspacePath nor Branch, and there is
|
|
// nothing meaningful to restore from. Surface this as a typed 409 instead of
|
|
// letting workspace.Restore fail with an opaque wrapped error.
|
|
if meta.WorkspacePath == "" || meta.Branch == "" {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, ErrIncompleteHandle)
|
|
}
|
|
// Resumability is decided inside restoreArgv, not here. A promptless session
|
|
// can still be fully resumable when the harness pins a deterministic session id
|
|
// (Claude Code). restoreArgv returns ErrNotResumable only for a promptless,
|
|
// unresumable non-orchestrator (a worker with no task and no native id to resume).
|
|
// Orchestrators always relaunch fresh with the system prompt only.
|
|
|
|
project, err := m.loadProject(ctx, rec.ProjectID)
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
|
|
}
|
|
ws, err := m.workspace.Restore(ctx, ports.WorkspaceConfig{
|
|
ProjectID: rec.ProjectID,
|
|
SessionID: id,
|
|
Kind: rec.Kind,
|
|
SessionPrefix: sessionPrefix(project),
|
|
Branch: meta.Branch,
|
|
})
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: workspace: %w", id, err)
|
|
}
|
|
agent, ok := m.agents.Agent(rec.Harness)
|
|
if !ok {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: no agent adapter for harness %q", id, rec.Harness)
|
|
}
|
|
if err := m.prepareWorkspace(ctx, agent, id, ws.Path); err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
|
|
}
|
|
// The system prompt is derived, not persisted: recompute it so a restored
|
|
// session keeps its standing instructions across the relaunch.
|
|
systemPrompt, err := m.buildSystemPrompt(ctx, rec.Kind, rec.ProjectID)
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: system prompt: %w", id, err)
|
|
}
|
|
systemPromptFile, err := m.prepareSystemPromptFile(id, rec.Harness, systemPrompt)
|
|
if err != nil {
|
|
m.cleanupSystemPromptDir(id)
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: system prompt file: %w", id, err)
|
|
}
|
|
// Restore re-applies the project's resolved agent config so a configured
|
|
// model/permissions carry across a restore, matching fresh spawn.
|
|
argv, err := restoreArgv(ctx, agent, id, ws.Path, meta, systemPrompt, systemPromptFile, effectiveAgentConfig(rec.Kind, project.Config), rec.Kind)
|
|
if err != nil {
|
|
m.cleanupSystemPromptDir(id)
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: %w", id, err)
|
|
}
|
|
handle, err := m.runtime.Create(ctx, ports.RuntimeConfig{
|
|
SessionID: id,
|
|
WorkspacePath: ws.Path,
|
|
Argv: argv,
|
|
Env: m.runtimeEnv(id, rec.ProjectID, rec.IssueID, project.Config.Env),
|
|
})
|
|
if err != nil {
|
|
m.cleanupSystemPromptDir(id)
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: runtime: %w", id, err)
|
|
}
|
|
metadata := domain.SessionMetadata{Branch: ws.Branch, WorkspacePath: ws.Path, RuntimeHandleID: handle.ID, AgentSessionID: meta.AgentSessionID, Prompt: meta.Prompt}
|
|
if err := m.lcm.MarkSpawned(ctx, id, metadata); err != nil {
|
|
_ = m.runtime.Destroy(ctx, handle)
|
|
m.cleanupSystemPromptDir(id)
|
|
return domain.SessionRecord{}, fmt.Errorf("restore %s: completed: %w", id, err)
|
|
}
|
|
return m.getRecord(ctx, id)
|
|
}
|
|
|
|
func (m *Manager) getRecord(ctx context.Context, id domain.SessionID) (domain.SessionRecord, error) {
|
|
rec, ok, err := m.store.GetSession(ctx, id)
|
|
if err != nil {
|
|
return domain.SessionRecord{}, fmt.Errorf("get %s: %w", id, err)
|
|
}
|
|
if !ok {
|
|
return domain.SessionRecord{}, fmt.Errorf("get %s: %w", id, ErrNotFound)
|
|
}
|
|
return rec, nil
|
|
}
|
|
|
|
// SaveAndTeardownAll captures uncommitted work and tears down every live
|
|
// session that has a workspace path. It is the shutdown path for the daemon:
|
|
// each session's uncommitted work is stashed into a preserve ref, the ref is
|
|
// written to session_worktrees (the "shutdown-saved" marker) BEFORE the
|
|
// worktree is force-removed. The DB write is committed before the worktree is
|
|
// destroyed so a crash between the two leaves the ref in place and the row
|
|
// present; RestoreAll will replay both.
|
|
//
|
|
// Failures on individual sessions are logged and do not abort the loop.
|
|
// ForceDestroy is never called if capture or the DB write did not succeed.
|
|
func (m *Manager) SaveAndTeardownAll(ctx context.Context) error {
|
|
recs, err := m.store.ListAllSessions(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("save-teardown-all: list sessions: %w", err)
|
|
}
|
|
for _, rec := range recs {
|
|
if rec.IsTerminated {
|
|
continue
|
|
}
|
|
if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" {
|
|
continue
|
|
}
|
|
if err := m.saveAndTeardownOne(ctx, rec); err != nil {
|
|
m.logger.Error("save-teardown-all: session failed, skipping", "sessionID", rec.ID, "error", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// saveAndTeardownOne runs the capture-then-destroy sequence for a single
|
|
// session. The DB write (UpsertSessionWorktree) is committed before
|
|
// ForceDestroy; if either capture or the DB write fails, ForceDestroy is
|
|
// not called.
|
|
func (m *Manager) saveAndTeardownOne(ctx context.Context, rec domain.SessionRecord) error {
|
|
ws := workspaceInfo(rec)
|
|
|
|
// 1. Capture uncommitted work (ref may be "" for clean worktrees).
|
|
ref, err := m.workspace.StashUncommitted(ctx, ws)
|
|
if err != nil {
|
|
return fmt.Errorf("save %s: stash: %w", rec.ID, err)
|
|
}
|
|
|
|
// 2. Write the shutdown-saved marker to the DB. The row's presence (even
|
|
// with an empty preserved_ref) is what RestoreAll uses to identify sessions
|
|
// saved by this run. This MUST be committed before ForceDestroy.
|
|
row := domain.SessionWorktreeRecord{
|
|
SessionID: rec.ID,
|
|
RepoName: domain.RootWorkspaceRepoName,
|
|
Branch: rec.Metadata.Branch,
|
|
WorktreePath: rec.Metadata.WorkspacePath,
|
|
PreservedRef: ref,
|
|
}
|
|
if err := m.store.UpsertSessionWorktree(ctx, row); err != nil {
|
|
return fmt.Errorf("save %s: upsert worktree row: %w", rec.ID, err)
|
|
}
|
|
|
|
// 3. Mark terminal via the LCM (same path Kill uses).
|
|
if err := m.lcm.MarkTerminated(ctx, rec.ID); err != nil {
|
|
return fmt.Errorf("save %s: mark terminated: %w", rec.ID, err)
|
|
}
|
|
|
|
// 4. Runtime teardown (best-effort; same pattern as Kill).
|
|
handle := runtimeHandle(rec.Metadata)
|
|
if handle.ID != "" {
|
|
if err := m.runtime.Destroy(ctx, handle); err != nil {
|
|
m.logger.Warn("save-teardown-all: runtime destroy failed", "sessionID", rec.ID, "error", err)
|
|
}
|
|
}
|
|
|
|
// 5. Force-remove the worktree (safe: work is captured in step 1 and the
|
|
// DB write in step 2 is already committed).
|
|
if err := m.workspace.ForceDestroy(ctx, ws); err != nil {
|
|
m.logger.Warn("save-teardown-all: force destroy failed", "sessionID", rec.ID, "error", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// reconcileLive handles a single non-terminated session on boot. If its runtime
|
|
// session is still alive (tmux is the persistence layer, so it survives a daemon
|
|
// crash) we adopt it: a no-op, the agent keeps running. If the runtime is gone,
|
|
// the agent died with the daemon, so we save-and-tear-down to the SAME end state
|
|
// a graceful shutdown produces: capture uncommitted work into a preserve ref,
|
|
// record the session_worktrees restore marker, mark terminated, and remove the
|
|
// worktree. RestoreAll (which Reconcile runs immediately after) then relaunches
|
|
// it on this same boot, resuming history. Crash recovery thus matches graceful
|
|
// restart instead of silently abandoning the session.
|
|
//
|
|
// If the work capture fails we mark terminated WITHOUT a marker and leave the
|
|
// worktree intact: better to skip the relaunch than to tear down un-preserved
|
|
// work or relaunch onto an inconsistent worktree.
|
|
func (m *Manager) reconcileLive(ctx context.Context, rec domain.SessionRecord) error {
|
|
if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" {
|
|
return nil
|
|
}
|
|
handle := runtimeHandle(rec.Metadata)
|
|
if handle.ID != "" {
|
|
alive, err := m.runtime.IsAlive(ctx, handle)
|
|
if err != nil {
|
|
// A failed probe is not proof of death: leave the session as-is.
|
|
return fmt.Errorf("reconcile %s: probe: %w", rec.ID, err)
|
|
}
|
|
if alive {
|
|
return nil // adopt: the session survived the crash.
|
|
}
|
|
}
|
|
// Runtime is gone: capture uncommitted work first.
|
|
ws := workspaceInfo(rec)
|
|
ref, err := m.workspace.StashUncommitted(ctx, ws)
|
|
if err != nil {
|
|
// Could not capture work: do NOT write a restore marker or tear down the
|
|
// worktree (that would risk losing un-preserved work). Mark terminated so
|
|
// a dead session is not left looking live; the worktree stays put.
|
|
m.logger.Warn("reconcile: stash uncommitted failed; terminating without restore marker", "sessionID", rec.ID, "error", err)
|
|
if mErr := m.lcm.MarkTerminated(ctx, rec.ID); mErr != nil {
|
|
return fmt.Errorf("reconcile %s: mark terminated: %w", rec.ID, mErr)
|
|
}
|
|
return nil
|
|
}
|
|
// Work captured. Record the shutdown-saved marker BEFORE tearing down the
|
|
// worktree, mirroring saveAndTeardownOne, so RestoreAll relaunches it.
|
|
row := domain.SessionWorktreeRecord{
|
|
SessionID: rec.ID,
|
|
RepoName: domain.RootWorkspaceRepoName,
|
|
Branch: rec.Metadata.Branch,
|
|
WorktreePath: rec.Metadata.WorkspacePath,
|
|
PreservedRef: ref,
|
|
}
|
|
if err := m.store.UpsertSessionWorktree(ctx, row); err != nil {
|
|
return fmt.Errorf("reconcile %s: upsert worktree marker: %w", rec.ID, err)
|
|
}
|
|
if err := m.lcm.MarkTerminated(ctx, rec.ID); err != nil {
|
|
return fmt.Errorf("reconcile %s: mark terminated: %w", rec.ID, err)
|
|
}
|
|
// Remove the worktree (work is captured in the ref): RestoreAll re-creates it
|
|
// clean and replays the ref. The dead runtime needs no Destroy.
|
|
if err := m.workspace.ForceDestroy(ctx, ws); err != nil {
|
|
m.logger.Warn("reconcile: force destroy failed after marker", "sessionID", rec.ID, "error", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// reconcileReap kills the leaked tmux session of a session the DB already marks
|
|
// terminated. This covers the teardown that marked the row terminated but failed
|
|
// to kill the runtime (e.g. ForceDestroy/Destroy errored after MarkTerminated).
|
|
// Destroy is idempotent, so an already-gone session is a no-op.
|
|
func (m *Manager) reconcileReap(ctx context.Context, rec domain.SessionRecord) error {
|
|
handle := runtimeHandle(rec.Metadata)
|
|
if handle.ID == "" {
|
|
return nil
|
|
}
|
|
alive, err := m.runtime.IsAlive(ctx, handle)
|
|
if err != nil {
|
|
return fmt.Errorf("reconcile reap %s: probe: %w", rec.ID, err)
|
|
}
|
|
if !alive {
|
|
return nil
|
|
}
|
|
if err := m.runtime.Destroy(ctx, handle); err != nil {
|
|
return fmt.Errorf("reconcile reap %s: destroy: %w", rec.ID, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Reconcile is the boot-time consistency pass. It replaces the bare RestoreAll
|
|
// call so that however the previous daemon died (clean shutdown, SIGKILL, or
|
|
// crash), live reality matches the DB:
|
|
//
|
|
// 1. Live pass: for each non-terminated session, adopt it if its runtime
|
|
// survived, else capture work and mark terminated (reconcileLive).
|
|
// 2. Reap pass: for each terminated session whose runtime leaked, kill it
|
|
// (reconcileReap). Runs before restore so a restored session does not
|
|
// collide with a leaked tmux of the same name.
|
|
// 3. Restore pass: relaunch shutdown-saved sessions (existing RestoreAll).
|
|
//
|
|
// Best-effort throughout: a per-session failure is logged and never aborts the
|
|
// pass or blocks boot.
|
|
func (m *Manager) Reconcile(ctx context.Context) error {
|
|
recs, err := m.store.ListAllSessions(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("reconcile: list sessions: %w", err)
|
|
}
|
|
for _, rec := range recs {
|
|
if rec.IsTerminated {
|
|
continue
|
|
}
|
|
if err := m.reconcileLive(ctx, rec); err != nil {
|
|
m.logger.Error("reconcile: live pass failed, skipping", "sessionID", rec.ID, "error", err)
|
|
}
|
|
}
|
|
for _, rec := range recs {
|
|
if !rec.IsTerminated {
|
|
continue
|
|
}
|
|
if err := m.reconcileReap(ctx, rec); err != nil {
|
|
m.logger.Error("reconcile: reap pass failed, skipping", "sessionID", rec.ID, "error", err)
|
|
}
|
|
}
|
|
return m.RestoreAll(ctx)
|
|
}
|
|
|
|
// RestoreAll relaunches every terminated session that was saved by the last
|
|
// SaveAndTeardownAll. The "shutdown-saved" marker is the presence of a
|
|
// session_worktrees row for the session; sessions the user killed before
|
|
// shutdown have no such row and are left terminated.
|
|
//
|
|
// For each saved session:
|
|
// 1. Ensure the worktree exists via workspace.Restore.
|
|
// 2. If a preserve ref is recorded, replay it via ApplyPreserved; on conflict
|
|
// log and continue (still relaunch the agent, never delete the ref).
|
|
// 3. Relaunch via the existing Restore method.
|
|
//
|
|
// Failures on individual sessions are logged and do not abort the loop.
|
|
func (m *Manager) RestoreAll(ctx context.Context) error {
|
|
recs, err := m.store.ListAllSessions(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("restore-all: list sessions: %w", err)
|
|
}
|
|
for _, rec := range recs {
|
|
if !rec.IsTerminated {
|
|
continue
|
|
}
|
|
// Check the shutdown-saved marker: is there a session_worktrees row?
|
|
rows, err := m.store.ListSessionWorktrees(ctx, rec.ID)
|
|
if err != nil {
|
|
m.logger.Error("restore-all: list worktrees failed", "sessionID", rec.ID, "error", err)
|
|
continue
|
|
}
|
|
if len(rows) == 0 {
|
|
// No marker: this session was killed by the user before shutdown.
|
|
continue
|
|
}
|
|
|
|
// Collect the preserve ref (may be "" for clean worktrees).
|
|
var preserveRef string
|
|
for _, r := range rows {
|
|
if r.PreservedRef != "" {
|
|
preserveRef = r.PreservedRef
|
|
break
|
|
}
|
|
}
|
|
|
|
// Step 1: ensure the worktree exists. workspace.Restore re-creates it
|
|
// if it was removed by SaveAndTeardownAll.
|
|
project, err := m.loadProject(ctx, rec.ProjectID)
|
|
if err != nil {
|
|
m.logger.Error("restore-all: load project failed", "sessionID", rec.ID, "error", err)
|
|
continue
|
|
}
|
|
ws, err := m.workspace.Restore(ctx, ports.WorkspaceConfig{
|
|
ProjectID: rec.ProjectID,
|
|
SessionID: rec.ID,
|
|
Kind: rec.Kind,
|
|
SessionPrefix: sessionPrefix(project),
|
|
Branch: rec.Metadata.Branch,
|
|
})
|
|
if err != nil {
|
|
m.logger.Error("restore-all: workspace restore failed", "sessionID", rec.ID, "error", err)
|
|
continue
|
|
}
|
|
|
|
// Step 2: replay preserve ref when one was recorded.
|
|
if preserveRef != "" {
|
|
if applyErr := m.workspace.ApplyPreserved(ctx, ws, preserveRef); applyErr != nil {
|
|
if errors.Is(applyErr, ports.ErrPreservedConflict) {
|
|
m.logger.Warn("restore-all: apply preserved produced conflicts; agent relaunched with conflict markers in place",
|
|
"sessionID", rec.ID, "ref", preserveRef, "error", applyErr)
|
|
} else {
|
|
m.logger.Error("restore-all: apply preserved failed", "sessionID", rec.ID, "error", applyErr)
|
|
}
|
|
// Continue: always relaunch even on conflict (never delete the ref here).
|
|
}
|
|
}
|
|
|
|
// Step 3: relaunch via the existing single-session Restore method.
|
|
if _, err := m.Restore(ctx, rec.ID); err != nil {
|
|
// A promptless, unresumable worker is intentionally left terminated
|
|
// (ErrNotResumable): expected, not an operational failure, so log it
|
|
// quietly rather than as an error.
|
|
if errors.Is(err, ErrNotResumable) {
|
|
m.logger.Warn("restore-all: session left terminated (nothing to resume)", "sessionID", rec.ID)
|
|
} else {
|
|
m.logger.Error("restore-all: relaunch failed", "sessionID", rec.ID, "error", err)
|
|
}
|
|
continue
|
|
}
|
|
if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil {
|
|
m.logger.Error("restore-all: delete consumed worktree marker failed", "sessionID", rec.ID, "error", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Send delivers a message to a running session's agent via the messenger.
|
|
func (m *Manager) Send(ctx context.Context, id domain.SessionID, message string) error {
|
|
if err := m.messenger.Send(ctx, id, message); err != nil {
|
|
return fmt.Errorf("send %s: %w", id, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CleanupSkip reports one terminal session whose workspace was preserved
|
|
// rather than reclaimed, and why.
|
|
type CleanupSkip struct {
|
|
SessionID domain.SessionID
|
|
Reason string
|
|
}
|
|
|
|
// CleanupResult reports what Cleanup reclaimed and what it preserved.
|
|
type CleanupResult struct {
|
|
Cleaned []domain.SessionID
|
|
Skipped []CleanupSkip
|
|
}
|
|
|
|
// Cleanup reclaims the workspaces of terminal sessions in a project. A workspace
|
|
// whose teardown is refused (uncommitted work) is never forced; it is reported
|
|
// in Skipped with the reason so the refusal is visible instead of silent.
|
|
func (m *Manager) Cleanup(ctx context.Context, project domain.ProjectID) (CleanupResult, error) {
|
|
recs, err := m.cleanupRecords(ctx, project)
|
|
if err != nil {
|
|
return CleanupResult{}, fmt.Errorf("cleanup %s: %w", project, err)
|
|
}
|
|
result := CleanupResult{Cleaned: make([]domain.SessionID, 0, len(recs)), Skipped: []CleanupSkip{}}
|
|
for _, rec := range recs {
|
|
if !rec.IsTerminated {
|
|
continue
|
|
}
|
|
ws := workspaceInfo(rec)
|
|
if ws.Path == "" {
|
|
m.cleanupSystemPromptDir(rec.ID)
|
|
continue
|
|
}
|
|
if h := runtimeHandle(rec.Metadata); h.ID != "" {
|
|
_ = m.runtime.Destroy(ctx, h) // best effort; usually already gone
|
|
}
|
|
if err := m.workspace.Destroy(ctx, ws); err != nil {
|
|
if !errors.Is(err, ports.ErrWorkspaceDirty) {
|
|
// The public reason stays a fixed string (the raw error carries
|
|
// internal filesystem paths); the full cause lands here.
|
|
m.logger.Warn("cleanup: workspace teardown failed", "sessionID", rec.ID, "path", ws.Path, "error", err)
|
|
}
|
|
result.Skipped = append(result.Skipped, CleanupSkip{SessionID: rec.ID, Reason: cleanupSkipReason(err)})
|
|
continue
|
|
}
|
|
m.cleanupSystemPromptDir(rec.ID)
|
|
result.Cleaned = append(result.Cleaned, rec.ID)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// cleanupSkipReason renders a workspace teardown refusal as a short
|
|
// user-facing reason for the cleanup report. Deliberately not the raw error:
|
|
// it flows to the API response and CLI output, and teardown errors embed
|
|
// internal filesystem paths.
|
|
func cleanupSkipReason(err error) string {
|
|
if errors.Is(err, ports.ErrWorkspaceDirty) {
|
|
return "workspace has uncommitted changes"
|
|
}
|
|
return "workspace teardown failed"
|
|
}
|
|
|
|
func (m *Manager) cleanupRecords(ctx context.Context, project domain.ProjectID) ([]domain.SessionRecord, error) {
|
|
if project == "" {
|
|
return m.store.ListAllSessions(ctx)
|
|
}
|
|
return m.store.ListSessions(ctx, project)
|
|
}
|
|
|
|
// ---- helpers ----
|
|
|
|
func seedRecord(cfg ports.SpawnConfig, now time.Time) domain.SessionRecord {
|
|
return domain.SessionRecord{
|
|
ProjectID: cfg.ProjectID,
|
|
IssueID: cfg.IssueID,
|
|
Kind: cfg.Kind,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
Harness: cfg.Harness,
|
|
DisplayName: cfg.DisplayName,
|
|
Activity: domain.Activity{State: domain.ActivityIdle, LastActivityAt: now},
|
|
}
|
|
}
|
|
|
|
func defaultSessionBranch(id domain.SessionID, kind domain.SessionKind, prefix string) string {
|
|
if kind == domain.KindOrchestrator {
|
|
return "ao/" + prefix + "-orchestrator"
|
|
}
|
|
// A fresh, unique branch per worker session: gitworktree can't add a worktree
|
|
// on a branch already checked out elsewhere (e.g. main). Put the root work
|
|
// branch under a session namespace so sibling PR branches such as
|
|
// ao/<session>/<topic> remain valid Git refs.
|
|
return "ao/" + string(id) + "/root"
|
|
}
|
|
|
|
func buildPrompt(cfg ports.SpawnConfig) string {
|
|
return buildTaskPrompt(taskPromptConfig{
|
|
Role: promptRoleForKind(cfg.Kind),
|
|
Prompt: cfg.Prompt,
|
|
IssueID: string(cfg.IssueID),
|
|
IssueContext: cfg.IssueContext,
|
|
})
|
|
}
|
|
|
|
func promptRoleForKind(kind domain.SessionKind) sessionPromptRole {
|
|
switch kind {
|
|
case domain.KindOrchestrator:
|
|
return sessionPromptRoleOrchestrator
|
|
case domain.KindWorker:
|
|
return sessionPromptRoleWorker
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func promptProjectContext(projectID domain.ProjectID, project domain.ProjectRecord) promptProject {
|
|
cfg := project.Config.WithDefaults()
|
|
id := project.ID
|
|
if strings.TrimSpace(id) == "" {
|
|
id = string(projectID)
|
|
}
|
|
return promptProject{
|
|
ID: id,
|
|
Name: project.DisplayName,
|
|
Repo: project.RepoOriginURL,
|
|
DefaultBranch: cfg.DefaultBranch,
|
|
Path: project.Path,
|
|
}
|
|
}
|
|
|
|
// buildSpawnTexts returns the user-facing prompt and the system prompt to
|
|
// deliver separately to the agent. Orchestrator role instructions and worker
|
|
// coordination hints are placed in the system prompt so they are treated as
|
|
// standing instructions rather than part of the human's task request. A
|
|
// promptless spawn delivers no user prompt at all: the agent simply lands at an
|
|
// empty input box rather than receiving an auto-generated kickoff turn.
|
|
func (m *Manager) buildSpawnTexts(ctx context.Context, cfg ports.SpawnConfig) (prompt, systemPrompt string, err error) {
|
|
prompt = buildPrompt(cfg)
|
|
systemPrompt, err = m.buildSystemPrompt(ctx, cfg.Kind, cfg.ProjectID)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
return prompt, systemPrompt, nil
|
|
}
|
|
|
|
// buildSystemPrompt derives the standing instructions for a session of the
|
|
// given kind from current store state. Restore recomputes them through here
|
|
// rather than persisting them, so a restored worker points at the orchestrator
|
|
// that is active now, not the one from its original spawn.
|
|
func (m *Manager) buildSystemPrompt(ctx context.Context, kind domain.SessionKind, projectID domain.ProjectID) (string, error) {
|
|
project, err := m.loadProject(ctx, projectID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
role := promptRoleForKind(kind)
|
|
cfg := systemPromptConfig{
|
|
Role: role,
|
|
Project: promptProjectContext(projectID, project),
|
|
}
|
|
switch kind {
|
|
case domain.KindOrchestrator:
|
|
cfg.OrchestratorRules = project.Config.OrchestratorRules
|
|
case domain.KindWorker:
|
|
orchestratorID, ok, err := m.activeOrchestratorSessionID(ctx, projectID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if ok {
|
|
cfg.OrchestratorSessionID = string(orchestratorID)
|
|
}
|
|
rules, err := buildProjectRules(projectRulesConfig{
|
|
ProjectPath: project.Path,
|
|
AgentRules: project.Config.AgentRules,
|
|
AgentRulesFile: project.Config.AgentRulesFile,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
cfg.ProjectRules = rules
|
|
}
|
|
cfg.AOSkillPointer = m.aoSkillPointer()
|
|
return buildSystemPromptText(cfg), nil
|
|
}
|
|
|
|
// aoSkillPointer is appended to every agent system prompt. It points the agent
|
|
// at the using-ao skill the daemon installs under the data dir, rather than
|
|
// inlining the whole CLI catalog. The path is absolute so it resolves from any
|
|
// project's worktree, not just the AO repo (the only place a repo-relative
|
|
// skills/ path would exist). The skill file carries exact flags and examples,
|
|
// so the standing prompt stays a short pointer rather than a command dump.
|
|
func (m *Manager) aoSkillPointer() string {
|
|
dir := skillassets.Dir(m.dataDir)
|
|
skillFile := filepath.Join(dir, "SKILL.md")
|
|
commandsGlob := filepath.Join(dir, "commands", "*.md")
|
|
return "## Using the ao CLI\n\n" +
|
|
"When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples."
|
|
}
|
|
|
|
func (m *Manager) activeOrchestratorSessionID(ctx context.Context, project domain.ProjectID) (domain.SessionID, bool, error) {
|
|
recs, err := m.store.ListSessions(ctx, project)
|
|
if err != nil {
|
|
return "", false, fmt.Errorf("list sessions for %s: %w", project, err)
|
|
}
|
|
for _, rec := range recs {
|
|
if rec.Kind == domain.KindOrchestrator && !rec.IsTerminated {
|
|
return rec.ID, true, nil
|
|
}
|
|
}
|
|
return "", false, nil
|
|
}
|
|
|
|
func (m *Manager) writeSystemPromptFile(id domain.SessionID, systemPrompt string) (string, error) {
|
|
if systemPrompt == "" || strings.TrimSpace(m.dataDir) == "" {
|
|
return "", nil
|
|
}
|
|
path := filepath.Join(m.systemPromptDir(id), "system.md")
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
return "", err
|
|
}
|
|
if err := os.WriteFile(path, []byte(strings.TrimRight(systemPrompt, "\n")+"\n"), 0o600); err != nil {
|
|
return "", err
|
|
}
|
|
return path, nil
|
|
}
|
|
|
|
func (m *Manager) prepareSystemPromptFile(id domain.SessionID, harness domain.AgentHarness, systemPrompt string) (string, error) {
|
|
path, err := m.writeSystemPromptFile(id, systemPrompt)
|
|
if err == nil || path != "" {
|
|
return path, err
|
|
}
|
|
if systemPromptFileRequired(harness) {
|
|
return "", err
|
|
}
|
|
m.logger.Warn("system prompt file unavailable; falling back to inline system prompt", "session", id, "harness", harness, "err", err)
|
|
return "", nil
|
|
}
|
|
|
|
func systemPromptFileRequired(harness domain.AgentHarness) bool {
|
|
switch harness {
|
|
case domain.HarnessAider,
|
|
domain.HarnessAuggie,
|
|
domain.HarnessOpenCode,
|
|
domain.HarnessCopilot:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (m *Manager) systemPromptDir(id domain.SessionID) string {
|
|
if strings.TrimSpace(m.dataDir) == "" {
|
|
return ""
|
|
}
|
|
return filepath.Join(m.dataDir, "prompts", string(id))
|
|
}
|
|
|
|
func (m *Manager) cleanupSystemPromptDir(id domain.SessionID) {
|
|
dir := m.systemPromptDir(id)
|
|
if dir == "" {
|
|
return
|
|
}
|
|
if err := os.RemoveAll(dir); err != nil {
|
|
m.logger.Warn("system prompt cleanup failed", "session", id, "path", dir, "err", err)
|
|
}
|
|
}
|
|
|
|
// spawnEnv builds the runtime environment: the per-project env vars first, then
|
|
// the AO-internal vars last so they always win (a project cannot override
|
|
// AO_SESSION_ID and friends).
|
|
func spawnEnv(id domain.SessionID, project domain.ProjectID, issue domain.IssueID, dataDir string, projectEnv map[string]string) map[string]string {
|
|
env := make(map[string]string, len(projectEnv)+4)
|
|
for k, v := range projectEnv {
|
|
env[k] = v
|
|
}
|
|
env[EnvSessionID] = string(id)
|
|
env[EnvProjectID] = string(project)
|
|
env[EnvIssueID] = string(issue)
|
|
env[EnvDataDir] = dataDir
|
|
return env
|
|
}
|
|
|
|
// runtimeEnv is spawnEnv plus the hook PATH pin: the session's PATH puts the
|
|
// running daemon's own directory first, so the bare `ao` in workspace hook
|
|
// commands resolves to the daemon that installed them rather than whatever
|
|
// `ao` is first on the inherited PATH (e.g. a legacy CLI without the hooks
|
|
// command, which fails every callback and silently kills activity tracking).
|
|
// When the pin cannot be applied the inherited PATH is kept and a warning is
|
|
// logged so the degradation isn't silent.
|
|
func (m *Manager) runtimeEnv(id domain.SessionID, project domain.ProjectID, issue domain.IssueID, projectEnv map[string]string) map[string]string {
|
|
env := spawnEnv(id, project, issue, m.dataDir, projectEnv)
|
|
path, err := HookPATH(m.executable, os.Getenv, projectEnv)
|
|
if err != nil {
|
|
m.logger.Warn("session PATH not pinned to the daemon binary; `ao hooks` callbacks may resolve to a different ao and activity tracking will stall",
|
|
"session", id, "error", err)
|
|
return env
|
|
}
|
|
env["PATH"] = path
|
|
return env
|
|
}
|
|
|
|
// HookPATH builds the PATH value pinned into a spawned session: the daemon
|
|
// executable's directory prepended to the base PATH (the project's PATH
|
|
// override when set, else the daemon's inherited PATH — matching what the
|
|
// runtime would have exported anyway). An error means the pin cannot be
|
|
// applied: the executable is unresolvable, or is not named "ao", in which case
|
|
// prepending its directory would not change what `ao` resolves to. Exported so
|
|
// the reviewer launcher can pin its pane's PATH the same way.
|
|
func HookPATH(executable func() (string, error), getenv func(string) string, projectEnv map[string]string) (string, error) {
|
|
exe, err := executable()
|
|
if err != nil {
|
|
return "", fmt.Errorf("resolve daemon executable: %w", err)
|
|
}
|
|
name := filepath.Base(exe)
|
|
if runtime.GOOS == "windows" {
|
|
name = strings.TrimSuffix(strings.ToLower(name), ".exe")
|
|
}
|
|
if name != hookBinaryName {
|
|
return "", fmt.Errorf("daemon executable %s is not named %q", exe, hookBinaryName)
|
|
}
|
|
base := projectEnv["PATH"]
|
|
if base == "" {
|
|
base = getenv("PATH")
|
|
}
|
|
dir := filepath.Dir(exe)
|
|
if base == "" {
|
|
return dir, nil
|
|
}
|
|
return dir + string(os.PathListSeparator) + base, nil
|
|
}
|
|
|
|
// provisionWorkspace applies the project's per-workspace setup after the
|
|
// worktree exists: symlink shared files from the project repo, then run any
|
|
// post-create commands. Either failing aborts the spawn so a half-provisioned
|
|
// workspace never launches an agent.
|
|
func (m *Manager) provisionWorkspace(ctx context.Context, project domain.ProjectRecord, workspacePath string) error {
|
|
if err := applySymlinks(project.Path, workspacePath, project.Config.Symlinks); err != nil {
|
|
return err
|
|
}
|
|
return runPostCreate(ctx, workspacePath, project.Config.PostCreate)
|
|
}
|
|
|
|
// applySymlinks links each repo-relative path into the workspace. A source that
|
|
// does not exist is skipped (symlinks are a convenience for optional files like
|
|
// .env); a real link failure aborts. Paths must be repo-relative with no
|
|
// parent traversal (no leading "/", no ".." segment) — a bad path is refused
|
|
// up front so a project config cannot escape the project or workspace tree.
|
|
func applySymlinks(projectPath, workspacePath string, symlinks []string) error {
|
|
for _, rel := range symlinks {
|
|
rel = strings.TrimSpace(rel)
|
|
if rel == "" {
|
|
continue
|
|
}
|
|
clean, err := safeRelPath(rel)
|
|
if err != nil {
|
|
return fmt.Errorf("symlink %q: %w", rel, err)
|
|
}
|
|
source := filepath.Join(projectPath, clean)
|
|
if _, err := os.Stat(source); err != nil {
|
|
continue
|
|
}
|
|
target := filepath.Join(workspacePath, clean)
|
|
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
|
|
return fmt.Errorf("symlink %q: %w", rel, err)
|
|
}
|
|
if _, err := os.Lstat(target); err == nil {
|
|
continue
|
|
}
|
|
if err := os.Symlink(source, target); err != nil {
|
|
return fmt.Errorf("symlink %q: %w", rel, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// safeRelPath confines rel to a repo-relative path: no absolute paths and no
|
|
// ".." segments (before or after Clean). The cleaned form is returned so
|
|
// callers join it against project/workspace roots safely.
|
|
func safeRelPath(rel string) (string, error) {
|
|
if filepath.IsAbs(rel) || strings.HasPrefix(rel, "/") || strings.HasPrefix(rel, `\`) {
|
|
return "", fmt.Errorf("path must be repo-relative")
|
|
}
|
|
clean := filepath.Clean(rel)
|
|
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || clean == "." || clean == "" {
|
|
return "", fmt.Errorf("path must be repo-relative")
|
|
}
|
|
for _, seg := range strings.Split(filepath.ToSlash(clean), "/") {
|
|
if seg == ".." {
|
|
return "", fmt.Errorf("path must be repo-relative")
|
|
}
|
|
}
|
|
return clean, nil
|
|
}
|
|
|
|
// runPostCreate runs each post-create command in the workspace via the platform
|
|
// shell, so OS-agnostic commands like "pnpm install" work. A non-zero exit
|
|
// aborts the spawn with the command output.
|
|
func runPostCreate(ctx context.Context, workspacePath string, commands []string) error {
|
|
for _, command := range commands {
|
|
command = strings.TrimSpace(command)
|
|
if command == "" {
|
|
continue
|
|
}
|
|
var cmd *exec.Cmd
|
|
if runtime.GOOS == "windows" {
|
|
cmd = aoprocess.CommandContext(ctx, "cmd", "/c", command)
|
|
} else {
|
|
cmd = aoprocess.CommandContext(ctx, "sh", "-c", command)
|
|
}
|
|
cmd.Dir = workspacePath
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("postCreate %q: %w: %s", command, err, strings.TrimSpace(string(out)))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// preLauncher is an optional Agent capability: a step the manager runs before
|
|
// launch. Claude Code implements it to record workspace trust in ~/.claude.json
|
|
// so its interactive "do you trust this folder?" dialog can't block the headless
|
|
// pane. Adapters that don't need it simply omit the method.
|
|
type preLauncher interface {
|
|
PreLaunch(ctx context.Context, cfg ports.LaunchConfig) error
|
|
}
|
|
|
|
// prepareWorkspace runs the per-session pre-launch steps before the runtime
|
|
// starts the agent: installing the workspace-local activity hooks (so early
|
|
// startup hooks can update the already-created session row), then any optional
|
|
// PreLaunch step. Shared by Spawn and Restore.
|
|
func (m *Manager) prepareWorkspace(ctx context.Context, agent ports.Agent, id domain.SessionID, workspacePath string) error {
|
|
if err := agent.GetAgentHooks(ctx, ports.WorkspaceHookConfig{
|
|
SessionID: string(id),
|
|
WorkspacePath: workspacePath,
|
|
DataDir: m.dataDir,
|
|
}); err != nil {
|
|
return fmt.Errorf("install hooks: %w", err)
|
|
}
|
|
if pl, ok := agent.(preLauncher); ok {
|
|
if err := pl.PreLaunch(ctx, ports.LaunchConfig{SessionID: string(id), WorkspacePath: workspacePath}); err != nil {
|
|
return fmt.Errorf("pre-launch: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// restoreArgv builds the argv to relaunch a torn-down session: the agent's
|
|
// native resume command when it can continue the session, else a fresh launch.
|
|
// The agent signals via ok=false (e.g. no native session id captured yet).
|
|
// Returns ErrNotResumable only for a promptless, unresumable non-orchestrator:
|
|
// a worker with no prompt and no native session id has nothing to restore from.
|
|
// Orchestrators are promptless by design and always relaunch fresh with the
|
|
// system prompt only.
|
|
func restoreArgv(ctx context.Context, agent ports.Agent, id domain.SessionID, workspacePath string, meta domain.SessionMetadata, systemPrompt, systemPromptFile string, agentConfig ports.AgentConfig, kind domain.SessionKind) ([]string, error) {
|
|
ref := ports.SessionRef{
|
|
ID: string(id),
|
|
WorkspacePath: workspacePath,
|
|
Metadata: map[string]string{ports.MetadataKeyAgentSessionID: meta.AgentSessionID},
|
|
}
|
|
cmd, ok, err := agent.GetRestoreCommand(ctx, ports.RestoreConfig{Session: ref, SystemPrompt: systemPrompt, SystemPromptFile: systemPromptFile, Config: agentConfig, Permissions: agentConfig.Permissions})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("restore command: %w", err)
|
|
}
|
|
if ok {
|
|
return cmd, nil
|
|
}
|
|
// Adapter cannot resume. A saved prompt is replayed fresh. An orchestrator is
|
|
// promptless by design and relaunches with the system prompt only. A promptless
|
|
// WORKER has no task and no session id to restore from: do not blank-relaunch it.
|
|
if meta.Prompt == "" && kind != domain.KindOrchestrator {
|
|
return nil, ErrNotResumable
|
|
}
|
|
// Fall through to GetLaunchCommand (replays meta.Prompt; empty for an orchestrator).
|
|
argv, err := agent.GetLaunchCommand(ctx, ports.LaunchConfig{
|
|
SessionID: string(id),
|
|
WorkspacePath: workspacePath,
|
|
Prompt: meta.Prompt,
|
|
SystemPrompt: systemPrompt,
|
|
SystemPromptFile: systemPromptFile,
|
|
Config: agentConfig,
|
|
Permissions: agentConfig.Permissions,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("launch command: %w", err)
|
|
}
|
|
return argv, nil
|
|
}
|
|
|
|
// validateAgentBinary checks that argv[0] resolves via the manager's
|
|
// lookPath (exec.LookPath in prod) before any runtime work happens. Adapters
|
|
// that can't resolve their binary now return ports.ErrAgentBinaryNotFound from
|
|
// GetLaunchCommand directly; this guard is a defense-in-depth for adapters
|
|
// that return an argv[0] like "claude" without verifying. Some adapters prefix
|
|
// their command with `env KEY=value`; in that case validate the first real
|
|
// executable after the environment assignments.
|
|
func (m *Manager) validateAgentBinary(argv []string) error {
|
|
if len(argv) == 0 {
|
|
return fmt.Errorf("agent: empty launch argv: %w", ports.ErrAgentBinaryNotFound)
|
|
}
|
|
bin, ok := launchBinary(argv)
|
|
if !ok {
|
|
return fmt.Errorf("agent: launch argv missing binary: %w", ports.ErrAgentBinaryNotFound)
|
|
}
|
|
if _, err := m.lookPath(bin); err != nil {
|
|
return fmt.Errorf("agent binary %q: %w", bin, ports.ErrAgentBinaryNotFound)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func launchBinary(argv []string) (string, bool) {
|
|
if len(argv) == 0 {
|
|
return "", false
|
|
}
|
|
if filepath.Base(argv[0]) != "env" {
|
|
return argv[0], true
|
|
}
|
|
for _, arg := range argv[1:] {
|
|
if strings.Contains(arg, "=") {
|
|
continue
|
|
}
|
|
return arg, true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func (m *Manager) validateRuntimePrerequisites() error {
|
|
if runtime.GOOS == "windows" {
|
|
return nil
|
|
}
|
|
if path, err := m.lookPath("tmux"); err != nil || path == "" {
|
|
return fmt.Errorf("%w: tmux required on macOS/Linux but not in PATH", ports.ErrRuntimePrerequisite)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func runtimeHandle(meta domain.SessionMetadata) ports.RuntimeHandle {
|
|
return ports.RuntimeHandle{ID: meta.RuntimeHandleID}
|
|
}
|
|
|
|
func workspaceInfo(rec domain.SessionRecord) ports.WorkspaceInfo {
|
|
return ports.WorkspaceInfo{
|
|
Path: rec.Metadata.WorkspacePath,
|
|
Branch: rec.Metadata.Branch,
|
|
SessionID: rec.ID,
|
|
ProjectID: rec.ProjectID,
|
|
}
|
|
}
|