* test(session_manager): prove sessions survive daemon restart and re-adopt (#2335) Add an end-to-end Reconcile() durability test covering the full mix of session states a daemon restart/upgrade leaves behind: an alive orchestrator and worker are adopted in place under their original ids (no id increment, runtime never torn down), a worker whose runtime died with the daemon is captured and relaunched under its original id, and a truly-dead unmarked session is not resurrected. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * chore: format with prettier [skip ci] --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
parent
57ba468fea
commit
3aa8e044b2
|
|
@ -1867,6 +1867,94 @@ func TestReconcileLive_ProbeErrorIsNotDeath(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestReconcile_AdoptAcrossDaemonRestart is the end-to-end durability proof for
|
||||||
|
// #2335: it drives the full boot-time Reconcile pass over the exact mix of
|
||||||
|
// session states a daemon restart/upgrade leaves behind and asserts agent
|
||||||
|
// sessions are decoupled from the daemon's lifetime:
|
||||||
|
//
|
||||||
|
// - an alive orchestrator is ADOPTED in place: same id, still live, its runtime
|
||||||
|
// never torn down, and NO new session minted (the id-increment regression
|
||||||
|
// guard: adoption failure used to mint a fresh orchestrator id 14->15->16).
|
||||||
|
// - an alive worker is adopted as a no-op.
|
||||||
|
// - a worker whose runtime died with the daemon has its work captured (stashed
|
||||||
|
// into a preserve ref, restore marker written) and is relaunched on this same
|
||||||
|
// boot under its ORIGINAL id.
|
||||||
|
// - a truly-dead session with no restore marker is NOT resurrected.
|
||||||
|
func TestReconcile_AdoptAcrossDaemonRestart(t *testing.T) {
|
||||||
|
st := newFakeStore()
|
||||||
|
st.projects["mer"] = domain.ProjectRecord{ID: "mer", Config: testRoleAgents()}
|
||||||
|
rt := &fakeRuntime{aliveByHandle: map[string]bool{
|
||||||
|
"orch": true, // orchestrator runtime survived the daemon exit
|
||||||
|
"w-alive": true, // worker runtime survived the daemon exit
|
||||||
|
// "w-dead" is absent -> that worker's runtime died with the daemon.
|
||||||
|
}}
|
||||||
|
ws := &fakeWorkspace{stashRef: "refs/ao/preserved/mer-3"}
|
||||||
|
lcm := &fakeLCM{store: st}
|
||||||
|
lookPath := func(string) (string, error) { return "/bin/true", nil }
|
||||||
|
m := New(Deps{Runtime: rt, Agents: fakeAgents{}, Workspace: ws, Store: st, Messenger: &fakeMessenger{}, Lifecycle: lcm, LookPath: lookPath})
|
||||||
|
|
||||||
|
// Alive orchestrator: the promptless session whose adoption failure used to
|
||||||
|
// mint a fresh orchestrator id. It must be adopted in place.
|
||||||
|
st.sessions["mer-1"] = domain.SessionRecord{
|
||||||
|
ID: "mer-1", ProjectID: "mer", Kind: domain.KindOrchestrator, Harness: domain.HarnessClaudeCode,
|
||||||
|
Metadata: domain.SessionMetadata{Branch: "ao/mer-1/root", WorkspacePath: "/ws/mer-1", RuntimeHandleID: "orch"},
|
||||||
|
}
|
||||||
|
// Alive worker: adopted as a no-op.
|
||||||
|
st.sessions["mer-2"] = domain.SessionRecord{
|
||||||
|
ID: "mer-2", ProjectID: "mer", Kind: domain.KindWorker, Harness: domain.HarnessClaudeCode,
|
||||||
|
Metadata: domain.SessionMetadata{Branch: "ao/mer-2/root", WorkspacePath: "/ws/mer-2", RuntimeHandleID: "w-alive", AgentSessionID: "agent-2"},
|
||||||
|
}
|
||||||
|
// Dead worker: its runtime died with the daemon; capture + relaunch under same id.
|
||||||
|
st.sessions["mer-3"] = domain.SessionRecord{
|
||||||
|
ID: "mer-3", ProjectID: "mer", Kind: domain.KindWorker, Harness: domain.HarnessClaudeCode,
|
||||||
|
Metadata: domain.SessionMetadata{Branch: "ao/mer-3/root", WorkspacePath: "/ws/mer-3", RuntimeHandleID: "w-dead", AgentSessionID: "agent-3"},
|
||||||
|
}
|
||||||
|
// Truly-dead session the user killed before restart (terminated, no marker).
|
||||||
|
st.sessions["mer-4"] = domain.SessionRecord{
|
||||||
|
ID: "mer-4", ProjectID: "mer", Kind: domain.KindWorker, Harness: domain.HarnessClaudeCode,
|
||||||
|
IsTerminated: true, Activity: domain.Activity{State: domain.ActivityExited},
|
||||||
|
Metadata: domain.SessionMetadata{Branch: "ao/mer-4/root", WorkspacePath: "/ws/mer-4"},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := m.Reconcile(ctx); err != nil {
|
||||||
|
t.Fatalf("Reconcile: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Alive orchestrator + worker adopted in place: same id, still live.
|
||||||
|
if st.sessions["mer-1"].IsTerminated {
|
||||||
|
t.Fatal("alive orchestrator must be adopted in place, not terminated")
|
||||||
|
}
|
||||||
|
if st.sessions["mer-2"].IsTerminated {
|
||||||
|
t.Fatal("alive worker must be adopted in place, not terminated")
|
||||||
|
}
|
||||||
|
// No id increment: Reconcile must never mint a new session row.
|
||||||
|
if st.num != 0 {
|
||||||
|
t.Fatalf("Reconcile minted %d new session(s); adoption must reuse existing ids", st.num)
|
||||||
|
}
|
||||||
|
// Adopted runtimes were never torn down.
|
||||||
|
if rt.destroyed != 0 {
|
||||||
|
t.Fatalf("adopted sessions must not be destroyed; Destroy called %d times", rt.destroyed)
|
||||||
|
}
|
||||||
|
// Dead worker captured, then relaunched under its original id on this same boot.
|
||||||
|
if lcm.terminated["mer-3"] != 1 {
|
||||||
|
t.Fatalf("dead worker must be marked terminated once before relaunch; got %d", lcm.terminated["mer-3"])
|
||||||
|
}
|
||||||
|
if st.sessions["mer-3"].IsTerminated {
|
||||||
|
t.Fatal("dead worker must be relaunched (not terminated) after Reconcile")
|
||||||
|
}
|
||||||
|
if rt.created != 1 {
|
||||||
|
t.Fatalf("exactly one runtime relaunch expected (the dead worker); got %d", rt.created)
|
||||||
|
}
|
||||||
|
// One-shot restore marker consumed so it never outlives one restart (#2319).
|
||||||
|
if rows := st.worktrees["mer-3"]; len(rows) != 0 {
|
||||||
|
t.Fatalf("restore marker for mer-3 must be deleted after relaunch; got %+v", rows)
|
||||||
|
}
|
||||||
|
// Truly-dead, unmarked session is NOT resurrected.
|
||||||
|
if !st.sessions["mer-4"].IsTerminated {
|
||||||
|
t.Fatal("terminated session with no restore marker must stay terminated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestReconcileReap_TerminatedButAliveTmuxDestroyed(t *testing.T) {
|
func TestReconcileReap_TerminatedButAliveTmuxDestroyed(t *testing.T) {
|
||||||
st := newFakeStore()
|
st := newFakeStore()
|
||||||
rt := &fakeRuntime{aliveByHandle: map[string]bool{"t1": true}}
|
rt := &fakeRuntime{aliveByHandle: map[string]bool{"t1": true}}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue