bkinnightskytw
/
agent-orchestrator
mirror of https://github.com/ComposioHQ/agent-orchestrator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(cli): port the E2E suite to cross-platform Go; slim the Docker harness 

Replaces the growing bash smoke test with a Go os/exec suite behind the `e2e`
build tag (backend/internal/cli/e2e_test.go). It builds the real binary and
drives start/status/doctor/stop + the daemon-control HTTP surface against
isolated state (temp dir + OS-assigned free port), and now runs natively on
ubuntu + macOS + WINDOWS in CI — finally covering the Windows
CREATE_NEW_PROCESS_GROUP detach path and per-OS os.UserConfigDir resolution
that a Linux container can't observe. `go test -tags e2e -v` logs every command
and its output, replacing the bash -v flag.

- backend/internal/cli/e2e_test.go: 8 table-style TestE2E_* cases; strips any
  inherited AO_* env so a real daemon's AO_PORT can't leak in.
- test/cli/install-check.sh: small, linear fresh-install proof the Dockerfile
  runs (binary on PATH, no toolchain) — kept as the hardening tier.
- test/cli/Dockerfile: run install-check.sh instead of the full bash suite.
- .github/workflows/cli-e2e.yml: `native` is now a go test matrix over
  ubuntu+macos+windows; `container` builds the image and runs it with --init.
- Removes test/cli/smoke.sh and test/cli/run-local.sh (superseded by `go test`).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
itrytoohard 2026-06-01 03:14:08 +05:30
parent 4f13dd1b83
commit c0bf99eb22
7 changed files with 448 additions and 446 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.github/workflows/cli-e2e.yml vendored
View File

@ -13,17 +13,20 @@ permissions:
contents: read
jobs:
# Primary tier: run the REAL `ao` binary on GitHub's native VM runners. These
# runners are the "VMs" — the only place that exercises the OS-specific code
# paths (unix Setsid process-group detach + macOS os.UserConfigDir resolution).
# Bash is available on both ubuntu and macos runners, so the one smoke.sh runs
# unchanged. State is isolated per run (own temp dir + a free loopback port).
# Primary tier: the cross-platform Go E2E suite (build tag `e2e`) runs the real
# `ao` binary against isolated state on every OS GitHub hosts. These runners
# are the "VMs" — the only place that exercises the OS-specific process-detach
# paths (unix Setsid vs Windows CREATE_NEW_PROCESS_GROUP) and os.UserConfigDir
# resolution. The suite builds its own binary and self-allocates a free port.
native:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
defaults:
run:
working-directory: backend
steps:
- uses: actions/checkout@v4
@ -32,25 +35,21 @@ jobs:
go-version: "1.25"
cache: false
- name: Build ao
run: cd backend && CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/ao" ./cmd/ao
- name: CLI E2E (native)
run: go test -tags e2e -v ./internal/cli/...
- name: CLI smoke test
run: AO_BIN="$RUNNER_TEMP/ao" bash test/cli/smoke.sh
# Secondary hardening tier: model "install ao on a fresh machine" in a clean,
# locked-down Linux container with no access to a developer's real state.
# --init gives the container a real PID-1 reaper (tini) so the live daemon the
# `start` test spawns (it detaches via setsid) is reaped after `stop` rather
# than lingering as a zombie. The suite doesn't depend on it (the stale case
# uses a fabricated dead PID), but it keeps process accounting clean.
# Secondary hardening tier: prove that a freshly installed binary works on a
# clean machine with no Go toolchain and no developer state. The Dockerfile
# installs `ao` on PATH in a slim image and runs test/cli/install-check.sh.
# --init gives a real PID-1 reaper so the daemon the check starts is reaped
# after `stop` instead of lingering as a zombie.
container:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build smoke image (fresh-machine install)
- name: Build fresh-install image
run: docker build -f test/cli/Dockerfile -t ao-cli-smoke .
- name: Run CLI smoke test in container
- name: Fresh-install check (container)
run: docker run --rm --init ao-cli-smoke

346
backend/internal/cli/e2e_test.go Normal file
View File

@ -0,0 +1,346 @@
//go:build e2e
// Package cli_test holds the end-to-end suite for the `ao` CLI. It builds the
// real binary and drives it (start/status/doctor/stop + the daemon-control HTTP
// surface) against fully isolated state — a per-test temp run-file, data dir,
// and an OS-assigned free loopback port — so it never touches a developer's real
// AO install. Unlike the Linux-only container smoke test, this runs natively on
// every OS in CI (ubuntu/macos/windows), which is the only way to exercise the
// unix setsid vs Windows CREATE_NEW_PROCESS_GROUP detach paths and the per-OS
// os.UserConfigDir resolution.
//
// It is gated behind the `e2e` build tag so it never runs in the normal
// `go test ./...` lane (it spawns processes and binds ports):
//
// go test -tags e2e ./internal/cli/... # run it
// go test -tags e2e -v -run TestE2E ./internal/cli/... # verbose, see every command
package cli_test
import (
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// aoBin is the path to the binary built once for the whole suite.
var aoBin string
func TestMain(m *testing.M) {
dir, err := os.MkdirTemp("", "ao-e2e-bin")
if err != nil {
fmt.Fprintln(os.Stderr, "e2e: mktemp:", err)
os.Exit(1)
}
aoBin = filepath.Join(dir, "ao")
if runtime.GOOS == "windows" {
aoBin += ".exe"
}
build := exec.Command("go", "build", "-o", aoBin, "github.com/aoagents/agent-orchestrator/backend/cmd/ao")
build.Stdout, build.Stderr = os.Stderr, os.Stderr
if err := build.Run(); err != nil {
fmt.Fprintln(os.Stderr, "e2e: build ao:", err)
os.Exit(1)
}
code := m.Run()
_ = os.RemoveAll(dir)
os.Exit(code)
}
// env is an isolated CLI environment: its own state files and free port.
type env struct {
runFile string
dataDir string
port int
}
func newEnv(t *testing.T) env {
t.Helper()
dir := t.TempDir()
return env{
runFile: filepath.Join(dir, "running.json"),
dataDir: filepath.Join(dir, "data"),
port: freePort(t),
}
}
// environ builds the child env: the ambient environment with every inherited
// AO_* var stripped (so a real daemon's AO_PORT can't leak in) plus our isolated
// settings. portOverride, when non-empty, replaces the numeric AO_PORT — used to
// inject an invalid value.
func (e env) environ(portOverride string) []string {
out := make([]string, 0, len(os.Environ())+3)
for _, kv := range os.Environ() {
if strings.HasPrefix(kv, "AO_") {
continue
}
out = append(out, kv)
}
port := fmt.Sprintf("%d", e.port)
if portOverride != "" {
port = portOverride
}
return append(out, "AO_RUN_FILE="+e.runFile, "AO_DATA_DIR="+e.dataDir, "AO_PORT="+port)
}
func freePort(t *testing.T) int {
t.Helper()
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("alloc free port: %v", err)
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port
}
// run executes `ao args...` in env e and returns combined output + exit code.
func (e env) run(t *testing.T, args ...string) (string, int) {
t.Helper()
return e.runEnv(t, e.environ(""), args...)
}
func (e env) runEnv(t *testing.T, environ []string, args ...string) (string, int) {
t.Helper()
cmd := exec.Command(aoBin, args...)
cmd.Env = environ
b, err := cmd.CombinedOutput()
out := string(b)
code := 0
if err != nil {
var ee *exec.ExitError
if asExit(err, &ee) {
code = ee.ExitCode()
} else {
t.Fatalf("run %v: %v\n%s", args, err, out)
}
}
t.Logf("$ ao %s\n%s(exit %d)", strings.Join(args, " "), out, code)
return out, code
}
func asExit(err error, target **exec.ExitError) bool {
if ee, ok := err.(*exec.ExitError); ok {
*target = ee
return true
}
return false
}
// startDaemon brings the daemon up and registers a stop on cleanup.
func (e env) startDaemon(t *testing.T) {
t.Helper()
out, code := e.run(t, "start")
if code != 0 {
t.Fatalf("start failed (exit %d): %s", code, out)
}
t.Cleanup(func() { e.run(t, "stop") })
}
func mustContain(t *testing.T, out, want string) {
t.Helper()
if !strings.Contains(out, want) {
t.Fatalf("expected output to contain %q; got:\n%s", want, out)
}
}
func mustNotContain(t *testing.T, out, notWant string) {
t.Helper()
if strings.Contains(out, notWant) {
t.Fatalf("expected output NOT to contain %q; got:\n%s", notWant, out)
}
}
// ---------------------------------------------------------------------------
func TestE2E_VersionAndHelp(t *testing.T) {
e := newEnv(t)
if out, code := e.run(t, "version"); code != 0 || strings.TrimSpace(out) == "" {
t.Fatalf("version: exit %d, out %q", code, out)
}
if _, code := e.run(t, "--version"); code != 0 {
t.Fatalf("--version exit %d", code)
}
out, code := e.run(t, "--help")
if code != 0 {
t.Fatalf("--help exit %d", code)
}
for _, want := range []string{"start", "stop", "status", "doctor", "completion", "version"} {
mustContain(t, out, want)
}
// the internal daemon command is hidden from help (rendered as "\n daemon")
mustNotContain(t, out, "\n daemon")
}
func TestE2E_DoctorDoesNotTouchTheStore(t *testing.T) {
e := newEnv(t)
out, code := e.run(t, "doctor")
if code != 0 {
t.Fatalf("doctor (fresh) exit %d: %s", code, out)
}
mustContain(t, out, "git")
mustContain(t, out, "database not created yet") // sqlite WARN, never migrated
// doctor must NOT create/migrate the DB — the daemon is the sole writer.
if _, err := os.Stat(filepath.Join(e.dataDir, "ao.db")); err == nil {
t.Fatal("doctor created ao.db; the CLI must not open/migrate the store")
}
if out, code := e.run(t, "doctor", "--json"); code != 0 || !strings.Contains(out, `"ok": true`) {
t.Fatalf("doctor --json: exit %d, out %s", code, out)
}
}
func TestE2E_StatusStopped(t *testing.T) {
e := newEnv(t)
out, code := e.run(t, "status", "--json")
if code != 0 { // status always exits 0
t.Fatalf("status exit %d", code)
}
mustContain(t, out, `"state": "stopped"`)
mustNotContain(t, out, "startedAt")
if out, code := e.run(t, "stop"); code != 0 || !strings.Contains(out, "stopped") {
t.Fatalf("stop-when-stopped: exit %d, out %s", code, out) // idempotent
}
}
func TestE2E_Lifecycle(t *testing.T) {
e := newEnv(t)
e.startDaemon(t)
out, _ := e.run(t, "status", "--json")
mustContain(t, out, `"state": "ready"`)
mustContain(t, out, fmt.Sprintf(`"port": %d`, e.port))
// idempotent
if out, code := e.run(t, "start"); code != 0 || !strings.Contains(out, "ready") {
t.Fatalf("idempotent start: exit %d, out %s", code, out)
}
// now the daemon (not the CLI) has created + migrated the store
if _, err := os.Stat(filepath.Join(e.dataDir, "ao.db")); err != nil {
t.Fatalf("daemon should have created ao.db: %v", err)
}
out, _ = e.run(t, "doctor")
mustContain(t, out, "migrations are applied by the daemon")
// /healthz identity
body := httpGet(t, e.port, "/healthz")
mustContain(t, body, "agent-orchestrator-daemon")
if out, code := e.run(t, "stop"); code != 0 || !strings.Contains(out, "stopped") {
t.Fatalf("stop: exit %d, out %s", code, out)
}
if _, err := os.Stat(e.runFile); !os.IsNotExist(err) {
t.Fatal("run-file should be removed after stop")
}
}
func TestE2E_ShutdownGuard(t *testing.T) {
e := newEnv(t)
e.startDaemon(t)
// A cross-site Origin header must be rejected without stopping the daemon.
if code := postShutdown(t, e.port, func(r *http.Request) { r.Header.Set("Origin", "https://evil.example") }); code != http.StatusForbidden {
t.Fatalf("cross-origin /shutdown = %d, want 403", code)
}
// A non-loopback Host (DNS-rebinding) must be rejected too.
if code := postShutdown(t, e.port, func(r *http.Request) { r.Host = "evil.example" }); code != http.StatusForbidden {
t.Fatalf("rebinding-host /shutdown = %d, want 403", code)
}
// The daemon survived both.
out, _ := e.run(t, "status", "--json")
mustContain(t, out, `"state": "ready"`)
}
func TestE2E_StaleRunFile(t *testing.T) {
e := newEnv(t)
// PID 2147483647 is never alive -> the CLI must classify this as stale.
content := fmt.Sprintf(`{"pid":2147483647,"port":%d,"startedAt":"2020-01-01T00:00:00Z"}`, e.port)
if err := os.MkdirAll(filepath.Dir(e.runFile), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(e.runFile, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
out, _ := e.run(t, "status", "--json")
mustContain(t, out, `"state": "stale"`)
if out, code := e.run(t, "stop"); code != 0 || !strings.Contains(out, "stopped") {
t.Fatalf("stop stale: exit %d, out %s", code, out)
}
if _, err := os.Stat(e.runFile); !os.IsNotExist(err) {
t.Fatal("stale run-file should be removed")
}
}
func TestE2E_ExitCodes(t *testing.T) {
e := newEnv(t)
if _, code := e.run(t, "status", "--definitely-not-a-flag"); code != 2 {
t.Fatalf("bad flag exit %d, want 2", code)
}
if _, code := e.run(t, "completion"); code != 2 { // missing required arg
t.Fatalf("missing-arg exit %d, want 2", code)
}
if _, code := e.run(t, "completion", "notashell"); code == 0 { // runtime error
t.Fatal("unsupported shell should be non-zero")
}
// invalid config is a runtime error (1), not a usage error (2).
if _, code := e.runEnv(t, e.environ("notaport"), "status"); code != 1 {
t.Fatalf("invalid AO_PORT exit %d, want 1", code)
}
}
func TestE2E_Completion(t *testing.T) {
e := newEnv(t)
for _, sh := range []string{"bash", "zsh", "fish", "powershell"} {
out, code := e.run(t, "completion", sh)
if code != 0 || strings.TrimSpace(out) == "" {
t.Fatalf("completion %s: exit %d, empty=%v", sh, code, strings.TrimSpace(out) == "")
}
}
}
// ---------------------------------------------------------------------------
// HTTP helpers (loopback)
func httpClient() *http.Client { return &http.Client{Timeout: 3 * time.Second} }
func httpGet(t *testing.T, port int, path string) string {
t.Helper()
resp, err := httpClient().Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, path))
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
b := make([]byte, 4096)
n, _ := resp.Body.Read(b)
return string(b[:n])
}
// postShutdown issues POST /shutdown with mutator applied, returns the status code.
func postShutdown(t *testing.T, port int, mutate func(*http.Request)) int {
t.Helper()
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/shutdown", port), nil)
if err != nil {
t.Fatal(err)
}
mutate(req)
resp, err := httpClient().Do(req)
if err != nil {
t.Fatalf("POST /shutdown: %v", err)
}
defer resp.Body.Close()
return resp.StatusCode
}

8
test/cli/Dockerfile
View File

@ -33,15 +33,15 @@ RUN apt-get update \
# "Install" the CLI the way a user would: drop the binary on PATH.
COPY --from=build /out/ao /usr/local/bin/ao
COPY test/cli/smoke.sh /usr/local/bin/ao-smoke.sh
RUN chmod +x /usr/local/bin/ao /usr/local/bin/ao-smoke.sh
COPY test/cli/install-check.sh /usr/local/bin/ao-install-check.sh
RUN chmod +x /usr/local/bin/ao /usr/local/bin/ao-install-check.sh
# Run as an unprivileged user with a real HOME, like a normal install.
RUN useradd --create-home --shell /bin/bash ao
USER ao
WORKDIR /home/ao
# Sanity: prove the install resolved before the suite runs.
# Sanity: prove the install resolved before the check runs.
RUN ao version
ENTRYPOINT ["/usr/local/bin/ao-smoke.sh"]
ENTRYPOINT ["/usr/local/bin/ao-install-check.sh"]

98
test/cli/README.md
View File

@ -1,75 +1,65 @@
# `ao` CLI end-to-end tests
These tests install and drive the **real `ao` binary** the way a user would —
`start` → `status` → `doctor` → `stop`, plus the daemon-control HTTP surface —
and assert the whole thing works end to end. They run against **isolated,
throwaway state** (their own temp run-file + data dir + a free loopback port),
so they never touch a developer's real AO installation.
These tests drive the **real `ao` binary** the way a user would — `start` →
`status` → `doctor` → `stop`, plus the daemon-control HTTP surface — and assert
the whole thing works. They run against **isolated, throwaway state** (a per-test
temp run-file + data dir + an OS-assigned free loopback port), so they never
touch a developer's real AO installation.
## Files
## Two tiers
| File | Purpose |
|------|---------|
| `smoke.sh` | The suite. Host-agnostic bash; drives the binary at `$AO_BIN` (default `ao` on PATH) and prints a PASS/FAIL line per assertion. |
| `Dockerfile` | Models **installing `ao` on a fresh machine**: builds the binary, drops it on `PATH` in a clean Debian image with only runtime deps (`git`, `tmux`, `curl`), then runs `smoke.sh` as a non-root user. |
| `run-local.sh` | Convenience wrapper: build from source and run `smoke.sh` natively against a temp binary. |
| Tier | What | Where |
|------|------|-------|
| **Comprehensive (primary)** | A cross-platform Go suite that builds `ao` and exercises the full behaviour. Runs natively on **ubuntu + macOS + windows** — the only way to cover the OS-specific process-detach paths (`setsid` vs `CREATE_NEW_PROCESS_GROUP`) and `os.UserConfigDir()` resolution. | `backend/internal/cli/e2e_test.go` (build tag `e2e`) |
| **Fresh-install (hardening)** | Proves a freshly installed binary works on a clean machine with no Go toolchain and no developer state. | `test/cli/Dockerfile` + `test/cli/install-check.sh` |
## Run it
**Native (fastest, uses your toolchain):**
**The Go suite (fastest, cross-platform):**
```bash
test/cli/run-local.sh # PASS/FAIL per assertion
test/cli/run-local.sh -v # also print every command and its full output
# or, against a binary you already built:
AO_BIN=/path/to/ao test/cli/smoke.sh [-v]
cd backend
go test -tags e2e ./internal/cli/... # run it
go test -tags e2e -v -run TestE2E ./internal/cli/... # verbose: prints every command + output
```
It builds its own `ao` binary; `git` must be on PATH (required by `doctor`).
`-v` logs each `ao` invocation and its full output, which is the audit trail you
get for free from `go test`.
**Fresh-machine install, in a clean container:**
```bash
docker build -f test/cli/Dockerfile -t ao-cli-smoke .
docker run --rm --init ao-cli-smoke
```
> `--init` gives the container a real PID-1 reaper (tini) so the live daemon
> spawned during the `start` test is reaped after `stop` instead of lingering as
> a zombie. The suite itself doesn't depend on it — the stale-daemon case uses a
> fabricated dead PID — but it keeps process accounting clean.
> `--init` gives the container a real PID-1 reaper (tini) so the daemon the
> check starts is reaped after `stop` instead of lingering as a zombie.
## What it covers
## What the Go suite covers
Install resolves on PATH · `version`/`--version` · `--help` (and hides the
internal `daemon` command) · `doctor` text + `--json` (and that it **does not**
open/migrate SQLite) · `status` stopped/stale/ready · `start` (fresh +
idempotent) · daemon-created store · `/healthz` identity · the `/shutdown`
CSRF/DNS-rebinding guard (403 + daemon survives) · `stop` (graceful + stale +
idempotent) · run-file cleanup/ownership · exit codes (`2` usage, `1` runtime) ·
completion for all four shells.
`TestE2E_VersionAndHelp` (version/`--version`/help, daemon hidden) ·
`TestE2E_DoctorDoesNotTouchTheStore` (doctor text + `--json`; proves it does
**not** create/migrate `ao.db`) · `TestE2E_StatusStopped` (stopped + idempotent
stop) · `TestE2E_Lifecycle` (start, ready, idempotent, daemon-created store,
`/healthz` identity, stop, run-file cleanup) · `TestE2E_ShutdownGuard` (the
`/shutdown` CSRF + DNS-rebinding 403 guard, daemon survives) ·
`TestE2E_StaleRunFile` (dead-PID run-file → stale → cleaned) · `TestE2E_ExitCodes`
(2 usage / 1 runtime / config error) · `TestE2E_Completion` (all four shells).
## Testing strategy — why it's shaped this way
## Why a Go suite (not bash, not Python)
We deliberately don't make Docker the *only* tier. A daemon that detaches with
`setsid` and outlives the launching process is exactly the workload that
container PID-1 semantics mishandle, and the OS-specific bits (`setsid` vs
Windows `CREATE_NEW_PROCESS_GROUP`, and `os.UserConfigDir()` resolving to
`~/Library/Application Support` on macOS, `%AppData%` on Windows, `~/.config`
on Linux) can't be observed from a Linux container at all.
The bash version grew past the point where bash was a good fit, and a Linux
container can't observe the macOS/Windows code paths at all. A Go `os/exec`
suite is the right home: it uses the repo's own toolchain (runs under `go test`),
gives real assertions and structured data, and — critically — runs natively on
the Windows and macOS runners, finally covering the `CREATE_NEW_PROCESS_GROUP`
detach path and per-OS config-dir resolution. The container stays as a thin
"clean install actually works" check.
So CI (`.github/workflows/cli-e2e.yml`) runs two tiers:
## Extending
1. **`native`** — the primary signal. Builds and runs the real binary on a
GitHub matrix of `ubuntu-latest` + `macos-latest` (those runners *are* the
VMs), covering the unix detach path and macOS config-dir resolution.
2. **`container`** — a hardening tier. The `Dockerfile` proves a clean-machine
install works and that the CLI has no hidden dependence on developer state,
run with `--init`.
### Extending
- Add an assertion: drop a `step`/`assert_*` pair into the relevant section of
`smoke.sh`. The helpers (`assert_eq`, `assert_contains`, `assert_not_contains`,
`run_rc`) keep cases one-liners.
- Cover Windows: add a `windows-latest` leg to the `native` matrix (Git Bash
ships on the runner) once the suite is confirmed green there, or add Go-based
`os/exec` E2E tests for the Windows process-group path.
- Deeper per-OS path assertions (that state resolves under the OS-native config
dir when `AO_RUN_FILE`/`AO_DATA_DIR` are unset) are best added as Go unit
tests in `internal/config`.
- **Add a case:** a new `TestE2E_*` function (or a `t.Run` subtest) in
`e2e_test.go`. Use `newEnv(t)` for isolated state and the `env.run`/`httpGet`/
`postShutdown` helpers.
- **Add an OS:** extend the `matrix.os` list in `.github/workflows/cli-e2e.yml`.
- Deeper per-OS path assertions (state resolves under the OS-native config dir
when `AO_RUN_FILE`/`AO_DATA_DIR` are unset) fit best as unit tests in
`internal/config`.

36
test/cli/install-check.sh Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env bash
#
# Fresh-machine install check. The Dockerfile installs `ao` on PATH in a clean
# image and runs this; it proves a freshly installed binary actually works on a
# machine with no Go toolchain and no developer state. The COMPREHENSIVE,
# cross-platform behavioural suite lives in Go (backend/internal/cli/e2e_test.go,
# `go test -tags e2e`); this stays deliberately small and linear.
set -euo pipefail
AO_BIN="${AO_BIN:-ao}"
tmp="$(mktemp -d)"
export AO_RUN_FILE="$tmp/running.json"
export AO_DATA_DIR="$tmp/data"
export AO_PORT="${AO_PORT:-3001}" # the container is isolated; 3001 is free
trap '"$AO_BIN" stop >/dev/null 2>&1 || true; rm -rf "$tmp"' EXIT
fail() { echo "FAIL: $1" >&2; exit 1; }
echo "ao binary : $(command -v "$AO_BIN")"
"$AO_BIN" version >/dev/null || fail "version"
"$AO_BIN" doctor >/dev/null || fail "doctor"
"$AO_BIN" start >/dev/null || fail "start"
"$AO_BIN" status --json | grep -q '"state": "ready"' || fail "daemon not ready after start"
# the /shutdown control endpoint rejects a cross-origin caller (403) and survives
code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \
-H 'Origin: https://evil.example' "http://127.0.0.1:$AO_PORT/shutdown")"
[ "$code" = "403" ] || fail "cross-origin /shutdown returned $code, want 403"
"$AO_BIN" status --json | grep -q '"state": "ready"' || fail "daemon died after rejected shutdown"
"$AO_BIN" stop >/dev/null || fail "stop"
"$AO_BIN" status --json | grep -q '"state": "stopped"' || fail "daemon not stopped"
echo "fresh-install check: OK"

35
test/cli/run-local.sh
View File

@ -1,35 +0,0 @@
#!/usr/bin/env bash
#
# Convenience wrapper: build `ao` from source and run the CLI smoke test against
# it natively, using an isolated temp state dir and a free port. Touches nothing
# in your real AO installation.
#
# test/cli/run-local.sh # PASS/FAIL per assertion
# test/cli/run-local.sh -v # also print every command and its full output
#
# To run the same suite the way a brand-new user would install it (clean Linux
# container, binary on PATH), use Docker instead:
#
# docker build -f test/cli/Dockerfile -t ao-cli-smoke . && docker run --rm --init ao-cli-smoke
set -euo pipefail
# Pass through -v/--verbose (anything else is forwarded to smoke.sh too).
smoke_args=()
for arg in "$@"; do
case "$arg" in
-v|--verbose) smoke_args+=("--verbose") ;;
-h|--help) echo "usage: run-local.sh [-v|--verbose]"; exit 0 ;;
*) smoke_args+=("$arg") ;;
esac
done
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
bindir="$(mktemp -d)"
trap 'rm -rf "$bindir"' EXIT
echo "building ao ..."
( cd "$repo_root/backend" && CGO_ENABLED=0 go build -trimpath -o "$bindir/ao" ./cmd/ao )
echo "running smoke test ..."
AO_BIN="$bindir/ao" bash "$repo_root/test/cli/smoke.sh" "${smoke_args[@]+"${smoke_args[@]}"}"

334
test/cli/smoke.sh
View File

@ -1,334 +0,0 @@
#!/usr/bin/env bash
#
# End-to-end smoke test for the `ao` CLI.
#
# It models a *fresh machine*: `ao` is expected to already be installed on PATH
# (the Dockerfile in this directory installs it, simulating a new user), and the
# whole test runs against isolated, throwaway state — its own temp run-file,
# data dir, and a free loopback port — so it never touches a developer's real
# AO installation.
#
# Run locally against a binary you built:
# AO_BIN=/path/to/ao test/cli/smoke.sh
# Verbose — print each command and its full output:
# AO_BIN=/path/to/ao test/cli/smoke.sh -v (or AO_SMOKE_VERBOSE=1)
# Or in the container (models install-on-a-fresh-machine):
# docker build -f test/cli/Dockerfile -t ao-cli-smoke . && docker run --rm --init ao-cli-smoke
#
# Exit code: 0 if every assertion passes, 1 otherwise.
set -uo pipefail
AO_BIN="${AO_BIN:-ao}"
# Verbose mode prints every command and its complete output, not just PASS/FAIL.
VERBOSE="${AO_SMOKE_VERBOSE:-0}"
for arg in "$@"; do
case "$arg" in
-v|--verbose) VERBOSE=1 ;;
-h|--help) echo "usage: [AO_BIN=...] smoke.sh [-v|--verbose]"; exit 0 ;;
*) echo "unknown argument: $arg" >&2; exit 2 ;;
esac
done
# ---------------------------------------------------------------------------
# Tiny assertion framework
# ---------------------------------------------------------------------------
PASS=0
FAIL=0
CURRENT=""
section() { printf '\n\033[1m== %s ==\033[0m\n' "$1"; }
step() {
CURRENT="$1"
if [ "$VERBOSE" = 1 ]; then printf ' • %s\n' "$1"; else printf ' • %s ... ' "$1"; fi
}
ok() {
PASS=$((PASS + 1))
if [ "$VERBOSE" = 1 ]; then printf ' \033[32m→ PASS\033[0m\n'; else printf '\033[32mPASS\033[0m\n'; fi
}
bad() {
FAIL=$((FAIL + 1))
if [ "$VERBOSE" = 1 ]; then printf ' \033[31m→ FAIL\033[0m %s\n' "$1"; else printf '\033[31mFAIL\033[0m\n %s\n' "$1"; fi
}
# vdump <command-label> <output> <exit-code> : in verbose mode, echo the command
# and its complete output, indented. A no-op otherwise.
vdump() {
[ "$VERBOSE" = 1 ] || return 0
printf ' \033[2m$ %s\033[0m\n' "$1"
if [ -n "$2" ]; then printf '%s\n' "$2" | sed 's/^/ | /'; fi
printf ' \033[2m(exit %s)\033[0m\n' "$3"
}
# assert_eq <actual> <expected> [msg]
assert_eq() {
if [ "$1" = "$2" ]; then ok; else bad "${3:-}: expected [$2], got [$1]"; fi
}
# assert_contains <haystack> <needle>
assert_contains() {
case "$1" in
*"$2"*) ok ;;
*) bad "expected output to contain [$2]; got: $(printf '%s' "$1" | head -c 400)" ;;
esac
}
# assert_not_contains <haystack> <needle>
assert_not_contains() {
case "$1" in
*"$2"*) bad "expected output to NOT contain [$2]" ;;
*) ok ;;
esac
}
# run_rc <cmd...> -> sets RC and OUT (stdout+stderr combined)
run_rc() {
OUT="$("$@" 2>&1)"
RC=$?
vdump "$*" "$OUT" "$RC"
}
# ---------------------------------------------------------------------------
# Isolated, throwaway environment
# ---------------------------------------------------------------------------
TMP="$(mktemp -d)"
export AO_RUN_FILE="$TMP/running.json"
export AO_DATA_DIR="$TMP/data"
# Pick a free loopback port (bash /dev/tcp probe; connect-refused == free).
find_free_port() {
local p
for p in $(seq 3071 3170); do
if ! (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null; then
echo "$p"; return 0
fi
exec 3>&- 2>/dev/null || true
done
echo 3071
}
# Always run against an isolated, free port. We deliberately do NOT honour an
# inherited AO_PORT — it might point at a real daemon, which is exactly the
# collision this isolation is meant to prevent. Override only via AO_SMOKE_PORT.
export AO_PORT="${AO_SMOKE_PORT:-$(find_free_port)}"
cleanup() {
"$AO_BIN" stop >/dev/null 2>&1 || true
rm -rf "$TMP"
}
trap cleanup EXIT
printf 'ao smoke test\n binary : %s\n port : %s\n state : %s\n' \
"$(command -v "$AO_BIN" || echo "$AO_BIN")" "$AO_PORT" "$TMP"
# ---------------------------------------------------------------------------
# 1. Install verification — `ao` is a real, runnable binary on this machine
# ---------------------------------------------------------------------------
section "install"
step "ao resolves on PATH / at AO_BIN"
if command -v "$AO_BIN" >/dev/null 2>&1; then ok; else bad "ao not found"; fi
step "ao version prints build metadata"
run_rc "$AO_BIN" version
if [ "$RC" -eq 0 ] && [ -n "$OUT" ]; then ok; else bad "rc=$RC out=$OUT"; fi
step "ao --version works"
run_rc "$AO_BIN" --version
assert_eq "$RC" "0" "--version rc"
step "ao --help lists product commands"
run_rc "$AO_BIN" --help
assert_contains "$OUT" "start"
step "ao --help lists status/stop/doctor"
assert_contains "$OUT" "doctor"
step "ao --help hides internal daemon command"
assert_not_contains "$OUT" $'\n daemon'
# ---------------------------------------------------------------------------
# 2. doctor on a fresh machine (no daemon yet)
# ---------------------------------------------------------------------------
section "doctor (fresh)"
step "doctor exits 0 when required tools present"
run_rc "$AO_BIN" doctor
assert_eq "$RC" "0" "doctor rc (git/tmux must be installed in the image)"
step "doctor reports git found"
assert_contains "$OUT" "git"
step "doctor does NOT migrate the store (sqlite WARN, db absent)"
assert_contains "$OUT" "database not created yet"
step "doctor data dir was created but ao.db was NOT (CLI is not the store writer)"
if [ ! -f "$AO_DATA_DIR/ao.db" ]; then ok; else bad "ao.db exists — doctor must not create/migrate the DB"; fi
step "doctor --json is valid JSON with ok=true"
run_rc "$AO_BIN" doctor --json
assert_contains "$OUT" '"ok": true'
# ---------------------------------------------------------------------------
# 3. status when stopped
# ---------------------------------------------------------------------------
section "status (stopped)"
step "status --json reports stopped"
run_rc "$AO_BIN" status --json
assert_contains "$OUT" '"state": "stopped"'
step "status exits 0 even when stopped (status never errors)"
assert_eq "$RC" "0" "status exit code"
step "stopped status omits startedAt"
assert_not_contains "$OUT" "startedAt"
step "stop is idempotent when already stopped"
run_rc "$AO_BIN" stop
if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "stopped"; else bad "stop-when-stopped rc=$RC"; fi
# ---------------------------------------------------------------------------
# 4. start → ready, and status reflects it
# ---------------------------------------------------------------------------
section "start"
step "start brings the daemon up and reports ready"
run_rc "$AO_BIN" start
if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "ready"; else bad "start rc=$RC out=$OUT"; fi
step "status --json reports ready with pid+port"
run_rc "$AO_BIN" status --json
assert_contains "$OUT" '"state": "ready"'
step "status carries the bound port"
assert_contains "$OUT" "\"port\": $AO_PORT"
step "start is idempotent (second start returns ready, no error)"
run_rc "$AO_BIN" start
if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "ready"; else bad "idempotent start rc=$RC"; fi
# Capture the live pid for later assertions.
PID="$("$AO_BIN" status --json | sed -n 's/.*"pid": \([0-9]*\).*/\1/p' | head -1)"
# ---------------------------------------------------------------------------
# 5. doctor while running — now the daemon (not the CLI) has created the store
# ---------------------------------------------------------------------------
section "doctor (running)"
step "daemon created and migrated the store"
if [ -f "$AO_DATA_DIR/ao.db" ]; then ok; else bad "daemon should have created ao.db"; fi
step "doctor now reports sqlite present + daemon-migrated"
run_rc "$AO_BIN" doctor
assert_contains "$OUT" "migrations are applied by the daemon"
# ---------------------------------------------------------------------------
# 6. Health endpoint identity (loopback)
# ---------------------------------------------------------------------------
section "health endpoint"
if command -v curl >/dev/null 2>&1; then
step "/healthz reports the AO daemon service + pid"
run_rc curl -fsS "http://127.0.0.1:$AO_PORT/healthz"
assert_contains "$OUT" "agent-orchestrator-daemon"
# -------------------------------------------------------------------------
# 7. /shutdown CSRF / DNS-rebinding guard (review fix M3)
# -------------------------------------------------------------------------
section "/shutdown guard"
step "cross-origin POST /shutdown is rejected (403)"
CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST \
-H 'Origin: https://evil.example' "http://127.0.0.1:$AO_PORT/shutdown")"
vdump "curl -X POST -H 'Origin: https://evil.example' http://127.0.0.1:$AO_PORT/shutdown" "HTTP $CODE" "-"
assert_eq "$CODE" "403" "cross-origin shutdown"
step "non-loopback Host POST /shutdown is rejected (403)"
CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST \
-H 'Host: evil.example' "http://127.0.0.1:$AO_PORT/shutdown")"
vdump "curl -X POST -H 'Host: evil.example' http://127.0.0.1:$AO_PORT/shutdown" "HTTP $CODE" "-"
assert_eq "$CODE" "403" "rebinding-host shutdown"
step "daemon survived the rejected shutdown attempts"
run_rc "$AO_BIN" status --json
assert_contains "$OUT" '"state": "ready"'
else
section "/shutdown guard"
step "curl unavailable — skipping HTTP-level guard checks"
printf '\033[33mSKIP\033[0m\n'
fi
# ---------------------------------------------------------------------------
# 8. stop → stopped, run-file cleaned up
# ---------------------------------------------------------------------------
section "stop"
step "stop gracefully stops the daemon"
run_rc "$AO_BIN" stop
if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "stopped"; else bad "stop rc=$RC out=$OUT"; fi
step "run-file removed after stop"
if [ ! -f "$AO_RUN_FILE" ]; then ok; else bad "running.json still present"; fi
step "status --json reports stopped after stop"
run_rc "$AO_BIN" status --json
assert_contains "$OUT" '"state": "stopped"'
# ---------------------------------------------------------------------------
# 9. stale run-file (dead PID) — deterministic, no real process needed
# ---------------------------------------------------------------------------
section "stale run-file"
# PID 2147483647 is never alive; the CLI must classify this as stale, not kill it.
printf '{"pid":2147483647,"port":%s,"startedAt":"2020-01-01T00:00:00Z"}\n' "$AO_PORT" > "$AO_RUN_FILE"
step "status reports stale for a dead-PID run-file"
run_rc "$AO_BIN" status --json
assert_contains "$OUT" '"state": "stale"'
step "status still exits 0 for a stale daemon (reports, never errors)"
assert_eq "$RC" "0" "stale status exit code"
step "stop clears a stale run-file and reports stopped"
run_rc "$AO_BIN" stop
assert_contains "$OUT" "stopped"
step "stale run-file removed"
if [ ! -f "$AO_RUN_FILE" ]; then ok; else bad "stale running.json not removed"; fi
# ---------------------------------------------------------------------------
# 10. exit codes: 2 for usage errors, 1 for runtime errors
# ---------------------------------------------------------------------------
section "exit codes"
step "unknown flag exits 2 (usage error)"
run_rc "$AO_BIN" status --definitely-not-a-flag
assert_eq "$RC" "2" "bad-flag exit code"
step "missing required arg exits 2 (completion needs a shell)"
run_rc "$AO_BIN" completion
assert_eq "$RC" "2" "missing-arg exit code"
step "unsupported shell exits non-zero (runtime error)"
run_rc "$AO_BIN" completion notashell
if [ "$RC" -ne 0 ]; then ok; else bad "expected non-zero for bad shell"; fi
step "invalid config (AO_PORT out of range) exits 1, not 2"
OUT="$(AO_PORT=99999 "$AO_BIN" status 2>&1)"; RC=$?
vdump "AO_PORT=99999 $AO_BIN status" "$OUT" "$RC"
assert_eq "$RC" "1" "config-error exit code (runtime, not usage)"
# ---------------------------------------------------------------------------
# 11. shell completion generators
# ---------------------------------------------------------------------------
section "completion"
for sh in bash zsh fish powershell; do
step "completion $sh generates a script"
run_rc "$AO_BIN" completion "$sh"
if [ "$RC" -eq 0 ] && [ -n "$OUT" ]; then ok; else bad "completion $sh rc=$RC"; fi
done
# ---------------------------------------------------------------------------
# Result
# ---------------------------------------------------------------------------
printf '\n\033[1m== result ==\033[0m\n passed: %s\n failed: %s\n' "$PASS" "$FAIL"
if [ "$FAIL" -ne 0 ]; then
printf '\033[31mSMOKE TEST FAILED\033[0m\n'
exit 1
fi
printf '\033[32mSMOKE TEST PASSED\033[0m\n'