diff --git a/.github/workflows/cli-e2e.yml b/.github/workflows/cli-e2e.yml index 35ef26cd4..307384328 100644 --- a/.github/workflows/cli-e2e.yml +++ b/.github/workflows/cli-e2e.yml @@ -13,17 +13,20 @@ permissions: contents: read jobs: - # Primary tier: run the REAL `ao` binary on GitHub's native VM runners. These - # runners are the "VMs" — the only place that exercises the OS-specific code - # paths (unix Setsid process-group detach + macOS os.UserConfigDir resolution). - # Bash is available on both ubuntu and macos runners, so the one smoke.sh runs - # unchanged. State is isolated per run (own temp dir + a free loopback port). + # Primary tier: the cross-platform Go E2E suite (build tag `e2e`) runs the real + # `ao` binary against isolated state on every OS GitHub hosts. These runners + # are the "VMs" — the only place that exercises the OS-specific process-detach + # paths (unix Setsid vs Windows CREATE_NEW_PROCESS_GROUP) and os.UserConfigDir + # resolution. The suite builds its own binary and self-allocates a free port. native: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: backend steps: - uses: actions/checkout@v4 @@ -32,25 +35,21 @@ jobs: go-version: "1.25" cache: false - - name: Build ao - run: cd backend && CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/ao" ./cmd/ao + - name: CLI E2E (native) + run: go test -tags e2e -v ./internal/cli/... - - name: CLI smoke test - run: AO_BIN="$RUNNER_TEMP/ao" bash test/cli/smoke.sh - - # Secondary hardening tier: model "install ao on a fresh machine" in a clean, - # locked-down Linux container with no access to a developer's real state. - # --init gives the container a real PID-1 reaper (tini) so the live daemon the - # `start` test spawns (it detaches via setsid) is reaped after `stop` rather - # than lingering as a zombie. The suite doesn't depend on it (the stale case - # uses a fabricated dead PID), but it keeps process accounting clean. + # Secondary hardening tier: prove that a freshly installed binary works on a + # clean machine with no Go toolchain and no developer state. The Dockerfile + # installs `ao` on PATH in a slim image and runs test/cli/install-check.sh. + # --init gives a real PID-1 reaper so the daemon the check starts is reaped + # after `stop` instead of lingering as a zombie. container: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build smoke image (fresh-machine install) + - name: Build fresh-install image run: docker build -f test/cli/Dockerfile -t ao-cli-smoke . - - name: Run CLI smoke test in container + - name: Fresh-install check (container) run: docker run --rm --init ao-cli-smoke diff --git a/backend/internal/cli/e2e_test.go b/backend/internal/cli/e2e_test.go new file mode 100644 index 000000000..f89e46711 --- /dev/null +++ b/backend/internal/cli/e2e_test.go @@ -0,0 +1,346 @@ +//go:build e2e + +// Package cli_test holds the end-to-end suite for the `ao` CLI. It builds the +// real binary and drives it (start/status/doctor/stop + the daemon-control HTTP +// surface) against fully isolated state — a per-test temp run-file, data dir, +// and an OS-assigned free loopback port — so it never touches a developer's real +// AO install. Unlike the Linux-only container smoke test, this runs natively on +// every OS in CI (ubuntu/macos/windows), which is the only way to exercise the +// unix setsid vs Windows CREATE_NEW_PROCESS_GROUP detach paths and the per-OS +// os.UserConfigDir resolution. +// +// It is gated behind the `e2e` build tag so it never runs in the normal +// `go test ./...` lane (it spawns processes and binds ports): +// +// go test -tags e2e ./internal/cli/... # run it +// go test -tags e2e -v -run TestE2E ./internal/cli/... # verbose, see every command +package cli_test + +import ( + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// aoBin is the path to the binary built once for the whole suite. +var aoBin string + +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "ao-e2e-bin") + if err != nil { + fmt.Fprintln(os.Stderr, "e2e: mktemp:", err) + os.Exit(1) + } + aoBin = filepath.Join(dir, "ao") + if runtime.GOOS == "windows" { + aoBin += ".exe" + } + build := exec.Command("go", "build", "-o", aoBin, "github.com/aoagents/agent-orchestrator/backend/cmd/ao") + build.Stdout, build.Stderr = os.Stderr, os.Stderr + if err := build.Run(); err != nil { + fmt.Fprintln(os.Stderr, "e2e: build ao:", err) + os.Exit(1) + } + code := m.Run() + _ = os.RemoveAll(dir) + os.Exit(code) +} + +// env is an isolated CLI environment: its own state files and free port. +type env struct { + runFile string + dataDir string + port int +} + +func newEnv(t *testing.T) env { + t.Helper() + dir := t.TempDir() + return env{ + runFile: filepath.Join(dir, "running.json"), + dataDir: filepath.Join(dir, "data"), + port: freePort(t), + } +} + +// environ builds the child env: the ambient environment with every inherited +// AO_* var stripped (so a real daemon's AO_PORT can't leak in) plus our isolated +// settings. portOverride, when non-empty, replaces the numeric AO_PORT — used to +// inject an invalid value. +func (e env) environ(portOverride string) []string { + out := make([]string, 0, len(os.Environ())+3) + for _, kv := range os.Environ() { + if strings.HasPrefix(kv, "AO_") { + continue + } + out = append(out, kv) + } + port := fmt.Sprintf("%d", e.port) + if portOverride != "" { + port = portOverride + } + return append(out, "AO_RUN_FILE="+e.runFile, "AO_DATA_DIR="+e.dataDir, "AO_PORT="+port) +} + +func freePort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("alloc free port: %v", err) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +// run executes `ao args...` in env e and returns combined output + exit code. +func (e env) run(t *testing.T, args ...string) (string, int) { + t.Helper() + return e.runEnv(t, e.environ(""), args...) +} + +func (e env) runEnv(t *testing.T, environ []string, args ...string) (string, int) { + t.Helper() + cmd := exec.Command(aoBin, args...) + cmd.Env = environ + b, err := cmd.CombinedOutput() + out := string(b) + code := 0 + if err != nil { + var ee *exec.ExitError + if asExit(err, &ee) { + code = ee.ExitCode() + } else { + t.Fatalf("run %v: %v\n%s", args, err, out) + } + } + t.Logf("$ ao %s\n%s(exit %d)", strings.Join(args, " "), out, code) + return out, code +} + +func asExit(err error, target **exec.ExitError) bool { + if ee, ok := err.(*exec.ExitError); ok { + *target = ee + return true + } + return false +} + +// startDaemon brings the daemon up and registers a stop on cleanup. +func (e env) startDaemon(t *testing.T) { + t.Helper() + out, code := e.run(t, "start") + if code != 0 { + t.Fatalf("start failed (exit %d): %s", code, out) + } + t.Cleanup(func() { e.run(t, "stop") }) +} + +func mustContain(t *testing.T, out, want string) { + t.Helper() + if !strings.Contains(out, want) { + t.Fatalf("expected output to contain %q; got:\n%s", want, out) + } +} + +func mustNotContain(t *testing.T, out, notWant string) { + t.Helper() + if strings.Contains(out, notWant) { + t.Fatalf("expected output NOT to contain %q; got:\n%s", notWant, out) + } +} + +// --------------------------------------------------------------------------- + +func TestE2E_VersionAndHelp(t *testing.T) { + e := newEnv(t) + + if out, code := e.run(t, "version"); code != 0 || strings.TrimSpace(out) == "" { + t.Fatalf("version: exit %d, out %q", code, out) + } + if _, code := e.run(t, "--version"); code != 0 { + t.Fatalf("--version exit %d", code) + } + + out, code := e.run(t, "--help") + if code != 0 { + t.Fatalf("--help exit %d", code) + } + for _, want := range []string{"start", "stop", "status", "doctor", "completion", "version"} { + mustContain(t, out, want) + } + // the internal daemon command is hidden from help (rendered as "\n daemon") + mustNotContain(t, out, "\n daemon") +} + +func TestE2E_DoctorDoesNotTouchTheStore(t *testing.T) { + e := newEnv(t) + + out, code := e.run(t, "doctor") + if code != 0 { + t.Fatalf("doctor (fresh) exit %d: %s", code, out) + } + mustContain(t, out, "git") + mustContain(t, out, "database not created yet") // sqlite WARN, never migrated + + // doctor must NOT create/migrate the DB — the daemon is the sole writer. + if _, err := os.Stat(filepath.Join(e.dataDir, "ao.db")); err == nil { + t.Fatal("doctor created ao.db; the CLI must not open/migrate the store") + } + + if out, code := e.run(t, "doctor", "--json"); code != 0 || !strings.Contains(out, `"ok": true`) { + t.Fatalf("doctor --json: exit %d, out %s", code, out) + } +} + +func TestE2E_StatusStopped(t *testing.T) { + e := newEnv(t) + out, code := e.run(t, "status", "--json") + if code != 0 { // status always exits 0 + t.Fatalf("status exit %d", code) + } + mustContain(t, out, `"state": "stopped"`) + mustNotContain(t, out, "startedAt") + + if out, code := e.run(t, "stop"); code != 0 || !strings.Contains(out, "stopped") { + t.Fatalf("stop-when-stopped: exit %d, out %s", code, out) // idempotent + } +} + +func TestE2E_Lifecycle(t *testing.T) { + e := newEnv(t) + e.startDaemon(t) + + out, _ := e.run(t, "status", "--json") + mustContain(t, out, `"state": "ready"`) + mustContain(t, out, fmt.Sprintf(`"port": %d`, e.port)) + + // idempotent + if out, code := e.run(t, "start"); code != 0 || !strings.Contains(out, "ready") { + t.Fatalf("idempotent start: exit %d, out %s", code, out) + } + + // now the daemon (not the CLI) has created + migrated the store + if _, err := os.Stat(filepath.Join(e.dataDir, "ao.db")); err != nil { + t.Fatalf("daemon should have created ao.db: %v", err) + } + out, _ = e.run(t, "doctor") + mustContain(t, out, "migrations are applied by the daemon") + + // /healthz identity + body := httpGet(t, e.port, "/healthz") + mustContain(t, body, "agent-orchestrator-daemon") + + if out, code := e.run(t, "stop"); code != 0 || !strings.Contains(out, "stopped") { + t.Fatalf("stop: exit %d, out %s", code, out) + } + if _, err := os.Stat(e.runFile); !os.IsNotExist(err) { + t.Fatal("run-file should be removed after stop") + } +} + +func TestE2E_ShutdownGuard(t *testing.T) { + e := newEnv(t) + e.startDaemon(t) + + // A cross-site Origin header must be rejected without stopping the daemon. + if code := postShutdown(t, e.port, func(r *http.Request) { r.Header.Set("Origin", "https://evil.example") }); code != http.StatusForbidden { + t.Fatalf("cross-origin /shutdown = %d, want 403", code) + } + // A non-loopback Host (DNS-rebinding) must be rejected too. + if code := postShutdown(t, e.port, func(r *http.Request) { r.Host = "evil.example" }); code != http.StatusForbidden { + t.Fatalf("rebinding-host /shutdown = %d, want 403", code) + } + // The daemon survived both. + out, _ := e.run(t, "status", "--json") + mustContain(t, out, `"state": "ready"`) +} + +func TestE2E_StaleRunFile(t *testing.T) { + e := newEnv(t) + // PID 2147483647 is never alive -> the CLI must classify this as stale. + content := fmt.Sprintf(`{"pid":2147483647,"port":%d,"startedAt":"2020-01-01T00:00:00Z"}`, e.port) + if err := os.MkdirAll(filepath.Dir(e.runFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(e.runFile, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + out, _ := e.run(t, "status", "--json") + mustContain(t, out, `"state": "stale"`) + + if out, code := e.run(t, "stop"); code != 0 || !strings.Contains(out, "stopped") { + t.Fatalf("stop stale: exit %d, out %s", code, out) + } + if _, err := os.Stat(e.runFile); !os.IsNotExist(err) { + t.Fatal("stale run-file should be removed") + } +} + +func TestE2E_ExitCodes(t *testing.T) { + e := newEnv(t) + + if _, code := e.run(t, "status", "--definitely-not-a-flag"); code != 2 { + t.Fatalf("bad flag exit %d, want 2", code) + } + if _, code := e.run(t, "completion"); code != 2 { // missing required arg + t.Fatalf("missing-arg exit %d, want 2", code) + } + if _, code := e.run(t, "completion", "notashell"); code == 0 { // runtime error + t.Fatal("unsupported shell should be non-zero") + } + // invalid config is a runtime error (1), not a usage error (2). + if _, code := e.runEnv(t, e.environ("notaport"), "status"); code != 1 { + t.Fatalf("invalid AO_PORT exit %d, want 1", code) + } +} + +func TestE2E_Completion(t *testing.T) { + e := newEnv(t) + for _, sh := range []string{"bash", "zsh", "fish", "powershell"} { + out, code := e.run(t, "completion", sh) + if code != 0 || strings.TrimSpace(out) == "" { + t.Fatalf("completion %s: exit %d, empty=%v", sh, code, strings.TrimSpace(out) == "") + } + } +} + +// --------------------------------------------------------------------------- +// HTTP helpers (loopback) + +func httpClient() *http.Client { return &http.Client{Timeout: 3 * time.Second} } + +func httpGet(t *testing.T, port int, path string) string { + t.Helper() + resp, err := httpClient().Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, path)) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + defer resp.Body.Close() + b := make([]byte, 4096) + n, _ := resp.Body.Read(b) + return string(b[:n]) +} + +// postShutdown issues POST /shutdown with mutator applied, returns the status code. +func postShutdown(t *testing.T, port int, mutate func(*http.Request)) int { + t.Helper() + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/shutdown", port), nil) + if err != nil { + t.Fatal(err) + } + mutate(req) + resp, err := httpClient().Do(req) + if err != nil { + t.Fatalf("POST /shutdown: %v", err) + } + defer resp.Body.Close() + return resp.StatusCode +} diff --git a/test/cli/Dockerfile b/test/cli/Dockerfile index ce429a9c6..fb5d85b2a 100644 --- a/test/cli/Dockerfile +++ b/test/cli/Dockerfile @@ -33,15 +33,15 @@ RUN apt-get update \ # "Install" the CLI the way a user would: drop the binary on PATH. COPY --from=build /out/ao /usr/local/bin/ao -COPY test/cli/smoke.sh /usr/local/bin/ao-smoke.sh -RUN chmod +x /usr/local/bin/ao /usr/local/bin/ao-smoke.sh +COPY test/cli/install-check.sh /usr/local/bin/ao-install-check.sh +RUN chmod +x /usr/local/bin/ao /usr/local/bin/ao-install-check.sh # Run as an unprivileged user with a real HOME, like a normal install. RUN useradd --create-home --shell /bin/bash ao USER ao WORKDIR /home/ao -# Sanity: prove the install resolved before the suite runs. +# Sanity: prove the install resolved before the check runs. RUN ao version -ENTRYPOINT ["/usr/local/bin/ao-smoke.sh"] +ENTRYPOINT ["/usr/local/bin/ao-install-check.sh"] diff --git a/test/cli/README.md b/test/cli/README.md index 231c44e9c..14c3a56c4 100644 --- a/test/cli/README.md +++ b/test/cli/README.md @@ -1,75 +1,65 @@ # `ao` CLI end-to-end tests -These tests install and drive the **real `ao` binary** the way a user would — -`start` → `status` → `doctor` → `stop`, plus the daemon-control HTTP surface — -and assert the whole thing works end to end. They run against **isolated, -throwaway state** (their own temp run-file + data dir + a free loopback port), -so they never touch a developer's real AO installation. +These tests drive the **real `ao` binary** the way a user would — `start` → +`status` → `doctor` → `stop`, plus the daemon-control HTTP surface — and assert +the whole thing works. They run against **isolated, throwaway state** (a per-test +temp run-file + data dir + an OS-assigned free loopback port), so they never +touch a developer's real AO installation. -## Files +## Two tiers -| File | Purpose | -|------|---------| -| `smoke.sh` | The suite. Host-agnostic bash; drives the binary at `$AO_BIN` (default `ao` on PATH) and prints a PASS/FAIL line per assertion. | -| `Dockerfile` | Models **installing `ao` on a fresh machine**: builds the binary, drops it on `PATH` in a clean Debian image with only runtime deps (`git`, `tmux`, `curl`), then runs `smoke.sh` as a non-root user. | -| `run-local.sh` | Convenience wrapper: build from source and run `smoke.sh` natively against a temp binary. | +| Tier | What | Where | +|------|------|-------| +| **Comprehensive (primary)** | A cross-platform Go suite that builds `ao` and exercises the full behaviour. Runs natively on **ubuntu + macOS + windows** — the only way to cover the OS-specific process-detach paths (`setsid` vs `CREATE_NEW_PROCESS_GROUP`) and `os.UserConfigDir()` resolution. | `backend/internal/cli/e2e_test.go` (build tag `e2e`) | +| **Fresh-install (hardening)** | Proves a freshly installed binary works on a clean machine with no Go toolchain and no developer state. | `test/cli/Dockerfile` + `test/cli/install-check.sh` | ## Run it -**Native (fastest, uses your toolchain):** +**The Go suite (fastest, cross-platform):** ```bash -test/cli/run-local.sh # PASS/FAIL per assertion -test/cli/run-local.sh -v # also print every command and its full output -# or, against a binary you already built: -AO_BIN=/path/to/ao test/cli/smoke.sh [-v] +cd backend +go test -tags e2e ./internal/cli/... # run it +go test -tags e2e -v -run TestE2E ./internal/cli/... # verbose: prints every command + output ``` +It builds its own `ao` binary; `git` must be on PATH (required by `doctor`). +`-v` logs each `ao` invocation and its full output, which is the audit trail you +get for free from `go test`. **Fresh-machine install, in a clean container:** ```bash docker build -f test/cli/Dockerfile -t ao-cli-smoke . docker run --rm --init ao-cli-smoke ``` -> `--init` gives the container a real PID-1 reaper (tini) so the live daemon -> spawned during the `start` test is reaped after `stop` instead of lingering as -> a zombie. The suite itself doesn't depend on it — the stale-daemon case uses a -> fabricated dead PID — but it keeps process accounting clean. +> `--init` gives the container a real PID-1 reaper (tini) so the daemon the +> check starts is reaped after `stop` instead of lingering as a zombie. -## What it covers +## What the Go suite covers -Install resolves on PATH · `version`/`--version` · `--help` (and hides the -internal `daemon` command) · `doctor` text + `--json` (and that it **does not** -open/migrate SQLite) · `status` stopped/stale/ready · `start` (fresh + -idempotent) · daemon-created store · `/healthz` identity · the `/shutdown` -CSRF/DNS-rebinding guard (403 + daemon survives) · `stop` (graceful + stale + -idempotent) · run-file cleanup/ownership · exit codes (`2` usage, `1` runtime) · -completion for all four shells. +`TestE2E_VersionAndHelp` (version/`--version`/help, daemon hidden) · +`TestE2E_DoctorDoesNotTouchTheStore` (doctor text + `--json`; proves it does +**not** create/migrate `ao.db`) · `TestE2E_StatusStopped` (stopped + idempotent +stop) · `TestE2E_Lifecycle` (start, ready, idempotent, daemon-created store, +`/healthz` identity, stop, run-file cleanup) · `TestE2E_ShutdownGuard` (the +`/shutdown` CSRF + DNS-rebinding 403 guard, daemon survives) · +`TestE2E_StaleRunFile` (dead-PID run-file → stale → cleaned) · `TestE2E_ExitCodes` +(2 usage / 1 runtime / config error) · `TestE2E_Completion` (all four shells). -## Testing strategy — why it's shaped this way +## Why a Go suite (not bash, not Python) -We deliberately don't make Docker the *only* tier. A daemon that detaches with -`setsid` and outlives the launching process is exactly the workload that -container PID-1 semantics mishandle, and the OS-specific bits (`setsid` vs -Windows `CREATE_NEW_PROCESS_GROUP`, and `os.UserConfigDir()` resolving to -`~/Library/Application Support` on macOS, `%AppData%` on Windows, `~/.config` -on Linux) can't be observed from a Linux container at all. +The bash version grew past the point where bash was a good fit, and a Linux +container can't observe the macOS/Windows code paths at all. A Go `os/exec` +suite is the right home: it uses the repo's own toolchain (runs under `go test`), +gives real assertions and structured data, and — critically — runs natively on +the Windows and macOS runners, finally covering the `CREATE_NEW_PROCESS_GROUP` +detach path and per-OS config-dir resolution. The container stays as a thin +"clean install actually works" check. -So CI (`.github/workflows/cli-e2e.yml`) runs two tiers: +## Extending -1. **`native`** — the primary signal. Builds and runs the real binary on a - GitHub matrix of `ubuntu-latest` + `macos-latest` (those runners *are* the - VMs), covering the unix detach path and macOS config-dir resolution. -2. **`container`** — a hardening tier. The `Dockerfile` proves a clean-machine - install works and that the CLI has no hidden dependence on developer state, - run with `--init`. - -### Extending - -- Add an assertion: drop a `step`/`assert_*` pair into the relevant section of - `smoke.sh`. The helpers (`assert_eq`, `assert_contains`, `assert_not_contains`, - `run_rc`) keep cases one-liners. -- Cover Windows: add a `windows-latest` leg to the `native` matrix (Git Bash - ships on the runner) once the suite is confirmed green there, or add Go-based - `os/exec` E2E tests for the Windows process-group path. -- Deeper per-OS path assertions (that state resolves under the OS-native config - dir when `AO_RUN_FILE`/`AO_DATA_DIR` are unset) are best added as Go unit - tests in `internal/config`. +- **Add a case:** a new `TestE2E_*` function (or a `t.Run` subtest) in + `e2e_test.go`. Use `newEnv(t)` for isolated state and the `env.run`/`httpGet`/ + `postShutdown` helpers. +- **Add an OS:** extend the `matrix.os` list in `.github/workflows/cli-e2e.yml`. +- Deeper per-OS path assertions (state resolves under the OS-native config dir + when `AO_RUN_FILE`/`AO_DATA_DIR` are unset) fit best as unit tests in + `internal/config`. diff --git a/test/cli/install-check.sh b/test/cli/install-check.sh new file mode 100755 index 000000000..f4bcacd47 --- /dev/null +++ b/test/cli/install-check.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Fresh-machine install check. The Dockerfile installs `ao` on PATH in a clean +# image and runs this; it proves a freshly installed binary actually works on a +# machine with no Go toolchain and no developer state. The COMPREHENSIVE, +# cross-platform behavioural suite lives in Go (backend/internal/cli/e2e_test.go, +# `go test -tags e2e`); this stays deliberately small and linear. + +set -euo pipefail + +AO_BIN="${AO_BIN:-ao}" +tmp="$(mktemp -d)" +export AO_RUN_FILE="$tmp/running.json" +export AO_DATA_DIR="$tmp/data" +export AO_PORT="${AO_PORT:-3001}" # the container is isolated; 3001 is free +trap '"$AO_BIN" stop >/dev/null 2>&1 || true; rm -rf "$tmp"' EXIT + +fail() { echo "FAIL: $1" >&2; exit 1; } + +echo "ao binary : $(command -v "$AO_BIN")" +"$AO_BIN" version >/dev/null || fail "version" +"$AO_BIN" doctor >/dev/null || fail "doctor" +"$AO_BIN" start >/dev/null || fail "start" + +"$AO_BIN" status --json | grep -q '"state": "ready"' || fail "daemon not ready after start" + +# the /shutdown control endpoint rejects a cross-origin caller (403) and survives +code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'Origin: https://evil.example' "http://127.0.0.1:$AO_PORT/shutdown")" +[ "$code" = "403" ] || fail "cross-origin /shutdown returned $code, want 403" +"$AO_BIN" status --json | grep -q '"state": "ready"' || fail "daemon died after rejected shutdown" + +"$AO_BIN" stop >/dev/null || fail "stop" +"$AO_BIN" status --json | grep -q '"state": "stopped"' || fail "daemon not stopped" + +echo "fresh-install check: OK" diff --git a/test/cli/run-local.sh b/test/cli/run-local.sh deleted file mode 100755 index f2f560ac0..000000000 --- a/test/cli/run-local.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Convenience wrapper: build `ao` from source and run the CLI smoke test against -# it natively, using an isolated temp state dir and a free port. Touches nothing -# in your real AO installation. -# -# test/cli/run-local.sh # PASS/FAIL per assertion -# test/cli/run-local.sh -v # also print every command and its full output -# -# To run the same suite the way a brand-new user would install it (clean Linux -# container, binary on PATH), use Docker instead: -# -# docker build -f test/cli/Dockerfile -t ao-cli-smoke . && docker run --rm --init ao-cli-smoke - -set -euo pipefail - -# Pass through -v/--verbose (anything else is forwarded to smoke.sh too). -smoke_args=() -for arg in "$@"; do - case "$arg" in - -v|--verbose) smoke_args+=("--verbose") ;; - -h|--help) echo "usage: run-local.sh [-v|--verbose]"; exit 0 ;; - *) smoke_args+=("$arg") ;; - esac -done - -repo_root="$(cd "$(dirname "$0")/../.." && pwd)" -bindir="$(mktemp -d)" -trap 'rm -rf "$bindir"' EXIT - -echo "building ao ..." -( cd "$repo_root/backend" && CGO_ENABLED=0 go build -trimpath -o "$bindir/ao" ./cmd/ao ) - -echo "running smoke test ..." -AO_BIN="$bindir/ao" bash "$repo_root/test/cli/smoke.sh" "${smoke_args[@]+"${smoke_args[@]}"}" diff --git a/test/cli/smoke.sh b/test/cli/smoke.sh deleted file mode 100755 index 4682d53b4..000000000 --- a/test/cli/smoke.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env bash -# -# End-to-end smoke test for the `ao` CLI. -# -# It models a *fresh machine*: `ao` is expected to already be installed on PATH -# (the Dockerfile in this directory installs it, simulating a new user), and the -# whole test runs against isolated, throwaway state — its own temp run-file, -# data dir, and a free loopback port — so it never touches a developer's real -# AO installation. -# -# Run locally against a binary you built: -# AO_BIN=/path/to/ao test/cli/smoke.sh -# Verbose — print each command and its full output: -# AO_BIN=/path/to/ao test/cli/smoke.sh -v (or AO_SMOKE_VERBOSE=1) -# Or in the container (models install-on-a-fresh-machine): -# docker build -f test/cli/Dockerfile -t ao-cli-smoke . && docker run --rm --init ao-cli-smoke -# -# Exit code: 0 if every assertion passes, 1 otherwise. - -set -uo pipefail - -AO_BIN="${AO_BIN:-ao}" - -# Verbose mode prints every command and its complete output, not just PASS/FAIL. -VERBOSE="${AO_SMOKE_VERBOSE:-0}" -for arg in "$@"; do - case "$arg" in - -v|--verbose) VERBOSE=1 ;; - -h|--help) echo "usage: [AO_BIN=...] smoke.sh [-v|--verbose]"; exit 0 ;; - *) echo "unknown argument: $arg" >&2; exit 2 ;; - esac -done - -# --------------------------------------------------------------------------- -# Tiny assertion framework -# --------------------------------------------------------------------------- -PASS=0 -FAIL=0 -CURRENT="" - -section() { printf '\n\033[1m== %s ==\033[0m\n' "$1"; } - -step() { - CURRENT="$1" - if [ "$VERBOSE" = 1 ]; then printf ' • %s\n' "$1"; else printf ' • %s ... ' "$1"; fi -} - -ok() { - PASS=$((PASS + 1)) - if [ "$VERBOSE" = 1 ]; then printf ' \033[32m→ PASS\033[0m\n'; else printf '\033[32mPASS\033[0m\n'; fi -} -bad() { - FAIL=$((FAIL + 1)) - if [ "$VERBOSE" = 1 ]; then printf ' \033[31m→ FAIL\033[0m %s\n' "$1"; else printf '\033[31mFAIL\033[0m\n %s\n' "$1"; fi -} - -# vdump : in verbose mode, echo the command -# and its complete output, indented. A no-op otherwise. -vdump() { - [ "$VERBOSE" = 1 ] || return 0 - printf ' \033[2m$ %s\033[0m\n' "$1" - if [ -n "$2" ]; then printf '%s\n' "$2" | sed 's/^/ | /'; fi - printf ' \033[2m(exit %s)\033[0m\n' "$3" -} - -# assert_eq [msg] -assert_eq() { - if [ "$1" = "$2" ]; then ok; else bad "${3:-}: expected [$2], got [$1]"; fi -} -# assert_contains -assert_contains() { - case "$1" in - *"$2"*) ok ;; - *) bad "expected output to contain [$2]; got: $(printf '%s' "$1" | head -c 400)" ;; - esac -} -# assert_not_contains -assert_not_contains() { - case "$1" in - *"$2"*) bad "expected output to NOT contain [$2]" ;; - *) ok ;; - esac -} - -# run_rc -> sets RC and OUT (stdout+stderr combined) -run_rc() { - OUT="$("$@" 2>&1)" - RC=$? - vdump "$*" "$OUT" "$RC" -} - -# --------------------------------------------------------------------------- -# Isolated, throwaway environment -# --------------------------------------------------------------------------- -TMP="$(mktemp -d)" -export AO_RUN_FILE="$TMP/running.json" -export AO_DATA_DIR="$TMP/data" - -# Pick a free loopback port (bash /dev/tcp probe; connect-refused == free). -find_free_port() { - local p - for p in $(seq 3071 3170); do - if ! (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null; then - echo "$p"; return 0 - fi - exec 3>&- 2>/dev/null || true - done - echo 3071 -} -# Always run against an isolated, free port. We deliberately do NOT honour an -# inherited AO_PORT — it might point at a real daemon, which is exactly the -# collision this isolation is meant to prevent. Override only via AO_SMOKE_PORT. -export AO_PORT="${AO_SMOKE_PORT:-$(find_free_port)}" - -cleanup() { - "$AO_BIN" stop >/dev/null 2>&1 || true - rm -rf "$TMP" -} -trap cleanup EXIT - -printf 'ao smoke test\n binary : %s\n port : %s\n state : %s\n' \ - "$(command -v "$AO_BIN" || echo "$AO_BIN")" "$AO_PORT" "$TMP" - -# --------------------------------------------------------------------------- -# 1. Install verification — `ao` is a real, runnable binary on this machine -# --------------------------------------------------------------------------- -section "install" - -step "ao resolves on PATH / at AO_BIN" -if command -v "$AO_BIN" >/dev/null 2>&1; then ok; else bad "ao not found"; fi - -step "ao version prints build metadata" -run_rc "$AO_BIN" version -if [ "$RC" -eq 0 ] && [ -n "$OUT" ]; then ok; else bad "rc=$RC out=$OUT"; fi - -step "ao --version works" -run_rc "$AO_BIN" --version -assert_eq "$RC" "0" "--version rc" - -step "ao --help lists product commands" -run_rc "$AO_BIN" --help -assert_contains "$OUT" "start" -step "ao --help lists status/stop/doctor" -assert_contains "$OUT" "doctor" -step "ao --help hides internal daemon command" -assert_not_contains "$OUT" $'\n daemon' - -# --------------------------------------------------------------------------- -# 2. doctor on a fresh machine (no daemon yet) -# --------------------------------------------------------------------------- -section "doctor (fresh)" - -step "doctor exits 0 when required tools present" -run_rc "$AO_BIN" doctor -assert_eq "$RC" "0" "doctor rc (git/tmux must be installed in the image)" - -step "doctor reports git found" -assert_contains "$OUT" "git" - -step "doctor does NOT migrate the store (sqlite WARN, db absent)" -assert_contains "$OUT" "database not created yet" - -step "doctor data dir was created but ao.db was NOT (CLI is not the store writer)" -if [ ! -f "$AO_DATA_DIR/ao.db" ]; then ok; else bad "ao.db exists — doctor must not create/migrate the DB"; fi - -step "doctor --json is valid JSON with ok=true" -run_rc "$AO_BIN" doctor --json -assert_contains "$OUT" '"ok": true' - -# --------------------------------------------------------------------------- -# 3. status when stopped -# --------------------------------------------------------------------------- -section "status (stopped)" - -step "status --json reports stopped" -run_rc "$AO_BIN" status --json -assert_contains "$OUT" '"state": "stopped"' -step "status exits 0 even when stopped (status never errors)" -assert_eq "$RC" "0" "status exit code" -step "stopped status omits startedAt" -assert_not_contains "$OUT" "startedAt" - -step "stop is idempotent when already stopped" -run_rc "$AO_BIN" stop -if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "stopped"; else bad "stop-when-stopped rc=$RC"; fi - -# --------------------------------------------------------------------------- -# 4. start → ready, and status reflects it -# --------------------------------------------------------------------------- -section "start" - -step "start brings the daemon up and reports ready" -run_rc "$AO_BIN" start -if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "ready"; else bad "start rc=$RC out=$OUT"; fi - -step "status --json reports ready with pid+port" -run_rc "$AO_BIN" status --json -assert_contains "$OUT" '"state": "ready"' -step "status carries the bound port" -assert_contains "$OUT" "\"port\": $AO_PORT" - -step "start is idempotent (second start returns ready, no error)" -run_rc "$AO_BIN" start -if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "ready"; else bad "idempotent start rc=$RC"; fi - -# Capture the live pid for later assertions. -PID="$("$AO_BIN" status --json | sed -n 's/.*"pid": \([0-9]*\).*/\1/p' | head -1)" - -# --------------------------------------------------------------------------- -# 5. doctor while running — now the daemon (not the CLI) has created the store -# --------------------------------------------------------------------------- -section "doctor (running)" - -step "daemon created and migrated the store" -if [ -f "$AO_DATA_DIR/ao.db" ]; then ok; else bad "daemon should have created ao.db"; fi - -step "doctor now reports sqlite present + daemon-migrated" -run_rc "$AO_BIN" doctor -assert_contains "$OUT" "migrations are applied by the daemon" - -# --------------------------------------------------------------------------- -# 6. Health endpoint identity (loopback) -# --------------------------------------------------------------------------- -section "health endpoint" - -if command -v curl >/dev/null 2>&1; then - step "/healthz reports the AO daemon service + pid" - run_rc curl -fsS "http://127.0.0.1:$AO_PORT/healthz" - assert_contains "$OUT" "agent-orchestrator-daemon" - - # ------------------------------------------------------------------------- - # 7. /shutdown CSRF / DNS-rebinding guard (review fix M3) - # ------------------------------------------------------------------------- - section "/shutdown guard" - - step "cross-origin POST /shutdown is rejected (403)" - CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ - -H 'Origin: https://evil.example' "http://127.0.0.1:$AO_PORT/shutdown")" - vdump "curl -X POST -H 'Origin: https://evil.example' http://127.0.0.1:$AO_PORT/shutdown" "HTTP $CODE" "-" - assert_eq "$CODE" "403" "cross-origin shutdown" - - step "non-loopback Host POST /shutdown is rejected (403)" - CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ - -H 'Host: evil.example' "http://127.0.0.1:$AO_PORT/shutdown")" - vdump "curl -X POST -H 'Host: evil.example' http://127.0.0.1:$AO_PORT/shutdown" "HTTP $CODE" "-" - assert_eq "$CODE" "403" "rebinding-host shutdown" - - step "daemon survived the rejected shutdown attempts" - run_rc "$AO_BIN" status --json - assert_contains "$OUT" '"state": "ready"' -else - section "/shutdown guard" - step "curl unavailable — skipping HTTP-level guard checks" - printf '\033[33mSKIP\033[0m\n' -fi - -# --------------------------------------------------------------------------- -# 8. stop → stopped, run-file cleaned up -# --------------------------------------------------------------------------- -section "stop" - -step "stop gracefully stops the daemon" -run_rc "$AO_BIN" stop -if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "stopped"; else bad "stop rc=$RC out=$OUT"; fi - -step "run-file removed after stop" -if [ ! -f "$AO_RUN_FILE" ]; then ok; else bad "running.json still present"; fi - -step "status --json reports stopped after stop" -run_rc "$AO_BIN" status --json -assert_contains "$OUT" '"state": "stopped"' - -# --------------------------------------------------------------------------- -# 9. stale run-file (dead PID) — deterministic, no real process needed -# --------------------------------------------------------------------------- -section "stale run-file" - -# PID 2147483647 is never alive; the CLI must classify this as stale, not kill it. -printf '{"pid":2147483647,"port":%s,"startedAt":"2020-01-01T00:00:00Z"}\n' "$AO_PORT" > "$AO_RUN_FILE" - -step "status reports stale for a dead-PID run-file" -run_rc "$AO_BIN" status --json -assert_contains "$OUT" '"state": "stale"' -step "status still exits 0 for a stale daemon (reports, never errors)" -assert_eq "$RC" "0" "stale status exit code" - -step "stop clears a stale run-file and reports stopped" -run_rc "$AO_BIN" stop -assert_contains "$OUT" "stopped" -step "stale run-file removed" -if [ ! -f "$AO_RUN_FILE" ]; then ok; else bad "stale running.json not removed"; fi - -# --------------------------------------------------------------------------- -# 10. exit codes: 2 for usage errors, 1 for runtime errors -# --------------------------------------------------------------------------- -section "exit codes" - -step "unknown flag exits 2 (usage error)" -run_rc "$AO_BIN" status --definitely-not-a-flag -assert_eq "$RC" "2" "bad-flag exit code" - -step "missing required arg exits 2 (completion needs a shell)" -run_rc "$AO_BIN" completion -assert_eq "$RC" "2" "missing-arg exit code" - -step "unsupported shell exits non-zero (runtime error)" -run_rc "$AO_BIN" completion notashell -if [ "$RC" -ne 0 ]; then ok; else bad "expected non-zero for bad shell"; fi - -step "invalid config (AO_PORT out of range) exits 1, not 2" -OUT="$(AO_PORT=99999 "$AO_BIN" status 2>&1)"; RC=$? -vdump "AO_PORT=99999 $AO_BIN status" "$OUT" "$RC" -assert_eq "$RC" "1" "config-error exit code (runtime, not usage)" - -# --------------------------------------------------------------------------- -# 11. shell completion generators -# --------------------------------------------------------------------------- -section "completion" - -for sh in bash zsh fish powershell; do - step "completion $sh generates a script" - run_rc "$AO_BIN" completion "$sh" - if [ "$RC" -eq 0 ] && [ -n "$OUT" ]; then ok; else bad "completion $sh rc=$RC"; fi -done - -# --------------------------------------------------------------------------- -# Result -# --------------------------------------------------------------------------- -printf '\n\033[1m== result ==\033[0m\n passed: %s\n failed: %s\n' "$PASS" "$FAIL" -if [ "$FAIL" -ne 0 ]; then - printf '\033[31mSMOKE TEST FAILED\033[0m\n' - exit 1 -fi -printf '\033[32mSMOKE TEST PASSED\033[0m\n'