From 3680ac5474196faf080d8c20f3421e2968c3462f Mon Sep 17 00:00:00 2001 From: itrytoohard Date: Mon, 1 Jun 2026 02:33:56 +0530 Subject: [PATCH] test(cli): add end-to-end smoke test + Docker/CI harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a fresh-machine, install→use→verify E2E test for the `ao` CLI and wires it into CI. The suite drives the real binary (start/status/doctor/stop + the daemon-control HTTP surface) against fully isolated state — its own temp run-file, data dir, and an auto-picked free loopback port — so it never collides with a developer's real AO install or daemon. - test/cli/smoke.sh: 40 assertions covering install resolution, version/help (daemon hidden), doctor text+json (and that it does NOT migrate SQLite), status stopped/stale/ready, start fresh+idempotent, daemon-created store, /healthz identity, the /shutdown CSRF + DNS-rebinding guard (403 + survives), graceful/stale/idempotent stop, run-file ownership cleanup, exit codes (2 usage / 1 runtime), and completion for all four shells. It deliberately ignores an inherited AO_PORT and self-allocates a free port for isolation. - test/cli/Dockerfile: models installing ao on a fresh machine — builds the binary, drops it on PATH in a clean Debian image with only runtime deps (git/tmux/curl), runs the suite as a non-root user. - test/cli/run-local.sh: build-from-source + native run convenience wrapper. - .github/workflows/cli-e2e.yml: two tiers — `native` runs the suite on a ubuntu+macos runner matrix (the real VMs, to cover the unix setsid detach and macOS os.UserConfigDir paths a Linux container can't), and `container` runs the fresh-machine Docker image with --init (real PID-1 reaper so the stale-daemon assertion is reliable). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cli-e2e.yml | 55 +++++++ test/cli/Dockerfile | 46 ++++++ test/cli/README.md | 73 +++++++++ test/cli/run-local.sh | 24 +++ test/cli/smoke.sh | 299 ++++++++++++++++++++++++++++++++++ 5 files changed, 497 insertions(+) create mode 100644 .github/workflows/cli-e2e.yml create mode 100644 test/cli/Dockerfile create mode 100644 test/cli/README.md create mode 100755 test/cli/run-local.sh create mode 100755 test/cli/smoke.sh diff --git a/.github/workflows/cli-e2e.yml b/.github/workflows/cli-e2e.yml new file mode 100644 index 000000000..e9daf56c1 --- /dev/null +++ b/.github/workflows/cli-e2e.yml @@ -0,0 +1,55 @@ +name: CLI E2E + +on: + push: + branches: [main] + pull_request: + paths: + - "backend/**" + - "test/cli/**" + - ".github/workflows/cli-e2e.yml" + +permissions: + contents: read + +jobs: + # Primary tier: run the REAL `ao` binary on GitHub's native VM runners. These + # runners are the "VMs" — the only place that exercises the OS-specific code + # paths (unix Setsid process-group detach + macOS os.UserConfigDir resolution). + # Bash is available on both ubuntu and macos runners, so the one smoke.sh runs + # unchanged. State is isolated per run (own temp dir + a free loopback port). + native: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: false + + - name: Build ao + run: cd backend && CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/ao" ./cmd/ao + + - name: CLI smoke test + run: AO_BIN="$RUNNER_TEMP/ao" bash test/cli/smoke.sh + + # Secondary hardening tier: model "install ao on a fresh machine" in a clean, + # locked-down Linux container with no access to a developer's real state. + # --init gives the container a real PID-1 reaper (tini) so the stale-daemon + # assertion is reliable — without it, an orphaned daemon can linger as a + # zombie and skew the check. + container: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build smoke image (fresh-machine install) + run: docker build -f test/cli/Dockerfile -t ao-cli-smoke . + + - name: Run CLI smoke test in container + run: docker run --rm --init ao-cli-smoke diff --git a/test/cli/Dockerfile b/test/cli/Dockerfile new file mode 100644 index 000000000..dfeb4c46f --- /dev/null +++ b/test/cli/Dockerfile @@ -0,0 +1,46 @@ +# End-to-end CLI smoke test, modelling "install ao on a fresh machine, then use it". +# +# Build context is the REPO ROOT: +# docker build -f test/cli/Dockerfile -t ao-cli-smoke . +# docker run --rm --init ao-cli-smoke +# +# --init gives the container a real PID-1 reaper (tini) so a stopped daemon is +# reaped promptly; the test is written to not depend on it, but it keeps process +# accounting clean. + +# ---- stage 1: build the binary (the "release" a user would download) ---- +FROM golang:1.25-bookworm AS build +WORKDIR /src + +# Cache modules first. +COPY backend/go.mod backend/go.sum ./backend/ +RUN cd backend && go mod download + +COPY backend ./backend +# Pure-Go SQLite (modernc) builds fine with CGO disabled -> a static binary. +RUN cd backend && CGO_ENABLED=0 go build -trimpath -o /out/ao ./cmd/ao + +# ---- stage 2: a clean machine with NO Go toolchain, just like an end user ---- +FROM debian:bookworm-slim AS run + +# Runtime deps a fresh user would need: git is required by `ao doctor`; tmux is +# the optional runtime it probes for; curl drives the HTTP-level guard checks; +# ca-certificates for good measure. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git tmux curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# "Install" the CLI the way a user would: drop the binary on PATH. +COPY --from=build /out/ao /usr/local/bin/ao +COPY test/cli/smoke.sh /usr/local/bin/ao-smoke.sh +RUN chmod +x /usr/local/bin/ao /usr/local/bin/ao-smoke.sh + +# Run as an unprivileged user with a real HOME, like a normal install. +RUN useradd --create-home --shell /bin/bash ao +USER ao +WORKDIR /home/ao + +# Sanity: prove the install resolved before the suite runs. +RUN ao version + +ENTRYPOINT ["/usr/local/bin/ao-smoke.sh"] diff --git a/test/cli/README.md b/test/cli/README.md new file mode 100644 index 000000000..51ce67fe4 --- /dev/null +++ b/test/cli/README.md @@ -0,0 +1,73 @@ +# `ao` CLI end-to-end tests + +These tests install and drive the **real `ao` binary** the way a user would — +`start` → `status` → `doctor` → `stop`, plus the daemon-control HTTP surface — +and assert the whole thing works end to end. They run against **isolated, +throwaway state** (their own temp run-file + data dir + a free loopback port), +so they never touch a developer's real AO installation. + +## Files + +| File | Purpose | +|------|---------| +| `smoke.sh` | The suite. Host-agnostic bash; drives the binary at `$AO_BIN` (default `ao` on PATH) and prints a PASS/FAIL line per assertion. | +| `Dockerfile` | Models **installing `ao` on a fresh machine**: builds the binary, drops it on `PATH` in a clean Debian image with only runtime deps (`git`, `tmux`, `curl`), then runs `smoke.sh` as a non-root user. | +| `run-local.sh` | Convenience wrapper: build from source and run `smoke.sh` natively against a temp binary. | + +## Run it + +**Native (fastest, uses your toolchain):** +```bash +test/cli/run-local.sh +# or, against a binary you already built: +AO_BIN=/path/to/ao test/cli/smoke.sh +``` + +**Fresh-machine install, in a clean container:** +```bash +docker build -f test/cli/Dockerfile -t ao-cli-smoke . +docker run --rm --init ao-cli-smoke +``` +> `--init` is important: it gives the container a real PID-1 reaper so the +> "stale daemon" assertion is reliable. Without it an orphaned daemon can linger +> as a zombie and skew the check. + +## What it covers + +Install resolves on PATH · `version`/`--version` · `--help` (and hides the +internal `daemon` command) · `doctor` text + `--json` (and that it **does not** +open/migrate SQLite) · `status` stopped/stale/ready · `start` (fresh + +idempotent) · daemon-created store · `/healthz` identity · the `/shutdown` +CSRF/DNS-rebinding guard (403 + daemon survives) · `stop` (graceful + stale + +idempotent) · run-file cleanup/ownership · exit codes (`2` usage, `1` runtime) · +completion for all four shells. + +## Testing strategy — why it's shaped this way + +We deliberately don't make Docker the *only* tier. A daemon that detaches with +`setsid` and outlives the launching process is exactly the workload that +container PID-1 semantics mishandle, and the OS-specific bits (`setsid` vs +Windows `CREATE_NEW_PROCESS_GROUP`, and `os.UserConfigDir()` resolving to +`~/Library/Application Support` on macOS, `%AppData%` on Windows, `~/.config` +on Linux) can't be observed from a Linux container at all. + +So CI (`.github/workflows/cli-e2e.yml`) runs two tiers: + +1. **`native`** — the primary signal. Builds and runs the real binary on a + GitHub matrix of `ubuntu-latest` + `macos-latest` (those runners *are* the + VMs), covering the unix detach path and macOS config-dir resolution. +2. **`container`** — a hardening tier. The `Dockerfile` proves a clean-machine + install works and that the CLI has no hidden dependence on developer state, + run with `--init`. + +### Extending + +- Add an assertion: drop a `step`/`assert_*` pair into the relevant section of + `smoke.sh`. The helpers (`assert_eq`, `assert_contains`, `assert_not_contains`, + `run_rc`) keep cases one-liners. +- Cover Windows: add a `windows-latest` leg to the `native` matrix (Git Bash + ships on the runner) once the suite is confirmed green there, or add Go-based + `os/exec` E2E tests for the Windows process-group path. +- Deeper per-OS path assertions (that state resolves under the OS-native config + dir when `AO_RUN_FILE`/`AO_DATA_DIR` are unset) are best added as Go unit + tests in `internal/config`. diff --git a/test/cli/run-local.sh b/test/cli/run-local.sh new file mode 100755 index 000000000..4cd786a6e --- /dev/null +++ b/test/cli/run-local.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Convenience wrapper: build `ao` from source and run the CLI smoke test against +# it natively, using an isolated temp state dir and a free port. Touches nothing +# in your real AO installation. +# +# test/cli/run-local.sh +# +# To run the same suite the way a brand-new user would install it (clean Linux +# container, binary on PATH), use Docker instead: +# +# docker build -f test/cli/Dockerfile -t ao-cli-smoke . && docker run --rm --init ao-cli-smoke + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +bindir="$(mktemp -d)" +trap 'rm -rf "$bindir"' EXIT + +echo "building ao ..." +( cd "$repo_root/backend" && CGO_ENABLED=0 go build -trimpath -o "$bindir/ao" ./cmd/ao ) + +echo "running smoke test ..." +AO_BIN="$bindir/ao" bash "$repo_root/test/cli/smoke.sh" diff --git a/test/cli/smoke.sh b/test/cli/smoke.sh new file mode 100755 index 000000000..46bd1a4b5 --- /dev/null +++ b/test/cli/smoke.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash +# +# End-to-end smoke test for the `ao` CLI. +# +# It models a *fresh machine*: `ao` is expected to already be installed on PATH +# (the Dockerfile in this directory installs it, simulating a new user), and the +# whole test runs against isolated, throwaway state — its own temp run-file, +# data dir, and a free loopback port — so it never touches a developer's real +# AO installation. +# +# Run locally against a binary you built: +# AO_BIN=/path/to/ao test/cli/smoke.sh +# Or in the container (models install-on-a-fresh-machine): +# docker build -f test/cli/Dockerfile -t ao-cli-smoke . && docker run --rm --init ao-cli-smoke +# +# Exit code: 0 if every assertion passes, 1 otherwise. + +set -uo pipefail + +AO_BIN="${AO_BIN:-ao}" + +# --------------------------------------------------------------------------- +# Tiny assertion framework +# --------------------------------------------------------------------------- +PASS=0 +FAIL=0 +CURRENT="" + +section() { printf '\n\033[1m== %s ==\033[0m\n' "$1"; } +step() { CURRENT="$1"; printf ' • %s ... ' "$1"; } + +ok() { PASS=$((PASS + 1)); printf '\033[32mPASS\033[0m\n'; } +bad() { FAIL=$((FAIL + 1)); printf '\033[31mFAIL\033[0m\n %s\n' "$1"; } + +# assert_eq [msg] +assert_eq() { + if [ "$1" = "$2" ]; then ok; else bad "${3:-}: expected [$2], got [$1]"; fi +} +# assert_contains +assert_contains() { + case "$1" in + *"$2"*) ok ;; + *) bad "expected output to contain [$2]; got: $(printf '%s' "$1" | head -c 400)" ;; + esac +} +# assert_not_contains +assert_not_contains() { + case "$1" in + *"$2"*) bad "expected output to NOT contain [$2]" ;; + *) ok ;; + esac +} + +# run_rc -> sets RC and OUT (stdout+stderr combined) +run_rc() { + OUT="$("$@" 2>&1)" + RC=$? +} + +# --------------------------------------------------------------------------- +# Isolated, throwaway environment +# --------------------------------------------------------------------------- +TMP="$(mktemp -d)" +export AO_RUN_FILE="$TMP/running.json" +export AO_DATA_DIR="$TMP/data" + +# Pick a free loopback port (bash /dev/tcp probe; connect-refused == free). +find_free_port() { + local p + for p in $(seq 3071 3170); do + if ! (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null; then + echo "$p"; return 0 + fi + exec 3>&- 2>/dev/null || true + done + echo 3071 +} +# Always run against an isolated, free port. We deliberately do NOT honour an +# inherited AO_PORT — it might point at a real daemon, which is exactly the +# collision this isolation is meant to prevent. Override only via AO_SMOKE_PORT. +export AO_PORT="${AO_SMOKE_PORT:-$(find_free_port)}" + +cleanup() { + "$AO_BIN" stop >/dev/null 2>&1 || true + rm -rf "$TMP" +} +trap cleanup EXIT + +printf 'ao smoke test\n binary : %s\n port : %s\n state : %s\n' \ + "$(command -v "$AO_BIN" || echo "$AO_BIN")" "$AO_PORT" "$TMP" + +# --------------------------------------------------------------------------- +# 1. Install verification — `ao` is a real, runnable binary on this machine +# --------------------------------------------------------------------------- +section "install" + +step "ao resolves on PATH / at AO_BIN" +if command -v "$AO_BIN" >/dev/null 2>&1; then ok; else bad "ao not found"; fi + +step "ao version prints build metadata" +run_rc "$AO_BIN" version +if [ "$RC" -eq 0 ] && [ -n "$OUT" ]; then ok; else bad "rc=$RC out=$OUT"; fi + +step "ao --version works" +run_rc "$AO_BIN" --version +assert_eq "$RC" "0" "--version rc" + +step "ao --help lists product commands" +run_rc "$AO_BIN" --help +assert_contains "$OUT" "start" +step "ao --help lists status/stop/doctor" +assert_contains "$OUT" "doctor" +step "ao --help hides internal daemon command" +assert_not_contains "$OUT" $'\n daemon' + +# --------------------------------------------------------------------------- +# 2. doctor on a fresh machine (no daemon yet) +# --------------------------------------------------------------------------- +section "doctor (fresh)" + +step "doctor exits 0 when required tools present" +run_rc "$AO_BIN" doctor +assert_eq "$RC" "0" "doctor rc (git/tmux must be installed in the image)" + +step "doctor reports git found" +assert_contains "$OUT" "git" + +step "doctor does NOT migrate the store (sqlite WARN, db absent)" +assert_contains "$OUT" "database not created yet" + +step "doctor data dir was created but ao.db was NOT (CLI is not the store writer)" +if [ ! -f "$AO_DATA_DIR/ao.db" ]; then ok; else bad "ao.db exists — doctor must not create/migrate the DB"; fi + +step "doctor --json is valid JSON with ok=true" +run_rc "$AO_BIN" doctor --json +assert_contains "$OUT" '"ok": true' + +# --------------------------------------------------------------------------- +# 3. status when stopped +# --------------------------------------------------------------------------- +section "status (stopped)" + +step "status --json reports stopped" +run_rc "$AO_BIN" status --json +assert_contains "$OUT" '"state": "stopped"' +step "status exits 0 even when stopped (status never errors)" +assert_eq "$RC" "0" "status exit code" +step "stopped status omits startedAt" +assert_not_contains "$OUT" "startedAt" + +step "stop is idempotent when already stopped" +run_rc "$AO_BIN" stop +if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "stopped"; else bad "stop-when-stopped rc=$RC"; fi + +# --------------------------------------------------------------------------- +# 4. start → ready, and status reflects it +# --------------------------------------------------------------------------- +section "start" + +step "start brings the daemon up and reports ready" +run_rc "$AO_BIN" start +if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "ready"; else bad "start rc=$RC out=$OUT"; fi + +step "status --json reports ready with pid+port" +run_rc "$AO_BIN" status --json +assert_contains "$OUT" '"state": "ready"' +step "status carries the bound port" +assert_contains "$OUT" "\"port\": $AO_PORT" + +step "start is idempotent (second start returns ready, no error)" +run_rc "$AO_BIN" start +if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "ready"; else bad "idempotent start rc=$RC"; fi + +# Capture the live pid for later assertions. +PID="$("$AO_BIN" status --json | sed -n 's/.*"pid": \([0-9]*\).*/\1/p' | head -1)" + +# --------------------------------------------------------------------------- +# 5. doctor while running — now the daemon (not the CLI) has created the store +# --------------------------------------------------------------------------- +section "doctor (running)" + +step "daemon created and migrated the store" +if [ -f "$AO_DATA_DIR/ao.db" ]; then ok; else bad "daemon should have created ao.db"; fi + +step "doctor now reports sqlite present + daemon-migrated" +run_rc "$AO_BIN" doctor +assert_contains "$OUT" "migrations are applied by the daemon" + +# --------------------------------------------------------------------------- +# 6. Health endpoint identity (loopback) +# --------------------------------------------------------------------------- +section "health endpoint" + +if command -v curl >/dev/null 2>&1; then + step "/healthz reports the AO daemon service + pid" + run_rc curl -fsS "http://127.0.0.1:$AO_PORT/healthz" + assert_contains "$OUT" "agent-orchestrator-daemon" + + # ------------------------------------------------------------------------- + # 7. /shutdown CSRF / DNS-rebinding guard (review fix M3) + # ------------------------------------------------------------------------- + section "/shutdown guard" + + step "cross-origin POST /shutdown is rejected (403)" + CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'Origin: https://evil.example' "http://127.0.0.1:$AO_PORT/shutdown")" + assert_eq "$CODE" "403" "cross-origin shutdown" + + step "non-loopback Host POST /shutdown is rejected (403)" + CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H 'Host: evil.example' "http://127.0.0.1:$AO_PORT/shutdown")" + assert_eq "$CODE" "403" "rebinding-host shutdown" + + step "daemon survived the rejected shutdown attempts" + run_rc "$AO_BIN" status --json + assert_contains "$OUT" '"state": "ready"' +else + section "/shutdown guard" + step "curl unavailable — skipping HTTP-level guard checks" + printf '\033[33mSKIP\033[0m\n' +fi + +# --------------------------------------------------------------------------- +# 8. stop → stopped, run-file cleaned up +# --------------------------------------------------------------------------- +section "stop" + +step "stop gracefully stops the daemon" +run_rc "$AO_BIN" stop +if [ "$RC" -eq 0 ]; then assert_contains "$OUT" "stopped"; else bad "stop rc=$RC out=$OUT"; fi + +step "run-file removed after stop" +if [ ! -f "$AO_RUN_FILE" ]; then ok; else bad "running.json still present"; fi + +step "status --json reports stopped after stop" +run_rc "$AO_BIN" status --json +assert_contains "$OUT" '"state": "stopped"' + +# --------------------------------------------------------------------------- +# 9. stale run-file (dead PID) — deterministic, no real process needed +# --------------------------------------------------------------------------- +section "stale run-file" + +# PID 2147483647 is never alive; the CLI must classify this as stale, not kill it. +printf '{"pid":2147483647,"port":%s,"startedAt":"2020-01-01T00:00:00Z"}\n' "$AO_PORT" > "$AO_RUN_FILE" + +step "status reports stale for a dead-PID run-file" +run_rc "$AO_BIN" status --json +assert_contains "$OUT" '"state": "stale"' +step "status still exits 0 for a stale daemon (reports, never errors)" +assert_eq "$RC" "0" "stale status exit code" + +step "stop clears a stale run-file and reports stopped" +run_rc "$AO_BIN" stop +assert_contains "$OUT" "stopped" +step "stale run-file removed" +if [ ! -f "$AO_RUN_FILE" ]; then ok; else bad "stale running.json not removed"; fi + +# --------------------------------------------------------------------------- +# 10. exit codes: 2 for usage errors, 1 for runtime errors +# --------------------------------------------------------------------------- +section "exit codes" + +step "unknown flag exits 2 (usage error)" +run_rc "$AO_BIN" status --definitely-not-a-flag +assert_eq "$RC" "2" "bad-flag exit code" + +step "missing required arg exits 2 (completion needs a shell)" +run_rc "$AO_BIN" completion +assert_eq "$RC" "2" "missing-arg exit code" + +step "unsupported shell exits non-zero (runtime error)" +run_rc "$AO_BIN" completion notashell +if [ "$RC" -ne 0 ]; then ok; else bad "expected non-zero for bad shell"; fi + +step "invalid config (AO_PORT out of range) exits 1, not 2" +OUT="$(AO_PORT=99999 "$AO_BIN" status 2>&1)"; RC=$? +assert_eq "$RC" "1" "config-error exit code (runtime, not usage)" + +# --------------------------------------------------------------------------- +# 11. shell completion generators +# --------------------------------------------------------------------------- +section "completion" + +for sh in bash zsh fish powershell; do + step "completion $sh generates a script" + run_rc "$AO_BIN" completion "$sh" + if [ "$RC" -eq 0 ] && [ -n "$OUT" ]; then ok; else bad "completion $sh rc=$RC"; fi +done + +# --------------------------------------------------------------------------- +# Result +# --------------------------------------------------------------------------- +printf '\n\033[1m== result ==\033[0m\n passed: %s\n failed: %s\n' "$PASS" "$FAIL" +if [ "$FAIL" -ne 0 ]; then + printf '\033[31mSMOKE TEST FAILED\033[0m\n' + exit 1 +fi +printf '\033[32mSMOKE TEST PASSED\033[0m\n'