| 1 | #!/usr/bin/env bash |
| 2 | # Tests for app-server-smoke.sh: drive it against a fake `codewhale` binary so |
| 3 | # the stdio-probe assertions, auth-list matrix parser, cheap-model mapping, and |
| 4 | # secret redaction are all exercised without a real build or any model tokens. |
| 5 | |
| 6 | set -euo pipefail |
| 7 | |
| 8 | SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" |
| 9 | SMOKE="$SCRIPT_DIR/app-server-smoke.sh" |
| 10 | TESTS=0 |
| 11 | FAILED=0 |
| 12 | |
| 13 | WORK="$(mktemp -d)" |
| 14 | trap 'rm -rf "$WORK"' EXIT |
| 15 | |
| 16 | FAKE="$WORK/codewhale" |
| 17 | cat >"$FAKE" <<'FAKE_EOF' |
| 18 | #!/usr/bin/env bash |
| 19 | set -euo pipefail |
| 20 | if [[ "${1:-}" == "app-server" ]]; then |
| 21 | cat >/dev/null # drain the JSON-RPC requests |
| 22 | printf '%s\n' \ |
| 23 | '{"jsonrpc":"2.0","id":1,"result":{"status":"ok","service":"deepseek-app-server","transport":"stdio"}}' \ |
| 24 | '{"jsonrpc":"2.0","id":2,"result":{"methods":["thread/request","prompt/run","prompt/request","thread/goal/set"]}}' \ |
| 25 | '{"jsonrpc":"2.0","id":3,"result":{"ok":true,"data":{"transport":"stdio+http"}}}' |
| 26 | exit 0 |
| 27 | fi |
| 28 | if [[ "${1:-}" == "auth" && "${2:-}" == "list" ]]; then |
| 29 | printf '%s\n' \ |
| 30 | 'provider config store env active' \ |
| 31 | 'deepseek yes - no config' \ |
| 32 | 'zai no - yes env' \ |
| 33 | 'openrouter no - no missing' \ |
| 34 | 'arcee yes - no config' |
| 35 | exit 0 |
| 36 | fi |
| 37 | # exec form: --provider <slug> --model <model> exec <prompt> |
| 38 | slug=""; model="" |
| 39 | while [[ $# -gt 0 ]]; do |
| 40 | case "$1" in |
| 41 | --provider) slug="$2"; shift 2 ;; |
| 42 | --model) model="$2"; shift 2 ;; |
| 43 | exec) shift; break ;; |
| 44 | *) shift ;; |
| 45 | esac |
| 46 | done |
| 47 | # Emit a fake secret on the reply line to prove redaction scrubs it. |
| 48 | printf 'pong [%s/%s] token=sk-DEADBEEFCAFE12345\n' "$slug" "$model" |
| 49 | exit 0 |
| 50 | FAKE_EOF |
| 51 | chmod +x "$FAKE" |
| 52 | |
| 53 | # run_smoke <expected_exit> -- <args...> : capture combined output, check exit. |
| 54 | LAST_OUT="" |
| 55 | run_smoke() { |
| 56 | local expected="$1"; shift |
| 57 | [[ "$1" == "--" ]] && shift |
| 58 | local rc=0 |
| 59 | LAST_OUT="$(bash "$SMOKE" --bin "$FAKE" "$@" 2>&1)" || rc=$? |
| 60 | TESTS=$((TESTS + 1)) |
| 61 | if [[ "$rc" != "$expected" ]]; then |
| 62 | printf '\033[1;31mFAIL\033[0m exit %s (wanted %s) for: %s\n' "$rc" "$expected" "$*" |
| 63 | printf '%s\n' "$LAST_OUT" | sed 's/^/ /' |
| 64 | FAILED=$((FAILED + 1)) |
| 65 | return 1 |
| 66 | fi |
| 67 | return 0 |
| 68 | } |
| 69 | |
| 70 | want() { TESTS=$((TESTS + 1)); if printf '%s' "$LAST_OUT" | grep -qF "$1"; then :; else printf '\033[1;31mFAIL\033[0m output missing: %s\n' "$1"; FAILED=$((FAILED + 1)); fi; } |
| 71 | want_not(){ TESTS=$((TESTS + 1)); if printf '%s' "$LAST_OUT" | grep -qF "$1"; then printf '\033[1;31mFAIL\033[0m output should not contain: %s\n' "$1"; FAILED=$((FAILED + 1)); fi; } |
| 72 | |
| 73 | # 1. Default: stdio probe only, passes, matrix skipped. |
| 74 | run_smoke 0 -- || true |
| 75 | want "healthz reports ok" |
| 76 | want "capabilities advertise thread/* methods" |
| 77 | want "app/capabilities reports transport" |
| 78 | want "matrix skipped" |
| 79 | |
| 80 | # 2. Dry-run matrix: maps known providers, flags unmapped, skips 'missing'. |
| 81 | run_smoke 0 -- --matrix || true |
| 82 | want "deepseek -> deepseek-chat" |
| 83 | want "zai -> glm-4-flash" |
| 84 | want "arcee -> (UNMAPPED" |
| 85 | want_not "openrouter" |
| 86 | |
| 87 | # 3. Real exec for a mapped provider passes; secret is redacted. |
| 88 | run_smoke 0 -- --matrix --real --provider deepseek || true |
| 89 | want "deepseek/deepseek-chat exec ok" |
| 90 | want "REDACTED" |
| 91 | want_not "sk-DEADBEEFCAFE12345" |
| 92 | |
| 93 | # 4. Real exec across all configured providers fails on the unmapped one. |
| 94 | run_smoke 1 -- --matrix --real || true |
| 95 | want "arcee has no cheap-model mapping" |
| 96 | |
| 97 | # 5. Override supplies a model for the otherwise-unmapped provider. |
| 98 | TESTS=$((TESTS + 1)) |
| 99 | LAST_OUT="$(SMOKE_MODEL_ARCEE=arcee-cheap bash "$SMOKE" --bin "$FAKE" --matrix 2>&1)" || { FAILED=$((FAILED + 1)); printf 'FAIL override run errored\n'; } |
| 100 | want "arcee -> arcee-cheap" |
| 101 | |
| 102 | echo "" |
| 103 | if [[ "$FAILED" -eq 0 ]]; then |
| 104 | printf '\033[1;32mapp-server-smoke.test.sh: all %s checks passed\033[0m\n' "$TESTS" |
| 105 | else |
| 106 | printf '\033[1;31mapp-server-smoke.test.sh: %s/%s checks failed\033[0m\n' "$FAILED" "$TESTS" |
| 107 | exit 1 |
| 108 | fi |
| 109 |