neuron/tests/gate-openai/selftest.sh

#!/usr/bin/env bash
# selftest.sh - proves stub-openai.py before any brain code exists.
# Drives the stub with curl through every scenario (plain, tools-off,
# single tool round-trip, escaping torture, parallel double-call,
# two-round mission, injected API errors, background, overrun), every
# validation rejection (dialect leaks, pairing, echo round-trip, scenario
# expectations), and all three hostile modes. Exit 0 = green.
set -u
cd "$(dirname "$0")" || exit 1
PY=python3
TMP="$(mktemp -d)"
PIDS=()
cleanup() {
  for p in "${PIDS[@]:-}"; do kill -9 "$p" >/dev/null 2>&1; done
  rm -rf "$TMP"
}
trap cleanup EXIT

PASS=0; FAIL=0
ok()  { printf 'ok   - %s\n' "$1"; PASS=$((PASS+1)); }
bad() { printf 'FAIL - %s\n' "$1"; FAIL=$((FAIL+1)); }
check() {  # check <name> <cmd...>  - pass if cmd exits 0; show output on fail
  local name="$1"; shift
  local out
  if out="$("$@" 2>&1)"; then ok "$name"
  else bad "$name"; [ -n "$out" ] && printf '%s\n' "$out" | sed 's/^/       /' | head -8
  fi
}

freeport() { "$PY" -c 'import socket;s=socket.socket();s.bind(("127.0.0.1",0));print(s.getsockname()[1]);s.close()'; }
waithealth() {
  local p="$1" i
  for i in $(seq 1 60); do
    curl -sf "http://127.0.0.1:$p/gate/health" >/dev/null 2>&1 && return 0
    sleep 0.1
  done
  echo "stub on :$p never became healthy"; return 1
}
post() {  # post <port> <bodyfile> <respfile> [extra curl args...] -> echoes http code
  local port="$1" body="$2" resp="$3"; shift 3
  curl -s -o "$resp" -w '%{http_code}' -H 'content-type: application/json' \
       "$@" --data-binary @"$body" "http://127.0.0.1:$port/v1/chat/completions"
}

# ---- embedded helper: builds OpenAI-dialect bodies, asserts on responses ----
cat > "$TMP/helpers.py" <<'PYEOF'
import copy, json, sys

TOOLS = [
    {"type": "function", "function": {
        "name": "write_file", "description": "Write content to a file on disk.",
        "parameters": {"type": "object",
                       "properties": {"path": {"type": "string"},
                                      "content": {"type": "string"}},
                       "required": ["path", "content"]}}},
    {"type": "function", "function": {
        "name": "read_file", "description": "Read contents of a file from disk.",
        "parameters": {"type": "object",
                       "properties": {"path": {"type": "string"}},
                       "required": ["path"]}}},
]

def dump(obj, out):
    json.dump(obj, open(out, "w"), ensure_ascii=False)

def base(prompt, tools=True):
    b = {"model": "gate-openai-model", "max_tokens": 1024,
         "messages": [
             {"role": "system", "content": "You are Neuron (gate fixture)."},
             {"role": "user", "content": prompt}]}
    if tools:
        b["tools"] = copy.deepcopy(TOOLS)
        b["tool_choice"] = "auto"
        b["parallel_tool_calls"] = False
    return b

def cmd_plain(out, prompt):
    dump(base(prompt), out)

def cmd_notools(out, prompt):
    dump(base(prompt, tools=False), out)

def cmd_mut(out, prompt, mutation):
    b = base(prompt)
    if mutation == "no-tool-choice":
        del b["tool_choice"]
    elif mutation == "ptc-true":
        b["parallel_tool_calls"] = True
    elif mutation == "top-system":
        b["system"] = "You are Neuron."
    elif mutation == "anth-tools":
        b["tools"] = [{"name": "write_file", "description": "x",
                       "input_schema": {"type": "object", "properties": {}}}]
    elif mutation == "anth-block":
        b["messages"][1] = {"role": "user", "content": [
            {"type": "tool_result", "tool_use_id": "toolu_x", "content": "hi"},
            {"type": "text", "text": prompt}]}
    else:
        raise SystemExit("unknown mutation " + mutation)
    dump(b, out)

def cmd_chain(out, prompt, variant, *resps):
    """Build the next leg: echo each response's assistant turn and answer its
    tool calls. `variant` applies to the LAST response only:
    ok | no-tool-turn | wrong-id | only-first | double-encode | object-args"""
    b = base(prompt)
    for idx, p in enumerate(resps):
        last = idx == len(resps) - 1
        msg = json.load(open(p))["choices"][0]["message"]
        tcs = msg.get("tool_calls")
        if not tcs:
            b["messages"].append({"role": "assistant",
                                  "content": msg.get("content")})
            continue
        v = variant if last else "ok"
        asst = {"role": "assistant", "content": msg.get("content"),
                "tool_calls": copy.deepcopy(tcs)}
        if v == "double-encode":
            for tc in asst["tool_calls"]:
                tc["function"]["arguments"] = json.dumps(
                    tc["function"]["arguments"])
        if v == "object-args":
            for tc in asst["tool_calls"]:
                tc["function"]["arguments"] = json.loads(
                    tc["function"]["arguments"])
        b["messages"].append(asst)
        if v == "no-tool-turn":
            continue
        use = tcs[:1] if v == "only-first" else tcs
        for tc in use:
            tid = "call_bogus_123" if v == "wrong-id" else tc["id"]
            b["messages"].append({"role": "tool", "tool_call_id": tid,
                                  "content": "{\"ok\":true,\"bytes\":42}"})
    dump(b, out)

def cmd_chk(resp, expr):
    r = json.load(open(resp))
    if not eval(expr, {"r": r, "json": json, "len": len, "str": str,
                       "isinstance": isinstance, "any": any, "all": all,
                       "sorted": sorted}):
        print("assertion failed:", expr)
        print("resp:", json.dumps(r, ensure_ascii=False)[:400])
        raise SystemExit(1)

def cmd_torture(resp, scen):
    r = json.load(open(resp))
    tc = r["choices"][0]["message"]["tool_calls"][0]
    raw = tc["function"]["arguments"]
    assert isinstance(raw, str), "arguments must be a JSON-encoded string"
    got = json.loads(raw)
    exp = json.load(open(scen))["classes"]["oa-torture"]["script"][0]["tool_calls"][0]["arguments"]
    assert got == exp, "decoded arguments != scripted torture payload"
    content = got["content"]
    for needle in ['"', "\\", "\n", "\t", "日本語", "naïve", "🚀"]:
        assert needle in content, "missing torture needle %r" % needle

def cmd_notjson(path):
    data = open(path, "rb").read()
    assert data, "file empty - no partial body arrived"
    try:
        json.loads(data.decode("utf-8", "replace"))
    except ValueError:
        return
    raise SystemExit("partial body unexpectedly parsed as complete JSON")

def cmd_pending(*paths):
    ids = []
    for p in paths:
        c = json.load(open(p))["choices"][0]
        assert c["finish_reason"] == "tool_calls", c["finish_reason"]
        tc = c["message"]["tool_calls"][0]
        assert tc["function"]["name"] == "write_file"
        json.loads(tc["function"]["arguments"])  # must decode
        ids.append(tc["id"])
    assert len(set(ids)) == len(ids), "call ids not distinct: %r" % ids

def cmd_logcheck(path):
    recs = [json.loads(l) for l in open(path) if l.strip()]
    seqs = [r["seq"] for r in recs]
    assert seqs == sorted(seqs) and len(set(seqs)) == len(seqs), "seq not monotonic"
    kinds = {}
    for r in recs:
        kinds[r["kind"]] = kinds.get(r["kind"], 0) + 1
    assert kinds.get("scenario", 0) >= 10, "too few scenario records: %r" % kinds
    assert kinds.get("background", 0) >= 1, "no background record"
    assert kinds.get("overrun", 0) >= 1, "no overrun record"
    rejected = [r for r in recs if r["validation"] == "rejected"]
    assert len(rejected) >= 10, "too few rejected records: %d" % len(rejected)
    assert any(r["delivered"].get("tool_calls") == ["write_file"]
               for r in recs), "no single write_file ground truth"
    assert any(r["delivered"].get("tool_calls") == ["write_file", "write_file"]
               for r in recs), "no parallel ground truth"

def main():
    fn = globals()["cmd_" + sys.argv[1].replace("-", "_")]
    fn(*sys.argv[2:])

if __name__ == "__main__":
    main()
PYEOF
mk() { "$PY" "$TMP/helpers.py" "$@"; }

echo "=== stub-openai selftest ==="

# ---- normal mode ------------------------------------------------------------
PORT="$(freeport)"
"$PY" stub-openai.py --port "$PORT" --scenarios scenarios-openai.json \
     --log "$TMP/req.jsonl" >"$TMP/stub.out" 2>&1 &
PIDS+=($!); disown
check "stub starts and answers /gate/health" waithealth "$PORT"

# 1. plain completion
mk plain "$TMP/plain.json" "oa-gate plain probe: explain the fixture topic simply."
code="$(post "$PORT" "$TMP/plain.json" "$TMP/r_plain.json")"
check "plain: HTTP 200" test "$code" = "200"
check "plain: chat.completion envelope, finish stop, real content" mk chk "$TMP/r_plain.json" \
  'r["object"]=="chat.completion" and r["choices"][0]["finish_reason"]=="stop" and isinstance(r["choices"][0]["message"]["content"],str) and len(r["choices"][0]["message"]["content"])>40'

# 2. tools-off lane (chat-only request accepted, tool-bearing request refused)
mk notools "$TMP/toolsoff.json" "oa-gate tools-off probe: plain chat with no tools offered."
code="$(post "$PORT" "$TMP/toolsoff.json" "$TMP/r_toolsoff.json")"
check "tools-off: chat-only request -> 200" test "$code" = "200"
mk plain "$TMP/toolsoff_bad.json" "oa-gate tools-off probe: plain chat with no tools offered."
code="$(post "$PORT" "$TMP/toolsoff_bad.json" "$TMP/r_toolsoff_bad.json")"
check "tools-off negative: offering tools -> 400 gate_expect" \
  bash -c "test $code = 400"
check "tools-off negative: reason names gate_expect" mk chk "$TMP/r_toolsoff_bad.json" \
  'r["error"]["code"]=="gate_expect"'

# 3. dialect-leak rejections (the loud-failure contract)
code="$(post "$PORT" "$TMP/plain.json" "$TMP/r_leak_hdr.json" -H 'anthropic-version: 2023-06-01')"
check "leak: anthropic-version header -> 400" test "$code" = "400"
check "leak: header reason names the leak" mk chk "$TMP/r_leak_hdr.json" \
  'r["error"]["code"]=="gate_dialect_leak" and "anthropic-version" in r["error"]["message"]'
mk mut "$TMP/leak_tools.json" "oa-gate plain probe: explain the fixture topic simply." anth-tools
code="$(post "$PORT" "$TMP/leak_tools.json" "$TMP/r_leak_tools.json")"
check "leak: input_schema tools -> 400 gate_dialect_leak" bash -c \
  "test $code = 400"
check "leak: input_schema reason" mk chk "$TMP/r_leak_tools.json" \
  'r["error"]["code"]=="gate_dialect_leak" and "input_schema" in r["error"]["message"]'
mk mut "$TMP/leak_sys.json" "oa-gate plain probe: explain the fixture topic simply." top-system
code="$(post "$PORT" "$TMP/leak_sys.json" "$TMP/r_leak_sys.json")"
check "leak: top-level system -> 400" test "$code" = "400"
mk mut "$TMP/leak_block.json" "oa-gate plain probe: explain the fixture topic simply." anth-block
code="$(post "$PORT" "$TMP/leak_block.json" "$TMP/r_leak_block.json")"
check "leak: Anthropic tool_result content block -> 400" test "$code" = "400"

# 4. scenario request expectations
mk mut "$TMP/no_tc.json" "oa-gate plain probe: explain the fixture topic simply." no-tool-choice
code="$(post "$PORT" "$TMP/no_tc.json" "$TMP/r_no_tc.json")"
check "expect: missing tool_choice -> 400" test "$code" = "400"
mk mut "$TMP/ptc.json" "oa-gate plain probe: explain the fixture topic simply." ptc-true
code="$(post "$PORT" "$TMP/ptc.json" "$TMP/r_ptc.json")"
check "expect: parallel_tool_calls true -> 400 (ADR-0005 pin)" test "$code" = "400"

# 5. single tool round-trip
ST_PROMPT="oa-gate single tool note: save the fixture note to a file."
mk plain "$TMP/st1.json" "$ST_PROMPT"
code="$(post "$PORT" "$TMP/st1.json" "$TMP/r_st1.json")"
check "single-tool leg1: HTTP 200" test "$code" = "200"
check "single-tool leg1: one write_file call, finish tool_calls, string args" mk chk "$TMP/r_st1.json" \
  'r["choices"][0]["finish_reason"]=="tool_calls" and len(r["choices"][0]["message"]["tool_calls"])==1 and r["choices"][0]["message"]["tool_calls"][0]["type"]=="function" and r["choices"][0]["message"]["tool_calls"][0]["function"]["name"]=="write_file" and isinstance(r["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],str) and json.loads(r["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"])["path"]=="openai-single-note.md"'
mk chain "$TMP/st2.json" "$ST_PROMPT" ok "$TMP/r_st1.json"
code="$(post "$PORT" "$TMP/st2.json" "$TMP/r_st2.json")"
check "single-tool leg2: echo + tool turn -> 200 final text" test "$code" = "200"
check "single-tool leg2: final names the file, finish stop" mk chk "$TMP/r_st2.json" \
  'r["choices"][0]["finish_reason"]=="stop" and "openai-single-note.md" in r["choices"][0]["message"]["content"]'
mk chain "$TMP/st2_no.json" "$ST_PROMPT" no-tool-turn "$TMP/r_st1.json"
code="$(post "$PORT" "$TMP/st2_no.json" "$TMP/r_st2_no.json")"
check "single-tool negative: echo without tool turn -> 400 gate_pairing" \
  bash -c "test $code = 400"
check "single-tool negative: pairing reason" mk chk "$TMP/r_st2_no.json" \
  'r["error"]["code"]=="gate_pairing"'
mk chain "$TMP/st2_wrong.json" "$ST_PROMPT" wrong-id "$TMP/r_st1.json"
code="$(post "$PORT" "$TMP/st2_wrong.json" "$TMP/r_st2_wrong.json")"
check "single-tool negative: wrong tool_call_id -> 400" test "$code" = "400"
mk chain "$TMP/st2_obj.json" "$ST_PROMPT" object-args "$TMP/r_st1.json"
code="$(post "$PORT" "$TMP/st2_obj.json" "$TMP/r_st2_obj.json")"
check "single-tool negative: arguments echoed as object -> 400 shape" \
  bash -c "test $code = 400"
check "single-tool negative: shape reason names STRING" mk chk "$TMP/r_st2_obj.json" \
  'r["error"]["code"]=="gate_tool_call_shape" and "STRING" in r["error"]["message"]'

# 6. escaping torture (the two-escaper trap, spec section 6)
T_PROMPT="oa-gate torture probe: write the escaping torture file."
mk plain "$TMP/t1.json" "$T_PROMPT"
code="$(post "$PORT" "$TMP/t1.json" "$TMP/r_t1.json")"
check "torture leg1: HTTP 200" test "$code" = "200"
check "torture leg1: arguments decode to the exact nasty payload" \
  mk torture "$TMP/r_t1.json" scenarios-openai.json
mk chain "$TMP/t2.json" "$T_PROMPT" ok "$TMP/r_t1.json"
code="$(post "$PORT" "$TMP/t2.json" "$TMP/r_t2.json")"
check "torture leg2: faithful echo -> 200 final" test "$code" = "200"
mk chain "$TMP/t2_dbl.json" "$T_PROMPT" double-encode "$TMP/r_t1.json"
code="$(post "$PORT" "$TMP/t2_dbl.json" "$TMP/r_t2_dbl.json")"
check "torture negative: double-encoded echo -> 400" test "$code" = "400"
check "torture negative: reason names the two-escaper trap" mk chk "$TMP/r_t2_dbl.json" \
  'r["error"]["code"]=="gate_echo_mismatch" and "two-escaper" in r["error"]["message"]'

# 7. parallel double-call
P_PROMPT="oa-gate parallel probe: run the two-write parallel case."
mk plain "$TMP/p1.json" "$P_PROMPT"
code="$(post "$PORT" "$TMP/p1.json" "$TMP/r_p1.json")"
check "parallel leg1: TWO tool_calls, distinct ids" mk chk "$TMP/r_p1.json" \
  'r["choices"][0]["finish_reason"]=="tool_calls" and len(r["choices"][0]["message"]["tool_calls"])==2 and r["choices"][0]["message"]["tool_calls"][0]["id"]!=r["choices"][0]["message"]["tool_calls"][1]["id"]'
mk chain "$TMP/p2.json" "$P_PROMPT" ok "$TMP/r_p1.json"
code="$(post "$PORT" "$TMP/p2.json" "$TMP/r_p2.json")"
check "parallel leg2: both results -> 200 final" test "$code" = "200"
mk chain "$TMP/p2_one.json" "$P_PROMPT" only-first "$TMP/r_p1.json"
code="$(post "$PORT" "$TMP/p2_one.json" "$TMP/r_p2_one.json")"
check "parallel negative: answering only one call -> 400 pairing" test "$code" = "400"

# 8. two-round mission (loop continuation + step indexing)
M_PROMPT="oa-gate mission probe: run the two-round mission."
mk plain "$TMP/m1.json" "$M_PROMPT"
code="$(post "$PORT" "$TMP/m1.json" "$TMP/r_m1.json")"
check "mission leg1: part-1 tool call" mk chk "$TMP/r_m1.json" \
  'json.loads(r["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"])["path"]=="mission-part-1.md"'
mk chain "$TMP/m2.json" "$M_PROMPT" ok "$TMP/r_m1.json"
code="$(post "$PORT" "$TMP/m2.json" "$TMP/r_m2.json")"
check "mission leg2: part-2 tool call (step indexed by assistant count)" mk chk "$TMP/r_m2.json" \
  'r["choices"][0]["finish_reason"]=="tool_calls" and json.loads(r["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"])["path"]=="mission-part-2.md"'
mk chain "$TMP/m3.json" "$M_PROMPT" ok "$TMP/r_m1.json" "$TMP/r_m2.json"
code="$(post "$PORT" "$TMP/m3.json" "$TMP/r_m3.json")"
check "mission leg3: final text, finish stop" mk chk "$TMP/r_m3.json" \
  'r["choices"][0]["finish_reason"]=="stop" and "Mission complete" in r["choices"][0]["message"]["content"]'
mk chain "$TMP/m4.json" "$M_PROMPT" ok "$TMP/r_m1.json" "$TMP/r_m2.json" "$TMP/r_m3.json"
code="$(post "$PORT" "$TMP/m4.json" "$TMP/r_m4.json")"
check "mission overrun: past-script request -> GATE-SCRIPT-EXHAUSTED" mk chk "$TMP/r_m4.json" \
  'r["choices"][0]["message"]["content"].startswith("GATE-SCRIPT-EXHAUSTED")'

# 9. injected API errors (OpenAI error envelope)
for want in 400 429 500 503; do
  case "$want" in
    400) marker="four hundred";; 429) marker="rate limit";;
    500) marker="five hundred";; 503) marker="unavailable";;
  esac
  mk plain "$TMP/e_$want.json" "oa-gate error $marker: trigger the injected failure."
  code="$(post "$PORT" "$TMP/e_$want.json" "$TMP/r_e_$want.json")"
  check "api-error $want: status returned" test "$code" = "$want"
  check "api-error $want: OpenAI error envelope" mk chk "$TMP/r_e_$want.json" \
    'isinstance(r["error"]["message"],str) and "gate-injected" in r["error"]["message"] and isinstance(r["error"]["type"],str)'
done

# 10. background (unmatched) request
mk plain "$TMP/bg.json" "hello there, just a boot probe with no marker"
code="$(post "$PORT" "$TMP/bg.json" "$TMP/r_bg.json")"
check "background: unmatched prompt -> benign ok" mk chk "$TMP/r_bg.json" \
  'r["choices"][0]["message"]["content"]=="ok"'

# 11. ground-truth log invariants
check "ground-truth JSONL log invariants" mk logcheck "$TMP/req.jsonl"

# 12. production-port refusal
rc=0
"$PY" stub-openai.py --port 7770 --scenarios scenarios-openai.json \
     --log "$TMP/never.jsonl" >/dev/null 2>&1 || rc=$?
check "refuses production port 7770" test "$rc" -ne 0

# ---- hostile mode: black-hole ----------------------------------------------
BH="$(freeport)"
"$PY" stub-openai.py --port "$BH" --log "$TMP/bh.jsonl" --mode black-hole \
     >/dev/null 2>&1 &
PIDS+=($!); disown
check "black-hole: healthy" waithealth "$BH"
rc=0
curl -s -o /dev/null --max-time 3 -H 'content-type: application/json' \
     --data-binary @"$TMP/plain.json" \
     "http://127.0.0.1:$BH/v1/chat/completions" || rc=$?
check "black-hole: client times out (curl rc 28)" test "$rc" -eq 28
check "black-hole: health still answers during the hang" \
  curl -sf --max-time 2 "http://127.0.0.1:$BH/gate/health"

# ---- hostile mode: mid-body-drop -------------------------------------------
MD="$(freeport)"
"$PY" stub-openai.py --port "$MD" --log "$TMP/md.jsonl" --mode mid-body-drop \
     >/dev/null 2>&1 &
PIDS+=($!); disown
check "mid-body-drop: healthy" waithealth "$MD"
rc=0
curl -s --max-time 5 -o "$TMP/half.json" -H 'content-type: application/json' \
     --data-binary @"$TMP/plain.json" \
     "http://127.0.0.1:$MD/v1/chat/completions" || rc=$?
check "mid-body-drop: transfer fails (curl rc $rc)" test "$rc" -ne 0
check "mid-body-drop: partial body is not parseable JSON" mk notjson "$TMP/half.json"

# ---- hostile mode: tool-pending-forever ------------------------------------
TP="$(freeport)"
"$PY" stub-openai.py --port "$TP" --log "$TMP/tp.jsonl" \
     --mode tool-pending-forever >/dev/null 2>&1 &
PIDS+=($!); disown
check "tool-pending-forever: healthy" waithealth "$TP"
for i in 1 2 3; do
  code="$(post "$TP" "$TMP/plain.json" "$TMP/r_tp$i.json")"
  check "tool-pending-forever: request $i -> 200" test "$code" = "200"
done
check "tool-pending-forever: three FRESH tool_calls, distinct ids" \
  mk pending "$TMP/r_tp1.json" "$TMP/r_tp2.json" "$TMP/r_tp3.json"
check "tool-pending-forever: /gate/stats counts 3 chat hits" \
  bash -c "curl -sf http://127.0.0.1:$TP/gate/stats | grep -q '\"chat_hits\": 3'"

# ---- summary ----------------------------------------------------------------
echo
echo "selftest: $PASS passed, $FAIL failed"
if [ "$FAIL" -ne 0 ]; then
  echo "SELFTEST RED"
  exit 1
fi
echo "SELFTEST GREEN (stub-openai gate scaffolding verified)"