#!/usr/bin/env bash
# run_comparison.sh — the whole harness, end to end, from two git refs.
#
# Builds a soul from each ref, boots each on its own throwaway port with its own
# throwaway HOME and its own disposable copy of the corpus, runs the gold set N
# times per ref, and prints the comparison with its noise threshold.
#
# SAFETY: never touches ~/.neuron, /Applications/Neuron*, ~/neuron-dev-stack, or
# any running service. Sources are exported with `git archive` into a scratch
# dir, so no worktree or branch state is mutated either. Ports are checked
# against the live set before anything boots. Every soul this script starts is
# killed and confirmed dead by run_eval.py; the sweep at the end is a backstop.
#
# usage:
#   run_comparison.sh [--baseline main] [--candidate feat/recall-through-activation]
#                     [--repeats 3] [--corpus <snapshot.json>] [--repo <path>]
set -euo pipefail

BASELINE="main"
CANDIDATE="feat/recall-through-activation"
REPEATS=3
CORPUS="$HOME/neuron-memory-backups/snapshot-pre-repair-20260806.json"
REPO="$HOME/Development/neuron"
BASE_PORT=7893

while [ $# -gt 0 ]; do
  case "$1" in
    --baseline)  BASELINE="$2"; shift 2 ;;
    --candidate) CANDIDATE="$2"; shift 2 ;;
    --repeats)   REPEATS="$2"; shift 2 ;;
    --corpus)    CORPUS="$2"; shift 2 ;;
    --repo)      REPO="$2"; shift 2 ;;
    *) echo "unknown arg: $1" >&2; exit 2 ;;
  esac
done

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK="$(mktemp -d "${TMPDIR:-/tmp}/retrieval-eval.XXXXXX")"
trap 'rm -rf "$WORK"' EXIT

[ -f "$CORPUS" ] || { echo "no corpus at $CORPUS" >&2; exit 2; }
echo "corpus: $CORPUS ($(du -h "$CORPUS" | cut -f1))"

slug() { printf '%s' "$1" | tr '/' '-'; }

build_ref() {   # ref -> binary path
  local ref="$1" out="$WORK/soul-$(slug "$1")"
  local src="$WORK/src-$(slug "$1")"
  mkdir -p "$src"
  git -C "$REPO" archive "$ref" | tar -x -C "$src"
  "$HERE/build-soul.sh" "$src" "$out" >&2
  printf '%s' "$out"
}

echo "== building $BASELINE =="
BIN_A="$(build_ref "$BASELINE")"
echo "== building $CANDIDATE =="
BIN_B="$(build_ref "$CANDIDATE")"

echo "== validating the gold set against this corpus =="
python3 "$HERE/build_gold_set.py" "$CORPUS" --check

port=$BASE_PORT
run_one() { # binary label out
  echo "== $2 =="
  python3 "$HERE/run_eval.py" --soul "$1" --corpus "$CORPUS" --label "$2" \
      --port "$port" --out "$3"
  port=$((port + 1))
}

A_MAIN="$WORK/results-a-1.json"; B_MAIN="$WORK/results-b-1.json"
A_REP=(); B_REP=()
for i in $(seq 1 "$REPEATS"); do
  a="$WORK/results-a-$i.json"; b="$WORK/results-b-$i.json"
  run_one "$BIN_A" "$(slug "$BASELINE")-r$i" "$a"
  run_one "$BIN_B" "$(slug "$CANDIDATE")-r$i" "$b"
  [ "$i" -gt 1 ] && { A_REP+=("$a"); B_REP+=("$b"); }
done

cp "$A_MAIN" "$HERE/results-$(slug "$BASELINE").json"
cp "$B_MAIN" "$HERE/results-$(slug "$CANDIDATE").json"

echo
python3 "$HERE/compare.py" \
  --baseline "$A_MAIN" --candidate "$B_MAIN" \
  ${A_REP[@]+--repeats-baseline "${A_REP[@]}"} \
  ${B_REP[@]+--repeats-candidate "${B_REP[@]}"} \
  --out "$HERE/comparison-$(slug "$BASELINE")-vs-$(slug "$CANDIDATE").json"

# Backstop: run_eval.py kills and confirms its own child, but a crashed run
# could leak one. Leaving a soul running is how the live engine got squeezed.
STRAY=$(pgrep -f "$WORK/soul-" || true)
if [ -n "$STRAY" ]; then
  echo "!! stray eval souls, killing: $STRAY" >&2
  kill -9 $STRAY 2>/dev/null || true
fi
pgrep -f "$WORK/soul-" >/dev/null && { echo "!! STILL RUNNING" >&2; exit 5; }
echo "process check: no eval souls running"