#!/usr/bin/env bash # neuron-self-improve.sh — self-improvement loop for Neuron # # Manages code experiments across alpha/beta/gamma zones and promotes winners to prod. # All cluster changes go through git → Argo CD. No direct kubectl mutations. # # Usage: # ./neuron-self-improve.sh snapshot # copy prod DB to slot # ./neuron-self-improve.sh deploy # point slot at a specific image # ./neuron-self-improve.sh promote # copy slot's image to prod # ./neuron-self-improve.sh status # show what's running in each zone # ./neuron-self-improve.sh rollback # revert slot to a previous image set -euo pipefail INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" REGISTRY="registry.neuralplatform.ai/neural-platform/neuron" KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}" # ── helpers ─────────────────────────────────────────────────────────────────── slot_manifest() { local slot=$1 if [[ "$slot" == "prod" ]]; then echo "$INFRA_DIR/servers/legion/apps/neuron.yaml" else echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml" fi } current_sha() { local slot=$1 grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1 } set_image() { local slot=$1 sha=$2 local manifest manifest="$(slot_manifest "$slot")" sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest" echo " set $slot → $sha" } git_push() { local msg=$1 cd "$INFRA_DIR" git add -A git commit -m "$msg" git push echo " pushed: $msg" } wait_healthy() { local slot=$1 local app="neuron" [[ "$slot" != "prod" ]] && app="neuron-${slot}" echo " waiting for $slot to be healthy..." local i=0 while [[ $i -lt 30 ]]; do local ready ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ --no-headers 2>/dev/null | grep "1/1" | wc -l) [[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0 sleep 10 ((i++)) done echo " WARNING: $slot did not become healthy within 5 minutes" return 1 } # ── commands ────────────────────────────────────────────────────────────────── cmd_snapshot() { local slot=${1:?Usage: snapshot } echo "==> Snapshotting prod DB into $slot..." KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF apiVersion: batch/v1 kind: Job metadata: name: snapshot-prod-to-${slot} namespace: neuron spec: ttlSecondsAfterFinished: 300 template: spec: restartPolicy: Never containers: - name: snapshot image: keinos/sqlite3:latest command: ["/bin/sh", "-c"] args: - sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db volumeMounts: - name: src mountPath: /src readOnly: true - name: dst mountPath: /dst volumes: - name: src persistentVolumeClaim: claimName: neuron-data - name: dst persistentVolumeClaim: claimName: neuron-${slot}-data EOF echo " waiting for snapshot job..." KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \ -n neuron --for=condition=complete --timeout=120s echo " snapshot complete" } cmd_deploy() { local slot=${1:?Usage: deploy } local sha=${2:?Usage: deploy } echo "==> Deploying $sha to $slot..." set_image "$slot" "$sha" git_push "chore(neuron): deploy ${sha:0:12} to $slot" wait_healthy "$slot" echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai" } cmd_promote() { local slot=${1:?Usage: promote } local sha sha=$(current_sha "$slot") [[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1 echo "==> Promoting $slot ($sha) to prod..." set_image "prod" "$sha" # Also stamp the :stable registry tag (pull and re-tag) git_push "chore(neuron): promote $slot (${sha:0:12}) to prod" wait_healthy "prod" echo "==> prod is now running $sha" echo "==> prod endpoint: https://neuron.neuralplatform.ai" } cmd_rollback() { local slot=${1:?Usage: rollback } local sha=${2:?Usage: rollback } echo "==> Rolling $slot back to $sha..." set_image "$slot" "$sha" git_push "chore(neuron): rollback $slot to ${sha:0:12}" wait_healthy "$slot" echo "==> $slot rolled back" } cmd_status() { echo "==> Neuron slot status" echo "" for slot in prod alpha beta gamma; do local app="neuron" [[ "$slot" != "prod" ]] && app="neuron-${slot}" local sha sha=$(current_sha "$slot" 2>/dev/null || echo "unknown") local ready ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ') printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready" done echo "" echo " Endpoints:" echo " prod: https://neuron.neuralplatform.ai" echo " alpha: https://neuron-alpha.neuralplatform.ai" echo " beta: https://neuron-beta.neuralplatform.ai" echo " gamma: https://neuron-gamma.neuralplatform.ai" } # ── self-improvement loop (autonomous) ──────────────────────────────────────── # When called with no args, prints the process for an autonomous agent to follow. cmd_loop_process() { cat << 'PROCESS' NEURON SELF-IMPROVEMENT LOOP ============================= Each iteration: 1. IDENTIFY — find an improvement opportunity in the codebase. Sources: Neuron backlog, observed failures, performance gaps, code review. 2. IMPLEMENT — make the change on a git branch in neural-platform/neuron. Commit message should describe the hypothesis being tested. 3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha. Wait for CI to complete and alpha pod to be healthy. 4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB: ./scripts/neuron-self-improve.sh snapshot This ensures the experiment runs against real data without touching prod. 5. EXPERIMENT — run the experiment against the alpha endpoint: https://neuron-alpha.neuralplatform.ai Use real workloads. Compare against prod or other slots. Multiple experiments can run simultaneously across alpha/beta/gamma. 6. EVALUATE — compare results. Criteria: - Does it behave correctly? - Does it perform better (latency, quality, fewer errors)? - Does it handle edge cases prod handles? Schema changes are acceptable — they become migrations against prod on promote. 7. PROMOTE WINNER — if alpha wins: ./scripts/neuron-self-improve.sh promote alpha This updates prod's image SHA in git → Argo CD deploys → prod is updated. The old prod SHA is still in the registry — rollback is always available. 8. ROLLBACK — if something goes wrong: ./scripts/neuron-self-improve.sh rollback prod 9. LOOP — go to step 1. Alpha is now free for the next experiment. RULES ------ - Never skip the snapshot step before running an experiment against a slot. - Prod data is never lost — slots only get copies. - Schema changes in experiment code → Alembic auto-migrates the snapshot DB. On promote, the migration runs against prod. Always verify migrations are additive/non-destructive before promoting. - All three experiment zones can run simultaneously with different hypotheses. - The registry retains every SHA — any version can be deployed to any slot. - This script handles git push; Argo CD handles k8s — never kubectl apply. PROCESS } # ── dispatch ────────────────────────────────────────────────────────────────── case "${1:-loop}" in snapshot) cmd_snapshot "${@:2}" ;; deploy) cmd_deploy "${@:2}" ;; promote) cmd_promote "${@:2}" ;; rollback) cmd_rollback "${@:2}" ;; status) cmd_status ;; loop) cmd_loop_process ;; *) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;; esac