From 8fd3d12907c7d05d59caf16ae7c469982248e80b Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Sat, 25 Apr 2026 20:33:57 -0500 Subject: [PATCH] simplify neuron self-improve loop to blue/green + stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the aspirational alpha/beta/gamma model with the actual deployment topology: prod runs blue/green in neuron-prod namespace, stage is the single experiment slot in neuron-stage namespace. The old script referenced neuron-alpha/beta/gamma deployments that never existed. The new script uses blue-green-deploy.sh for prod promotion and kubectl set image for stage experiments. Loop: snapshot → deploy stage → evaluate → promote via blue/green. --- servers/legion/scripts/neuron-self-improve.sh | 260 +++++++++--------- 1 file changed, 126 insertions(+), 134 deletions(-) diff --git a/servers/legion/scripts/neuron-self-improve.sh b/servers/legion/scripts/neuron-self-improve.sh index 1a9d16d..5055544 100755 --- a/servers/legion/scripts/neuron-self-improve.sh +++ b/servers/legion/scripts/neuron-self-improve.sh @@ -1,46 +1,27 @@ #!/usr/bin/env bash # neuron-self-improve.sh — self-improvement loop for Neuron # -# Manages code experiments across alpha/beta/gamma zones and promotes winners to prod. -# All cluster changes go through git → Argo CD. No direct kubectl mutations. +# Model: stage is the experiment slot. prod runs blue/green. +# Promote flow: implement → CI → stage → verify → blue-green-deploy → prod. # # Usage: -# ./neuron-self-improve.sh snapshot # copy prod DB to slot -# ./neuron-self-improve.sh deploy # point slot at a specific image -# ./neuron-self-improve.sh promote # copy slot's image to prod -# ./neuron-self-improve.sh status # show what's running in each zone -# ./neuron-self-improve.sh rollback # revert slot to a previous image +# ./neuron-self-improve.sh snapshot # copy prod DB to stage +# ./neuron-self-improve.sh deploy # point stage at a specific image +# ./neuron-self-improve.sh promote # blue/green flip prod to +# ./neuron-self-improve.sh status # show active blue/green slot + stage +# ./neuron-self-improve.sh rollback # roll prod back to via blue/green set -euo pipefail INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REGISTRY="registry.neuralplatform.ai/neural-platform/neuron" +REGISTRY="registry.neuralplatform.ai/neuron-technologies/neuron-mcp" KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}" +KC_PROD="kubectl --kubeconfig=$KUBECONFIG -n neuron-prod" +KC_STAGE="kubectl --kubeconfig=$KUBECONFIG -n neuron-stage" + # ── helpers ─────────────────────────────────────────────────────────────────── -slot_manifest() { - local slot=$1 - if [[ "$slot" == "prod" ]]; then - echo "$INFRA_DIR/servers/legion/apps/neuron.yaml" - else - echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml" - fi -} - -current_sha() { - local slot=$1 - grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1 -} - -set_image() { - local slot=$1 sha=$2 - local manifest - manifest="$(slot_manifest "$slot")" - sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest" - echo " set $slot → $sha" -} - git_push() { local msg=$1 cd "$INFRA_DIR" @@ -50,35 +31,42 @@ git_push() { echo " pushed: $msg" } -wait_healthy() { - local slot=$1 - local app="neuron" - [[ "$slot" != "prod" ]] && app="neuron-${slot}" - echo " waiting for $slot to be healthy..." - local i=0 - while [[ $i -lt 30 ]]; do - local ready - ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ - --no-headers 2>/dev/null | grep "1/1" | wc -l) - [[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0 - sleep 10 - ((i++)) - done - echo " WARNING: $slot did not become healthy within 5 minutes" - return 1 +active_prod_slot() { + $KC_PROD get svc neuron-mcp -o jsonpath='{.spec.selector.slot}' 2>/dev/null || echo "unknown" +} + +current_prod_tag() { + local slot + slot=$(active_prod_slot) + $KC_PROD get deployment "neuron-mcp-$slot" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \ + | cut -d: -f2 || echo "unknown" +} + +wait_ready() { + local ns=$1 label=$2 timeout=${3:-180} + echo " waiting for $label to be ready..." + kubectl --kubeconfig="$KUBECONFIG" -n "$ns" \ + rollout status deployment "$label" --timeout="${timeout}s" + echo " $label is ready" } # ── commands ────────────────────────────────────────────────────────────────── cmd_snapshot() { - local slot=${1:?Usage: snapshot } - echo "==> Snapshotting prod DB into $slot..." - KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF + echo "==> Snapshotting prod DB into stage..." + local prod_slot + prod_slot=$(active_prod_slot) + local prod_pvc="neuron-prod-data" + local stage_pvc="neuron-stage-data" + + # Run a one-shot job that copies prod DB to stage PVC + $KC_PROD apply -f - << EOF apiVersion: batch/v1 kind: Job metadata: - name: snapshot-prod-to-${slot} - namespace: neuron + name: snapshot-prod-to-stage + namespace: neuron-prod spec: ttlSecondsAfterFinished: 300 template: @@ -89,7 +77,9 @@ spec: image: keinos/sqlite3:latest command: ["/bin/sh", "-c"] args: - - sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db + - | + sqlite3 /src/neuron.db ".backup /dst/neuron.db" + echo "Done — $(ls -lh /dst/neuron.db | awk '{print \$5}')" volumeMounts: - name: src mountPath: /src @@ -99,125 +89,127 @@ spec: volumes: - name: src persistentVolumeClaim: - claimName: neuron-data + claimName: ${prod_pvc} - name: dst persistentVolumeClaim: - claimName: neuron-${slot}-data + claimName: ${stage_pvc} EOF + echo " waiting for snapshot job..." - KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \ - -n neuron --for=condition=complete --timeout=120s - echo " snapshot complete" + $KC_PROD wait job snapshot-prod-to-stage --for=condition=complete --timeout=120s + echo " snapshot complete — stage DB is now a copy of prod" } cmd_deploy() { - local slot=${1:?Usage: deploy } - local sha=${2:?Usage: deploy } - echo "==> Deploying $sha to $slot..." - set_image "$slot" "$sha" - git_push "chore(neuron): deploy ${sha:0:12} to $slot" - wait_healthy "$slot" - echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai" + local sha=${1:?Usage: deploy } + echo "==> Deploying $sha to stage..." + + # Update stage deployment image in-place (stage is a single deployment, not blue/green) + $KC_STAGE set image deployment/neuron-mcp "neuron-mcp=${REGISTRY}:${sha}" + $KC_STAGE rollout restart deployment/neuron-mcp + wait_ready "neuron-stage" "neuron-mcp" + + echo "==> stage is live" + echo " endpoint: https://neuron.neuralplatform.ai/stage (or stage ingress if configured)" + echo " MCP URL: (see stage ingress/services.yaml)" } cmd_promote() { - local slot=${1:?Usage: promote } - local sha - sha=$(current_sha "$slot") - [[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1 - echo "==> Promoting $slot ($sha) to prod..." - set_image "prod" "$sha" - # Also stamp the :stable registry tag (pull and re-tag) - git_push "chore(neuron): promote $slot (${sha:0:12}) to prod" - wait_healthy "prod" - echo "==> prod is now running $sha" - echo "==> prod endpoint: https://neuron.neuralplatform.ai" + local tag=${1:?Usage: promote } + echo "==> Promoting $tag to prod via blue/green flip..." + "$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag" } cmd_rollback() { - local slot=${1:?Usage: rollback } - local sha=${2:?Usage: rollback } - echo "==> Rolling $slot back to $sha..." - set_image "$slot" "$sha" - git_push "chore(neuron): rollback $slot to ${sha:0:12}" - wait_healthy "$slot" - echo "==> $slot rolled back" + local tag=${1:?Usage: rollback } + echo "==> Rolling prod back to $tag via blue/green..." + "$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag" } cmd_status() { - echo "==> Neuron slot status" + echo "==> Neuron deployment status" echo "" - for slot in prod alpha beta gamma; do - local app="neuron" - [[ "$slot" != "prod" ]] && app="neuron-${slot}" - local sha - sha=$(current_sha "$slot" 2>/dev/null || echo "unknown") - local ready - ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ - --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ') - printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready" - done + + local active_slot + active_slot=$(active_prod_slot) + local prod_tag + prod_tag=$(current_prod_tag) + + printf " %-8s slot=%-5s tag=%s\n" "prod" "$active_slot" "$prod_tag" + + # Stage + local stage_tag + stage_tag=$($KC_STAGE get deployment neuron-mcp \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \ + | cut -d: -f2 || echo "unknown") + local stage_ready + stage_ready=$($KC_STAGE get pods -l app=neuron-mcp \ + --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ') + printf " %-8s tag=%-20s pods_ready=%s\n" "stage" "$stage_tag" "$stage_ready" + echo "" - echo " Endpoints:" - echo " prod: https://neuron.neuralplatform.ai" - echo " alpha: https://neuron-alpha.neuralplatform.ai" - echo " beta: https://neuron-beta.neuralplatform.ai" - echo " gamma: https://neuron-gamma.neuralplatform.ai" + echo " Prod endpoint: https://neuron.neuralplatform.ai" + echo " Prod MCP: https://neuron.neuralplatform.ai/mcp" } # ── self-improvement loop (autonomous) ──────────────────────────────────────── -# When called with no args, prints the process for an autonomous agent to follow. +# When called with no args (or 'loop'), prints the process for an autonomous agent. cmd_loop_process() { cat << 'PROCESS' NEURON SELF-IMPROVEMENT LOOP ============================= +Deployment model: stage (experiment) → prod (blue/green). + Each iteration: -1. IDENTIFY — find an improvement opportunity in the codebase. - Sources: Neuron backlog, observed failures, performance gaps, code review. +1. IDENTIFY — find an improvement opportunity. + Sources: Neuron backlog, observed failures, graph reliability gaps, + session startup issues, performance, quality observations. -2. IMPLEMENT — make the change on a git branch in neural-platform/neuron. - Commit message should describe the hypothesis being tested. +2. IMPLEMENT — make the change on a git branch in neuron-technologies/neuron. + Commit message describes the hypothesis. -3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha. - Wait for CI to complete and alpha pod to be healthy. +3. CI — push to main; CI builds image and publishes to registry. + Image tag = git SHA (short). Wait for CI to complete. -4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB: - ./scripts/neuron-self-improve.sh snapshot - This ensures the experiment runs against real data without touching prod. +4. SNAPSHOT — before running the experiment, copy prod DB to stage: + ./scripts/neuron-self-improve.sh snapshot + This gives stage a real-data copy without touching prod. -5. EXPERIMENT — run the experiment against the alpha endpoint: - https://neuron-alpha.neuralplatform.ai - Use real workloads. Compare against prod or other slots. - Multiple experiments can run simultaneously across alpha/beta/gamma. +5. DEPLOY TO STAGE — point stage at the new image: + ./scripts/neuron-self-improve.sh deploy + Stage runs against the snapshot DB. Experiment freely. -6. EVALUATE — compare results. Criteria: - - Does it behave correctly? - - Does it perform better (latency, quality, fewer errors)? - - Does it handle edge cases prod handles? - Schema changes are acceptable — they become migrations against prod on promote. +6. EVALUATE — test the stage endpoint. Criteria: + - Correct behavior on same inputs? + - At least one dimension improved (latency, quality, error rate)? + - No regressions on cases prod handles? + - Schema changes are additive/non-destructive? -7. PROMOTE WINNER — if alpha wins: - ./scripts/neuron-self-improve.sh promote alpha - This updates prod's image SHA in git → Argo CD deploys → prod is updated. - The old prod SHA is still in the registry — rollback is always available. +7. PROMOTE — if stage passes, promote to prod via blue/green: + ./scripts/neuron-self-improve.sh promote + The blue-green-deploy.sh handles: scale idle slot → health check → + flip service selector → scale old slot to 0. + The old slot is kept at 0 for instant rollback. -8. ROLLBACK — if something goes wrong: - ./scripts/neuron-self-improve.sh rollback prod +8. ROLLBACK — if something goes wrong post-promote: + ./scripts/neuron-self-improve.sh rollback + Blue/green means rollback is always a slot flip — no redeployment. -9. LOOP — go to step 1. Alpha is now free for the next experiment. +9. LOOP — go to step 1. Stage is free for the next experiment. RULES ------ -- Never skip the snapshot step before running an experiment against a slot. -- Prod data is never lost — slots only get copies. -- Schema changes in experiment code → Alembic auto-migrates the snapshot DB. - On promote, the migration runs against prod. Always verify migrations are - additive/non-destructive before promoting. -- All three experiment zones can run simultaneously with different hypotheses. -- The registry retains every SHA — any version can be deployed to any slot. -- This script handles git push; Argo CD handles k8s — never kubectl apply. +- Always snapshot before deploying to stage (stage DB starts from prod). +- Prod is never touched during the experiment — only on promote. +- Schema changes: test Alembic migration on stage snapshot first. + On promote, the same migration runs against prod. Must be additive. +- The registry retains every SHA — any version is deployable to any slot. +- blue-green-deploy.sh handles the infra flip. This script orchestrates. +- Never kubectl apply directly — all changes via git + Argo CD. + Exception: blue-green-deploy.sh uses kubectl for the slot flip because + that is a runtime operation, not a config change. PROCESS } @@ -225,11 +217,11 @@ PROCESS # ── dispatch ────────────────────────────────────────────────────────────────── case "${1:-loop}" in - snapshot) cmd_snapshot "${@:2}" ;; + snapshot) cmd_snapshot ;; deploy) cmd_deploy "${@:2}" ;; promote) cmd_promote "${@:2}" ;; rollback) cmd_rollback "${@:2}" ;; status) cmd_status ;; loop) cmd_loop_process ;; - *) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;; + *) echo "Usage: $0 snapshot|deploy |promote |rollback |status|loop" && exit 1 ;; esac