simplify neuron self-improve loop to blue/green + stage
Replace the aspirational alpha/beta/gamma model with the actual deployment topology: prod runs blue/green in neuron-prod namespace, stage is the single experiment slot in neuron-stage namespace. The old script referenced neuron-alpha/beta/gamma deployments that never existed. The new script uses blue-green-deploy.sh for prod promotion and kubectl set image for stage experiments. Loop: snapshot → deploy stage → evaluate → promote via blue/green.
This commit is contained in:
Will Anderson
@@ -1,46 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# neuron-self-improve.sh — self-improvement loop for Neuron
|
||||
#
|
||||
# Manages code experiments across alpha/beta/gamma zones and promotes winners to prod.
|
||||
# All cluster changes go through git → Argo CD. No direct kubectl mutations.
|
||||
# Model: stage is the experiment slot. prod runs blue/green.
|
||||
# Promote flow: implement → CI → stage → verify → blue-green-deploy → prod.
|
||||
#
|
||||
# Usage:
|
||||
# ./neuron-self-improve.sh snapshot <slot> # copy prod DB to slot
|
||||
# ./neuron-self-improve.sh deploy <slot> <sha> # point slot at a specific image
|
||||
# ./neuron-self-improve.sh promote <slot> # copy slot's image to prod
|
||||
# ./neuron-self-improve.sh status # show what's running in each zone
|
||||
# ./neuron-self-improve.sh rollback <slot> <sha> # revert slot to a previous image
|
||||
# ./neuron-self-improve.sh snapshot # copy prod DB to stage
|
||||
# ./neuron-self-improve.sh deploy <sha> # point stage at a specific image
|
||||
# ./neuron-self-improve.sh promote <tag> # blue/green flip prod to <tag>
|
||||
# ./neuron-self-improve.sh status # show active blue/green slot + stage
|
||||
# ./neuron-self-improve.sh rollback <tag> # roll prod back to <tag> via blue/green
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REGISTRY="registry.neuralplatform.ai/neural-platform/neuron"
|
||||
REGISTRY="registry.neuralplatform.ai/neuron-technologies/neuron-mcp"
|
||||
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
||||
|
||||
KC_PROD="kubectl --kubeconfig=$KUBECONFIG -n neuron-prod"
|
||||
KC_STAGE="kubectl --kubeconfig=$KUBECONFIG -n neuron-stage"
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
slot_manifest() {
|
||||
local slot=$1
|
||||
if [[ "$slot" == "prod" ]]; then
|
||||
echo "$INFRA_DIR/servers/legion/apps/neuron.yaml"
|
||||
else
|
||||
echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml"
|
||||
fi
|
||||
}
|
||||
|
||||
current_sha() {
|
||||
local slot=$1
|
||||
grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1
|
||||
}
|
||||
|
||||
set_image() {
|
||||
local slot=$1 sha=$2
|
||||
local manifest
|
||||
manifest="$(slot_manifest "$slot")"
|
||||
sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest"
|
||||
echo " set $slot → $sha"
|
||||
}
|
||||
|
||||
git_push() {
|
||||
local msg=$1
|
||||
cd "$INFRA_DIR"
|
||||
@@ -50,35 +31,42 @@ git_push() {
|
||||
echo " pushed: $msg"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local slot=$1
|
||||
local app="neuron"
|
||||
[[ "$slot" != "prod" ]] && app="neuron-${slot}"
|
||||
echo " waiting for $slot to be healthy..."
|
||||
local i=0
|
||||
while [[ $i -lt 30 ]]; do
|
||||
local ready
|
||||
ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \
|
||||
--no-headers 2>/dev/null | grep "1/1" | wc -l)
|
||||
[[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0
|
||||
sleep 10
|
||||
((i++))
|
||||
done
|
||||
echo " WARNING: $slot did not become healthy within 5 minutes"
|
||||
return 1
|
||||
active_prod_slot() {
|
||||
$KC_PROD get svc neuron-mcp -o jsonpath='{.spec.selector.slot}' 2>/dev/null || echo "unknown"
|
||||
}
|
||||
|
||||
current_prod_tag() {
|
||||
local slot
|
||||
slot=$(active_prod_slot)
|
||||
$KC_PROD get deployment "neuron-mcp-$slot" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \
|
||||
| cut -d: -f2 || echo "unknown"
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
local ns=$1 label=$2 timeout=${3:-180}
|
||||
echo " waiting for $label to be ready..."
|
||||
kubectl --kubeconfig="$KUBECONFIG" -n "$ns" \
|
||||
rollout status deployment "$label" --timeout="${timeout}s"
|
||||
echo " $label is ready"
|
||||
}
|
||||
|
||||
# ── commands ──────────────────────────────────────────────────────────────────
|
||||
|
||||
cmd_snapshot() {
|
||||
local slot=${1:?Usage: snapshot <slot>}
|
||||
echo "==> Snapshotting prod DB into $slot..."
|
||||
KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF
|
||||
echo "==> Snapshotting prod DB into stage..."
|
||||
local prod_slot
|
||||
prod_slot=$(active_prod_slot)
|
||||
local prod_pvc="neuron-prod-data"
|
||||
local stage_pvc="neuron-stage-data"
|
||||
|
||||
# Run a one-shot job that copies prod DB to stage PVC
|
||||
$KC_PROD apply -f - << EOF
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: snapshot-prod-to-${slot}
|
||||
namespace: neuron
|
||||
name: snapshot-prod-to-stage
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 300
|
||||
template:
|
||||
@@ -89,7 +77,9 @@ spec:
|
||||
image: keinos/sqlite3:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db
|
||||
- |
|
||||
sqlite3 /src/neuron.db ".backup /dst/neuron.db"
|
||||
echo "Done — $(ls -lh /dst/neuron.db | awk '{print \$5}')"
|
||||
volumeMounts:
|
||||
- name: src
|
||||
mountPath: /src
|
||||
@@ -99,125 +89,127 @@ spec:
|
||||
volumes:
|
||||
- name: src
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-data
|
||||
claimName: ${prod_pvc}
|
||||
- name: dst
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-${slot}-data
|
||||
claimName: ${stage_pvc}
|
||||
EOF
|
||||
|
||||
echo " waiting for snapshot job..."
|
||||
KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \
|
||||
-n neuron --for=condition=complete --timeout=120s
|
||||
echo " snapshot complete"
|
||||
$KC_PROD wait job snapshot-prod-to-stage --for=condition=complete --timeout=120s
|
||||
echo " snapshot complete — stage DB is now a copy of prod"
|
||||
}
|
||||
|
||||
cmd_deploy() {
|
||||
local slot=${1:?Usage: deploy <slot> <sha>}
|
||||
local sha=${2:?Usage: deploy <slot> <sha>}
|
||||
echo "==> Deploying $sha to $slot..."
|
||||
set_image "$slot" "$sha"
|
||||
git_push "chore(neuron): deploy ${sha:0:12} to $slot"
|
||||
wait_healthy "$slot"
|
||||
echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai"
|
||||
local sha=${1:?Usage: deploy <sha>}
|
||||
echo "==> Deploying $sha to stage..."
|
||||
|
||||
# Update stage deployment image in-place (stage is a single deployment, not blue/green)
|
||||
$KC_STAGE set image deployment/neuron-mcp "neuron-mcp=${REGISTRY}:${sha}"
|
||||
$KC_STAGE rollout restart deployment/neuron-mcp
|
||||
wait_ready "neuron-stage" "neuron-mcp"
|
||||
|
||||
echo "==> stage is live"
|
||||
echo " endpoint: https://neuron.neuralplatform.ai/stage (or stage ingress if configured)"
|
||||
echo " MCP URL: (see stage ingress/services.yaml)"
|
||||
}
|
||||
|
||||
cmd_promote() {
|
||||
local slot=${1:?Usage: promote <slot>}
|
||||
local sha
|
||||
sha=$(current_sha "$slot")
|
||||
[[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1
|
||||
echo "==> Promoting $slot ($sha) to prod..."
|
||||
set_image "prod" "$sha"
|
||||
# Also stamp the :stable registry tag (pull and re-tag)
|
||||
git_push "chore(neuron): promote $slot (${sha:0:12}) to prod"
|
||||
wait_healthy "prod"
|
||||
echo "==> prod is now running $sha"
|
||||
echo "==> prod endpoint: https://neuron.neuralplatform.ai"
|
||||
local tag=${1:?Usage: promote <tag>}
|
||||
echo "==> Promoting $tag to prod via blue/green flip..."
|
||||
"$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag"
|
||||
}
|
||||
|
||||
cmd_rollback() {
|
||||
local slot=${1:?Usage: rollback <slot> <sha>}
|
||||
local sha=${2:?Usage: rollback <slot> <sha>}
|
||||
echo "==> Rolling $slot back to $sha..."
|
||||
set_image "$slot" "$sha"
|
||||
git_push "chore(neuron): rollback $slot to ${sha:0:12}"
|
||||
wait_healthy "$slot"
|
||||
echo "==> $slot rolled back"
|
||||
local tag=${1:?Usage: rollback <tag>}
|
||||
echo "==> Rolling prod back to $tag via blue/green..."
|
||||
"$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag"
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
echo "==> Neuron slot status"
|
||||
echo "==> Neuron deployment status"
|
||||
echo ""
|
||||
for slot in prod alpha beta gamma; do
|
||||
local app="neuron"
|
||||
[[ "$slot" != "prod" ]] && app="neuron-${slot}"
|
||||
local sha
|
||||
sha=$(current_sha "$slot" 2>/dev/null || echo "unknown")
|
||||
local ready
|
||||
ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \
|
||||
--no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ')
|
||||
printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready"
|
||||
done
|
||||
|
||||
local active_slot
|
||||
active_slot=$(active_prod_slot)
|
||||
local prod_tag
|
||||
prod_tag=$(current_prod_tag)
|
||||
|
||||
printf " %-8s slot=%-5s tag=%s\n" "prod" "$active_slot" "$prod_tag"
|
||||
|
||||
# Stage
|
||||
local stage_tag
|
||||
stage_tag=$($KC_STAGE get deployment neuron-mcp \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \
|
||||
| cut -d: -f2 || echo "unknown")
|
||||
local stage_ready
|
||||
stage_ready=$($KC_STAGE get pods -l app=neuron-mcp \
|
||||
--no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ')
|
||||
printf " %-8s tag=%-20s pods_ready=%s\n" "stage" "$stage_tag" "$stage_ready"
|
||||
|
||||
echo ""
|
||||
echo " Endpoints:"
|
||||
echo " prod: https://neuron.neuralplatform.ai"
|
||||
echo " alpha: https://neuron-alpha.neuralplatform.ai"
|
||||
echo " beta: https://neuron-beta.neuralplatform.ai"
|
||||
echo " gamma: https://neuron-gamma.neuralplatform.ai"
|
||||
echo " Prod endpoint: https://neuron.neuralplatform.ai"
|
||||
echo " Prod MCP: https://neuron.neuralplatform.ai/mcp"
|
||||
}
|
||||
|
||||
# ── self-improvement loop (autonomous) ────────────────────────────────────────
|
||||
# When called with no args, prints the process for an autonomous agent to follow.
|
||||
# When called with no args (or 'loop'), prints the process for an autonomous agent.
|
||||
cmd_loop_process() {
|
||||
cat << 'PROCESS'
|
||||
NEURON SELF-IMPROVEMENT LOOP
|
||||
=============================
|
||||
|
||||
Deployment model: stage (experiment) → prod (blue/green).
|
||||
|
||||
Each iteration:
|
||||
|
||||
1. IDENTIFY — find an improvement opportunity in the codebase.
|
||||
Sources: Neuron backlog, observed failures, performance gaps, code review.
|
||||
1. IDENTIFY — find an improvement opportunity.
|
||||
Sources: Neuron backlog, observed failures, graph reliability gaps,
|
||||
session startup issues, performance, quality observations.
|
||||
|
||||
2. IMPLEMENT — make the change on a git branch in neural-platform/neuron.
|
||||
Commit message should describe the hypothesis being tested.
|
||||
2. IMPLEMENT — make the change on a git branch in neuron-technologies/neuron.
|
||||
Commit message describes the hypothesis.
|
||||
|
||||
3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha.
|
||||
Wait for CI to complete and alpha pod to be healthy.
|
||||
3. CI — push to main; CI builds image and publishes to registry.
|
||||
Image tag = git SHA (short). Wait for CI to complete.
|
||||
|
||||
4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB:
|
||||
./scripts/neuron-self-improve.sh snapshot <slot>
|
||||
This ensures the experiment runs against real data without touching prod.
|
||||
4. SNAPSHOT — before running the experiment, copy prod DB to stage:
|
||||
./scripts/neuron-self-improve.sh snapshot
|
||||
This gives stage a real-data copy without touching prod.
|
||||
|
||||
5. EXPERIMENT — run the experiment against the alpha endpoint:
|
||||
https://neuron-alpha.neuralplatform.ai
|
||||
Use real workloads. Compare against prod or other slots.
|
||||
Multiple experiments can run simultaneously across alpha/beta/gamma.
|
||||
5. DEPLOY TO STAGE — point stage at the new image:
|
||||
./scripts/neuron-self-improve.sh deploy <sha>
|
||||
Stage runs against the snapshot DB. Experiment freely.
|
||||
|
||||
6. EVALUATE — compare results. Criteria:
|
||||
- Does it behave correctly?
|
||||
- Does it perform better (latency, quality, fewer errors)?
|
||||
- Does it handle edge cases prod handles?
|
||||
Schema changes are acceptable — they become migrations against prod on promote.
|
||||
6. EVALUATE — test the stage endpoint. Criteria:
|
||||
- Correct behavior on same inputs?
|
||||
- At least one dimension improved (latency, quality, error rate)?
|
||||
- No regressions on cases prod handles?
|
||||
- Schema changes are additive/non-destructive?
|
||||
|
||||
7. PROMOTE WINNER — if alpha wins:
|
||||
./scripts/neuron-self-improve.sh promote alpha
|
||||
This updates prod's image SHA in git → Argo CD deploys → prod is updated.
|
||||
The old prod SHA is still in the registry — rollback is always available.
|
||||
7. PROMOTE — if stage passes, promote to prod via blue/green:
|
||||
./scripts/neuron-self-improve.sh promote <tag>
|
||||
The blue-green-deploy.sh handles: scale idle slot → health check →
|
||||
flip service selector → scale old slot to 0.
|
||||
The old slot is kept at 0 for instant rollback.
|
||||
|
||||
8. ROLLBACK — if something goes wrong:
|
||||
./scripts/neuron-self-improve.sh rollback prod <previous-sha>
|
||||
8. ROLLBACK — if something goes wrong post-promote:
|
||||
./scripts/neuron-self-improve.sh rollback <previous-tag>
|
||||
Blue/green means rollback is always a slot flip — no redeployment.
|
||||
|
||||
9. LOOP — go to step 1. Alpha is now free for the next experiment.
|
||||
9. LOOP — go to step 1. Stage is free for the next experiment.
|
||||
|
||||
RULES
|
||||
------
|
||||
- Never skip the snapshot step before running an experiment against a slot.
|
||||
- Prod data is never lost — slots only get copies.
|
||||
- Schema changes in experiment code → Alembic auto-migrates the snapshot DB.
|
||||
On promote, the migration runs against prod. Always verify migrations are
|
||||
additive/non-destructive before promoting.
|
||||
- All three experiment zones can run simultaneously with different hypotheses.
|
||||
- The registry retains every SHA — any version can be deployed to any slot.
|
||||
- This script handles git push; Argo CD handles k8s — never kubectl apply.
|
||||
- Always snapshot before deploying to stage (stage DB starts from prod).
|
||||
- Prod is never touched during the experiment — only on promote.
|
||||
- Schema changes: test Alembic migration on stage snapshot first.
|
||||
On promote, the same migration runs against prod. Must be additive.
|
||||
- The registry retains every SHA — any version is deployable to any slot.
|
||||
- blue-green-deploy.sh handles the infra flip. This script orchestrates.
|
||||
- Never kubectl apply directly — all changes via git + Argo CD.
|
||||
Exception: blue-green-deploy.sh uses kubectl for the slot flip because
|
||||
that is a runtime operation, not a config change.
|
||||
|
||||
PROCESS
|
||||
}
|
||||
@@ -225,11 +217,11 @@ PROCESS
|
||||
# ── dispatch ──────────────────────────────────────────────────────────────────
|
||||
|
||||
case "${1:-loop}" in
|
||||
snapshot) cmd_snapshot "${@:2}" ;;
|
||||
snapshot) cmd_snapshot ;;
|
||||
deploy) cmd_deploy "${@:2}" ;;
|
||||
promote) cmd_promote "${@:2}" ;;
|
||||
rollback) cmd_rollback "${@:2}" ;;
|
||||
status) cmd_status ;;
|
||||
loop) cmd_loop_process ;;
|
||||
*) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;;
|
||||
*) echo "Usage: $0 snapshot|deploy <sha>|promote <tag>|rollback <tag>|status|loop" && exit 1 ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user