infrastructure/servers/legion/scripts/neuron-self-improve.sh

#!/usr/bin/env bash
# neuron-self-improve.sh — self-improvement loop for Neuron
#
# Model: stage is the experiment slot. prod runs blue/green.
# Promote flow: implement → CI → stage → verify → blue-green-deploy → prod.
#
# Usage:
#   ./neuron-self-improve.sh snapshot              # copy prod DB to stage
#   ./neuron-self-improve.sh deploy <sha>          # point stage at a specific image
#   ./neuron-self-improve.sh promote <tag>         # blue/green flip prod to <tag>
#   ./neuron-self-improve.sh status                # show active blue/green slot + stage
#   ./neuron-self-improve.sh rollback <tag>        # roll prod back to <tag> via blue/green

set -euo pipefail

INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REGISTRY="registry.neuralplatform.ai/neuron-technologies/neuron-mcp"
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"

KC_PROD="kubectl --kubeconfig=$KUBECONFIG -n neuron-prod"
KC_STAGE="kubectl --kubeconfig=$KUBECONFIG -n neuron-stage"

# ── helpers ───────────────────────────────────────────────────────────────────

git_push() {
  local msg=$1
  cd "$INFRA_DIR"
  git add -A
  git commit -m "$msg"
  git push
  echo "  pushed: $msg"
}

active_prod_slot() {
  $KC_PROD get svc neuron-mcp -o jsonpath='{.spec.selector.slot}' 2>/dev/null || echo "unknown"
}

current_prod_tag() {
  local slot
  slot=$(active_prod_slot)
  $KC_PROD get deployment "neuron-mcp-$slot" \
    -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \
    | cut -d: -f2 || echo "unknown"
}

wait_ready() {
  local ns=$1 label=$2 timeout=${3:-180}
  echo "  waiting for $label to be ready..."
  kubectl --kubeconfig="$KUBECONFIG" -n "$ns" \
    rollout status deployment "$label" --timeout="${timeout}s"
  echo "  $label is ready"
}

# ── commands ──────────────────────────────────────────────────────────────────

cmd_snapshot() {
  echo "==> Snapshotting prod DB into stage..."
  local prod_slot
  prod_slot=$(active_prod_slot)
  local prod_pvc="neuron-prod-data"
  local stage_pvc="neuron-stage-data"

  # Run a one-shot job that copies prod DB to stage PVC
  $KC_PROD apply -f - << EOF
apiVersion: batch/v1
kind: Job
metadata:
  name: snapshot-prod-to-stage
  namespace: neuron-prod
spec:
  ttlSecondsAfterFinished: 300
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: snapshot
          image: keinos/sqlite3:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              sqlite3 /src/neuron.db ".backup /dst/neuron.db"
              echo "Done — $(ls -lh /dst/neuron.db | awk '{print \$5}')"
          volumeMounts:
            - name: src
              mountPath: /src
              readOnly: true
            - name: dst
              mountPath: /dst
      volumes:
        - name: src
          persistentVolumeClaim:
            claimName: ${prod_pvc}
        - name: dst
          persistentVolumeClaim:
            claimName: ${stage_pvc}
EOF

  echo "  waiting for snapshot job..."
  $KC_PROD wait job snapshot-prod-to-stage --for=condition=complete --timeout=120s
  echo "  snapshot complete — stage DB is now a copy of prod"
}

cmd_deploy() {
  local sha=${1:?Usage: deploy <sha>}
  echo "==> Deploying $sha to stage..."

  # Update stage deployment image in-place (stage is a single deployment, not blue/green)
  $KC_STAGE set image deployment/neuron-mcp "neuron-mcp=${REGISTRY}:${sha}"
  $KC_STAGE rollout restart deployment/neuron-mcp
  wait_ready "neuron-stage" "neuron-mcp"

  echo "==> stage is live"
  echo "  endpoint: https://neuron.neuralplatform.ai/stage  (or stage ingress if configured)"
  echo "  MCP URL: (see stage ingress/services.yaml)"
}

cmd_promote() {
  local tag=${1:?Usage: promote <tag>}
  echo "==> Promoting $tag to prod via blue/green flip..."
  "$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag"
}

cmd_rollback() {
  local tag=${1:?Usage: rollback <tag>}
  echo "==> Rolling prod back to $tag via blue/green..."
  "$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag"
}

cmd_status() {
  echo "==> Neuron deployment status"
  echo ""

  local active_slot
  active_slot=$(active_prod_slot)
  local prod_tag
  prod_tag=$(current_prod_tag)

  printf "  %-8s  slot=%-5s  tag=%s\n" "prod" "$active_slot" "$prod_tag"

  # Stage
  local stage_tag
  stage_tag=$($KC_STAGE get deployment neuron-mcp \
    -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \
    | cut -d: -f2 || echo "unknown")
  local stage_ready
  stage_ready=$($KC_STAGE get pods -l app=neuron-mcp \
    --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ')
  printf "  %-8s  tag=%-20s  pods_ready=%s\n" "stage" "$stage_tag" "$stage_ready"

  echo ""
  echo "  Prod endpoint:  https://neuron.neuralplatform.ai"
  echo "  Prod MCP:       https://neuron.neuralplatform.ai/mcp"
}

# ── self-improvement loop (autonomous) ────────────────────────────────────────
# When called with no args (or 'loop'), prints the process for an autonomous agent.
cmd_loop_process() {
  cat << 'PROCESS'
NEURON SELF-IMPROVEMENT LOOP
=============================

Deployment model: stage (experiment) → prod (blue/green).

Each iteration:

1. IDENTIFY — find an improvement opportunity.
   Sources: Neuron backlog, observed failures, graph reliability gaps,
   session startup issues, performance, quality observations.

2. IMPLEMENT — make the change on a git branch in neuron-technologies/neuron.
   Commit message describes the hypothesis.

3. CI — push to main; CI builds image and publishes to registry.
   Image tag = git SHA (short). Wait for CI to complete.

4. SNAPSHOT — before running the experiment, copy prod DB to stage:
     ./scripts/neuron-self-improve.sh snapshot
   This gives stage a real-data copy without touching prod.

5. DEPLOY TO STAGE — point stage at the new image:
     ./scripts/neuron-self-improve.sh deploy <sha>
   Stage runs against the snapshot DB. Experiment freely.

6. EVALUATE — test the stage endpoint. Criteria:
   - Correct behavior on same inputs?
   - At least one dimension improved (latency, quality, error rate)?
   - No regressions on cases prod handles?
   - Schema changes are additive/non-destructive?

7. PROMOTE — if stage passes, promote to prod via blue/green:
     ./scripts/neuron-self-improve.sh promote <tag>
   The blue-green-deploy.sh handles: scale idle slot → health check →
   flip service selector → scale old slot to 0.
   The old slot is kept at 0 for instant rollback.

8. ROLLBACK — if something goes wrong post-promote:
     ./scripts/neuron-self-improve.sh rollback <previous-tag>
   Blue/green means rollback is always a slot flip — no redeployment.

9. LOOP — go to step 1. Stage is free for the next experiment.

RULES
------
- Always snapshot before deploying to stage (stage DB starts from prod).
- Prod is never touched during the experiment — only on promote.
- Schema changes: test Alembic migration on stage snapshot first.
  On promote, the same migration runs against prod. Must be additive.
- The registry retains every SHA — any version is deployable to any slot.
- blue-green-deploy.sh handles the infra flip. This script orchestrates.
- Never kubectl apply directly — all changes via git + Argo CD.
  Exception: blue-green-deploy.sh uses kubectl for the slot flip because
  that is a runtime operation, not a config change.

PROCESS
}

# ── dispatch ──────────────────────────────────────────────────────────────────

case "${1:-loop}" in
  snapshot) cmd_snapshot ;;
  deploy)   cmd_deploy   "${@:2}" ;;
  promote)  cmd_promote  "${@:2}" ;;
  rollback) cmd_rollback "${@:2}" ;;
  status)   cmd_status ;;
  loop)     cmd_loop_process ;;
  *)        echo "Usage: $0 snapshot|deploy <sha>|promote <tag>|rollback <tag>|status|loop" && exit 1 ;;
esac