diff --git a/servers/legion/apps/neuron-gamma.yaml b/servers/legion/apps/neuron-gamma.yaml new file mode 100644 index 0000000..7524e60 --- /dev/null +++ b/servers/legion/apps/neuron-gamma.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: neuron-gamma + namespace: neuron + labels: + app: neuron-gamma +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: neuron-gamma + template: + metadata: + labels: + app: neuron-gamma + spec: + containers: + - name: neuron + image: registry.neuralplatform.ai/neural-platform/neuron:latest + imagePullPolicy: Always + command: + - bash + - -c + - mkdir -p /root/.neuron && ln -sf /data/neuron.db /root/.neuron/neuron.db && python -m synapse platform serve --mcp-host 0.0.0.0 --mcp-port 8001 --webhook-port 3847 + ports: + - name: mcp + containerPort: 8001 + - name: webhook + containerPort: 3847 + envFrom: + - configMapRef: + name: neuron-config + - secretRef: + name: neuron-secrets + volumeMounts: + - name: data + mountPath: /data + - name: app-config + mountPath: /app/config/default.yaml + subPath: default.yaml + resources: + requests: + memory: 256Mi + cpu: 100m + ephemeral-storage: 256Mi + limits: + memory: 1Gi + cpu: 500m + ephemeral-storage: 1Gi + livenessProbe: + tcpSocket: + port: mcp + initialDelaySeconds: 15 + periodSeconds: 30 + readinessProbe: + tcpSocket: + port: mcp + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: data + persistentVolumeClaim: + claimName: neuron-gamma-data + - name: app-config + configMap: + name: neuron-config + items: + - key: default.yaml + path: default.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: neuron-gamma + namespace: neuron +spec: + selector: + app: neuron-gamma + ports: + - name: mcp + port: 8001 + targetPort: 8001 + - name: webhook + port: 3847 + targetPort: 3847 + type: ClusterIP diff --git a/servers/legion/k8s/neuron/ingress.yaml b/servers/legion/k8s/neuron/ingress.yaml index aac1b56..7c416d1 100644 --- a/servers/legion/k8s/neuron/ingress.yaml +++ b/servers/legion/k8s/neuron/ingress.yaml @@ -12,6 +12,15 @@ spec: - secretName: neuron-tls hosts: - neuron.neuralplatform.ai + - secretName: neuron-alpha-tls + hosts: + - neuron-alpha.neuralplatform.ai + - secretName: neuron-beta-tls + hosts: + - neuron-beta.neuralplatform.ai + - secretName: neuron-gamma-tls + hosts: + - neuron-gamma.neuralplatform.ai rules: - host: neuron.neuralplatform.ai http: @@ -23,3 +32,33 @@ spec: name: neuron port: number: 8001 + - host: neuron-alpha.neuralplatform.ai + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: neuron-alpha + port: + number: 8001 + - host: neuron-beta.neuralplatform.ai + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: neuron-beta + port: + number: 8001 + - host: neuron-gamma.neuralplatform.ai + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: neuron-gamma + port: + number: 8001 diff --git a/servers/legion/k8s/neuron/pvc-gamma.yaml b/servers/legion/k8s/neuron/pvc-gamma.yaml new file mode 100644 index 0000000..1533c02 --- /dev/null +++ b/servers/legion/k8s/neuron/pvc-gamma.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: neuron-gamma-data + namespace: neuron +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/servers/legion/k8s/neuron/snapshot-job.yaml b/servers/legion/k8s/neuron/snapshot-job.yaml new file mode 100644 index 0000000..3c400a6 --- /dev/null +++ b/servers/legion/k8s/neuron/snapshot-job.yaml @@ -0,0 +1,44 @@ +# Snapshot Job — copies prod DB into an experiment slot via sqlite3 .backup +# Usage: set SLOT to alpha, beta, or gamma before applying +# +# SLOT=alpha && sed "s/SLOT/${SLOT}/g" snapshot-job.yaml | kubectl apply -f - +# +# The job completes and self-deletes after 5 minutes. +# Always snapshot before deploying a new experiment to a slot. +apiVersion: batch/v1 +kind: Job +metadata: + name: snapshot-prod-to-SLOT + namespace: neuron +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + restartPolicy: Never + initContainers: + - name: stop-check + image: registry.neuralplatform.ai/neural-platform/neuron:latest + command: ["/bin/sh", "-c", "echo 'Snapshotting prod DB to SLOT — ensure neuron-SLOT pod is stopped first'"] + containers: + - name: snapshot + image: keinos/sqlite3:latest + command: ["/bin/sh", "-c"] + args: + - | + echo "Starting backup from prod to SLOT..." + sqlite3 /src/neuron.db ".backup /dst/neuron.db" + echo "Done. Prod DB snapshotted to SLOT." + ls -lh /dst/neuron.db + volumeMounts: + - name: src + mountPath: /src + readOnly: true + - name: dst + mountPath: /dst + volumes: + - name: src + persistentVolumeClaim: + claimName: neuron-data + - name: dst + persistentVolumeClaim: + claimName: neuron-SLOT-data diff --git a/servers/legion/main.tf b/servers/legion/main.tf index cd83305..e02a6a7 100644 --- a/servers/legion/main.tf +++ b/servers/legion/main.tf @@ -52,7 +52,7 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "legion" { tunnel_id = var.cloudflare_tunnel_id config { - # neuron.neuralplatform.ai — MCP server (SSE transport, requires CF-Access headers) + # neuron.neuralplatform.ai — MCP server, stable prod ingress_rule { hostname = "neuron.neuralplatform.ai" service = "https://traefik.kube-system.svc:443" @@ -61,6 +61,33 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "legion" { } } + # neuron-alpha.neuralplatform.ai — experiment zone 1 (CI auto-deploys here) + ingress_rule { + hostname = "neuron-alpha.neuralplatform.ai" + service = "https://traefik.kube-system.svc:443" + origin_request { + no_tls_verify = true + } + } + + # neuron-beta.neuralplatform.ai — experiment zone 2 + ingress_rule { + hostname = "neuron-beta.neuralplatform.ai" + service = "https://traefik.kube-system.svc:443" + origin_request { + no_tls_verify = true + } + } + + # neuron-gamma.neuralplatform.ai — experiment zone 3 + ingress_rule { + hostname = "neuron-gamma.neuralplatform.ai" + service = "https://traefik.kube-system.svc:443" + origin_request { + no_tls_verify = true + } + } + ingress_rule { hostname = "vault.neuralplatform.ai" service = "https://traefik.kube-system.svc:443" diff --git a/servers/legion/scripts/neuron-self-improve.sh b/servers/legion/scripts/neuron-self-improve.sh new file mode 100755 index 0000000..1a9d16d --- /dev/null +++ b/servers/legion/scripts/neuron-self-improve.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# neuron-self-improve.sh — self-improvement loop for Neuron +# +# Manages code experiments across alpha/beta/gamma zones and promotes winners to prod. +# All cluster changes go through git → Argo CD. No direct kubectl mutations. +# +# Usage: +# ./neuron-self-improve.sh snapshot # copy prod DB to slot +# ./neuron-self-improve.sh deploy # point slot at a specific image +# ./neuron-self-improve.sh promote # copy slot's image to prod +# ./neuron-self-improve.sh status # show what's running in each zone +# ./neuron-self-improve.sh rollback # revert slot to a previous image + +set -euo pipefail + +INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REGISTRY="registry.neuralplatform.ai/neural-platform/neuron" +KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}" + +# ── helpers ─────────────────────────────────────────────────────────────────── + +slot_manifest() { + local slot=$1 + if [[ "$slot" == "prod" ]]; then + echo "$INFRA_DIR/servers/legion/apps/neuron.yaml" + else + echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml" + fi +} + +current_sha() { + local slot=$1 + grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1 +} + +set_image() { + local slot=$1 sha=$2 + local manifest + manifest="$(slot_manifest "$slot")" + sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest" + echo " set $slot → $sha" +} + +git_push() { + local msg=$1 + cd "$INFRA_DIR" + git add -A + git commit -m "$msg" + git push + echo " pushed: $msg" +} + +wait_healthy() { + local slot=$1 + local app="neuron" + [[ "$slot" != "prod" ]] && app="neuron-${slot}" + echo " waiting for $slot to be healthy..." + local i=0 + while [[ $i -lt 30 ]]; do + local ready + ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ + --no-headers 2>/dev/null | grep "1/1" | wc -l) + [[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0 + sleep 10 + ((i++)) + done + echo " WARNING: $slot did not become healthy within 5 minutes" + return 1 +} + +# ── commands ────────────────────────────────────────────────────────────────── + +cmd_snapshot() { + local slot=${1:?Usage: snapshot } + echo "==> Snapshotting prod DB into $slot..." + KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF +apiVersion: batch/v1 +kind: Job +metadata: + name: snapshot-prod-to-${slot} + namespace: neuron +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + restartPolicy: Never + containers: + - name: snapshot + image: keinos/sqlite3:latest + command: ["/bin/sh", "-c"] + args: + - sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db + volumeMounts: + - name: src + mountPath: /src + readOnly: true + - name: dst + mountPath: /dst + volumes: + - name: src + persistentVolumeClaim: + claimName: neuron-data + - name: dst + persistentVolumeClaim: + claimName: neuron-${slot}-data +EOF + echo " waiting for snapshot job..." + KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \ + -n neuron --for=condition=complete --timeout=120s + echo " snapshot complete" +} + +cmd_deploy() { + local slot=${1:?Usage: deploy } + local sha=${2:?Usage: deploy } + echo "==> Deploying $sha to $slot..." + set_image "$slot" "$sha" + git_push "chore(neuron): deploy ${sha:0:12} to $slot" + wait_healthy "$slot" + echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai" +} + +cmd_promote() { + local slot=${1:?Usage: promote } + local sha + sha=$(current_sha "$slot") + [[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1 + echo "==> Promoting $slot ($sha) to prod..." + set_image "prod" "$sha" + # Also stamp the :stable registry tag (pull and re-tag) + git_push "chore(neuron): promote $slot (${sha:0:12}) to prod" + wait_healthy "prod" + echo "==> prod is now running $sha" + echo "==> prod endpoint: https://neuron.neuralplatform.ai" +} + +cmd_rollback() { + local slot=${1:?Usage: rollback } + local sha=${2:?Usage: rollback } + echo "==> Rolling $slot back to $sha..." + set_image "$slot" "$sha" + git_push "chore(neuron): rollback $slot to ${sha:0:12}" + wait_healthy "$slot" + echo "==> $slot rolled back" +} + +cmd_status() { + echo "==> Neuron slot status" + echo "" + for slot in prod alpha beta gamma; do + local app="neuron" + [[ "$slot" != "prod" ]] && app="neuron-${slot}" + local sha + sha=$(current_sha "$slot" 2>/dev/null || echo "unknown") + local ready + ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ + --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ') + printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready" + done + echo "" + echo " Endpoints:" + echo " prod: https://neuron.neuralplatform.ai" + echo " alpha: https://neuron-alpha.neuralplatform.ai" + echo " beta: https://neuron-beta.neuralplatform.ai" + echo " gamma: https://neuron-gamma.neuralplatform.ai" +} + +# ── self-improvement loop (autonomous) ──────────────────────────────────────── +# When called with no args, prints the process for an autonomous agent to follow. +cmd_loop_process() { + cat << 'PROCESS' +NEURON SELF-IMPROVEMENT LOOP +============================= + +Each iteration: + +1. IDENTIFY — find an improvement opportunity in the codebase. + Sources: Neuron backlog, observed failures, performance gaps, code review. + +2. IMPLEMENT — make the change on a git branch in neural-platform/neuron. + Commit message should describe the hypothesis being tested. + +3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha. + Wait for CI to complete and alpha pod to be healthy. + +4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB: + ./scripts/neuron-self-improve.sh snapshot + This ensures the experiment runs against real data without touching prod. + +5. EXPERIMENT — run the experiment against the alpha endpoint: + https://neuron-alpha.neuralplatform.ai + Use real workloads. Compare against prod or other slots. + Multiple experiments can run simultaneously across alpha/beta/gamma. + +6. EVALUATE — compare results. Criteria: + - Does it behave correctly? + - Does it perform better (latency, quality, fewer errors)? + - Does it handle edge cases prod handles? + Schema changes are acceptable — they become migrations against prod on promote. + +7. PROMOTE WINNER — if alpha wins: + ./scripts/neuron-self-improve.sh promote alpha + This updates prod's image SHA in git → Argo CD deploys → prod is updated. + The old prod SHA is still in the registry — rollback is always available. + +8. ROLLBACK — if something goes wrong: + ./scripts/neuron-self-improve.sh rollback prod + +9. LOOP — go to step 1. Alpha is now free for the next experiment. + +RULES +------ +- Never skip the snapshot step before running an experiment against a slot. +- Prod data is never lost — slots only get copies. +- Schema changes in experiment code → Alembic auto-migrates the snapshot DB. + On promote, the migration runs against prod. Always verify migrations are + additive/non-destructive before promoting. +- All three experiment zones can run simultaneously with different hypotheses. +- The registry retains every SHA — any version can be deployed to any slot. +- This script handles git push; Argo CD handles k8s — never kubectl apply. + +PROCESS +} + +# ── dispatch ────────────────────────────────────────────────────────────────── + +case "${1:-loop}" in + snapshot) cmd_snapshot "${@:2}" ;; + deploy) cmd_deploy "${@:2}" ;; + promote) cmd_promote "${@:2}" ;; + rollback) cmd_rollback "${@:2}" ;; + status) cmd_status ;; + loop) cmd_loop_process ;; + *) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;; +esac