feat(neuron): gamma zone, all-slot ingress, snapshot job, self-improvement loop script
This commit is contained in:
Will Anderson
@@ -0,0 +1,89 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: neuron-gamma
|
||||||
|
namespace: neuron
|
||||||
|
labels:
|
||||||
|
app: neuron-gamma
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: neuron-gamma
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: neuron-gamma
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: neuron
|
||||||
|
image: registry.neuralplatform.ai/neural-platform/neuron:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
command:
|
||||||
|
- bash
|
||||||
|
- -c
|
||||||
|
- mkdir -p /root/.neuron && ln -sf /data/neuron.db /root/.neuron/neuron.db && python -m synapse platform serve --mcp-host 0.0.0.0 --mcp-port 8001 --webhook-port 3847
|
||||||
|
ports:
|
||||||
|
- name: mcp
|
||||||
|
containerPort: 8001
|
||||||
|
- name: webhook
|
||||||
|
containerPort: 3847
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: neuron-config
|
||||||
|
- secretRef:
|
||||||
|
name: neuron-secrets
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
- name: app-config
|
||||||
|
mountPath: /app/config/default.yaml
|
||||||
|
subPath: default.yaml
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 256Mi
|
||||||
|
cpu: 100m
|
||||||
|
ephemeral-storage: 256Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
|
cpu: 500m
|
||||||
|
ephemeral-storage: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: mcp
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 30
|
||||||
|
readinessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: mcp
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
volumes:
|
||||||
|
- name: data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: neuron-gamma-data
|
||||||
|
- name: app-config
|
||||||
|
configMap:
|
||||||
|
name: neuron-config
|
||||||
|
items:
|
||||||
|
- key: default.yaml
|
||||||
|
path: default.yaml
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: neuron-gamma
|
||||||
|
namespace: neuron
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: neuron-gamma
|
||||||
|
ports:
|
||||||
|
- name: mcp
|
||||||
|
port: 8001
|
||||||
|
targetPort: 8001
|
||||||
|
- name: webhook
|
||||||
|
port: 3847
|
||||||
|
targetPort: 3847
|
||||||
|
type: ClusterIP
|
||||||
@@ -12,6 +12,15 @@ spec:
|
|||||||
- secretName: neuron-tls
|
- secretName: neuron-tls
|
||||||
hosts:
|
hosts:
|
||||||
- neuron.neuralplatform.ai
|
- neuron.neuralplatform.ai
|
||||||
|
- secretName: neuron-alpha-tls
|
||||||
|
hosts:
|
||||||
|
- neuron-alpha.neuralplatform.ai
|
||||||
|
- secretName: neuron-beta-tls
|
||||||
|
hosts:
|
||||||
|
- neuron-beta.neuralplatform.ai
|
||||||
|
- secretName: neuron-gamma-tls
|
||||||
|
hosts:
|
||||||
|
- neuron-gamma.neuralplatform.ai
|
||||||
rules:
|
rules:
|
||||||
- host: neuron.neuralplatform.ai
|
- host: neuron.neuralplatform.ai
|
||||||
http:
|
http:
|
||||||
@@ -23,3 +32,33 @@ spec:
|
|||||||
name: neuron
|
name: neuron
|
||||||
port:
|
port:
|
||||||
number: 8001
|
number: 8001
|
||||||
|
- host: neuron-alpha.neuralplatform.ai
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: neuron-alpha
|
||||||
|
port:
|
||||||
|
number: 8001
|
||||||
|
- host: neuron-beta.neuralplatform.ai
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: neuron-beta
|
||||||
|
port:
|
||||||
|
number: 8001
|
||||||
|
- host: neuron-gamma.neuralplatform.ai
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: neuron-gamma
|
||||||
|
port:
|
||||||
|
number: 8001
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: neuron-gamma-data
|
||||||
|
namespace: neuron
|
||||||
|
spec:
|
||||||
|
storageClassName: local-path
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 5Gi
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# Snapshot Job — copies prod DB into an experiment slot via sqlite3 .backup
|
||||||
|
# Usage: set SLOT to alpha, beta, or gamma before applying
|
||||||
|
#
|
||||||
|
# SLOT=alpha && sed "s/SLOT/${SLOT}/g" snapshot-job.yaml | kubectl apply -f -
|
||||||
|
#
|
||||||
|
# The job completes and self-deletes after 5 minutes.
|
||||||
|
# Always snapshot before deploying a new experiment to a slot.
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: snapshot-prod-to-SLOT
|
||||||
|
namespace: neuron
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 300
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
initContainers:
|
||||||
|
- name: stop-check
|
||||||
|
image: registry.neuralplatform.ai/neural-platform/neuron:latest
|
||||||
|
command: ["/bin/sh", "-c", "echo 'Snapshotting prod DB to SLOT — ensure neuron-SLOT pod is stopped first'"]
|
||||||
|
containers:
|
||||||
|
- name: snapshot
|
||||||
|
image: keinos/sqlite3:latest
|
||||||
|
command: ["/bin/sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
echo "Starting backup from prod to SLOT..."
|
||||||
|
sqlite3 /src/neuron.db ".backup /dst/neuron.db"
|
||||||
|
echo "Done. Prod DB snapshotted to SLOT."
|
||||||
|
ls -lh /dst/neuron.db
|
||||||
|
volumeMounts:
|
||||||
|
- name: src
|
||||||
|
mountPath: /src
|
||||||
|
readOnly: true
|
||||||
|
- name: dst
|
||||||
|
mountPath: /dst
|
||||||
|
volumes:
|
||||||
|
- name: src
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: neuron-data
|
||||||
|
- name: dst
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: neuron-SLOT-data
|
||||||
+28
-1
@@ -52,7 +52,7 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "legion" {
|
|||||||
tunnel_id = var.cloudflare_tunnel_id
|
tunnel_id = var.cloudflare_tunnel_id
|
||||||
|
|
||||||
config {
|
config {
|
||||||
# neuron.neuralplatform.ai — MCP server (SSE transport, requires CF-Access headers)
|
# neuron.neuralplatform.ai — MCP server, stable prod
|
||||||
ingress_rule {
|
ingress_rule {
|
||||||
hostname = "neuron.neuralplatform.ai"
|
hostname = "neuron.neuralplatform.ai"
|
||||||
service = "https://traefik.kube-system.svc:443"
|
service = "https://traefik.kube-system.svc:443"
|
||||||
@@ -61,6 +61,33 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "legion" {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# neuron-alpha.neuralplatform.ai — experiment zone 1 (CI auto-deploys here)
|
||||||
|
ingress_rule {
|
||||||
|
hostname = "neuron-alpha.neuralplatform.ai"
|
||||||
|
service = "https://traefik.kube-system.svc:443"
|
||||||
|
origin_request {
|
||||||
|
no_tls_verify = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# neuron-beta.neuralplatform.ai — experiment zone 2
|
||||||
|
ingress_rule {
|
||||||
|
hostname = "neuron-beta.neuralplatform.ai"
|
||||||
|
service = "https://traefik.kube-system.svc:443"
|
||||||
|
origin_request {
|
||||||
|
no_tls_verify = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# neuron-gamma.neuralplatform.ai — experiment zone 3
|
||||||
|
ingress_rule {
|
||||||
|
hostname = "neuron-gamma.neuralplatform.ai"
|
||||||
|
service = "https://traefik.kube-system.svc:443"
|
||||||
|
origin_request {
|
||||||
|
no_tls_verify = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ingress_rule {
|
ingress_rule {
|
||||||
hostname = "vault.neuralplatform.ai"
|
hostname = "vault.neuralplatform.ai"
|
||||||
service = "https://traefik.kube-system.svc:443"
|
service = "https://traefik.kube-system.svc:443"
|
||||||
|
|||||||
+235
@@ -0,0 +1,235 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# neuron-self-improve.sh — self-improvement loop for Neuron
|
||||||
|
#
|
||||||
|
# Manages code experiments across alpha/beta/gamma zones and promotes winners to prod.
|
||||||
|
# All cluster changes go through git → Argo CD. No direct kubectl mutations.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./neuron-self-improve.sh snapshot <slot> # copy prod DB to slot
|
||||||
|
# ./neuron-self-improve.sh deploy <slot> <sha> # point slot at a specific image
|
||||||
|
# ./neuron-self-improve.sh promote <slot> # copy slot's image to prod
|
||||||
|
# ./neuron-self-improve.sh status # show what's running in each zone
|
||||||
|
# ./neuron-self-improve.sh rollback <slot> <sha> # revert slot to a previous image
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
REGISTRY="registry.neuralplatform.ai/neural-platform/neuron"
|
||||||
|
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
||||||
|
|
||||||
|
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
slot_manifest() {
|
||||||
|
local slot=$1
|
||||||
|
if [[ "$slot" == "prod" ]]; then
|
||||||
|
echo "$INFRA_DIR/servers/legion/apps/neuron.yaml"
|
||||||
|
else
|
||||||
|
echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
current_sha() {
|
||||||
|
local slot=$1
|
||||||
|
grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1
|
||||||
|
}
|
||||||
|
|
||||||
|
set_image() {
|
||||||
|
local slot=$1 sha=$2
|
||||||
|
local manifest
|
||||||
|
manifest="$(slot_manifest "$slot")"
|
||||||
|
sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest"
|
||||||
|
echo " set $slot → $sha"
|
||||||
|
}
|
||||||
|
|
||||||
|
git_push() {
|
||||||
|
local msg=$1
|
||||||
|
cd "$INFRA_DIR"
|
||||||
|
git add -A
|
||||||
|
git commit -m "$msg"
|
||||||
|
git push
|
||||||
|
echo " pushed: $msg"
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_healthy() {
|
||||||
|
local slot=$1
|
||||||
|
local app="neuron"
|
||||||
|
[[ "$slot" != "prod" ]] && app="neuron-${slot}"
|
||||||
|
echo " waiting for $slot to be healthy..."
|
||||||
|
local i=0
|
||||||
|
while [[ $i -lt 30 ]]; do
|
||||||
|
local ready
|
||||||
|
ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \
|
||||||
|
--no-headers 2>/dev/null | grep "1/1" | wc -l)
|
||||||
|
[[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0
|
||||||
|
sleep 10
|
||||||
|
((i++))
|
||||||
|
done
|
||||||
|
echo " WARNING: $slot did not become healthy within 5 minutes"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── commands ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
cmd_snapshot() {
|
||||||
|
local slot=${1:?Usage: snapshot <slot>}
|
||||||
|
echo "==> Snapshotting prod DB into $slot..."
|
||||||
|
KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: snapshot-prod-to-${slot}
|
||||||
|
namespace: neuron
|
||||||
|
spec:
|
||||||
|
ttlSecondsAfterFinished: 300
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: snapshot
|
||||||
|
image: keinos/sqlite3:latest
|
||||||
|
command: ["/bin/sh", "-c"]
|
||||||
|
args:
|
||||||
|
- sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db
|
||||||
|
volumeMounts:
|
||||||
|
- name: src
|
||||||
|
mountPath: /src
|
||||||
|
readOnly: true
|
||||||
|
- name: dst
|
||||||
|
mountPath: /dst
|
||||||
|
volumes:
|
||||||
|
- name: src
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: neuron-data
|
||||||
|
- name: dst
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: neuron-${slot}-data
|
||||||
|
EOF
|
||||||
|
echo " waiting for snapshot job..."
|
||||||
|
KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \
|
||||||
|
-n neuron --for=condition=complete --timeout=120s
|
||||||
|
echo " snapshot complete"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_deploy() {
|
||||||
|
local slot=${1:?Usage: deploy <slot> <sha>}
|
||||||
|
local sha=${2:?Usage: deploy <slot> <sha>}
|
||||||
|
echo "==> Deploying $sha to $slot..."
|
||||||
|
set_image "$slot" "$sha"
|
||||||
|
git_push "chore(neuron): deploy ${sha:0:12} to $slot"
|
||||||
|
wait_healthy "$slot"
|
||||||
|
echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_promote() {
|
||||||
|
local slot=${1:?Usage: promote <slot>}
|
||||||
|
local sha
|
||||||
|
sha=$(current_sha "$slot")
|
||||||
|
[[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1
|
||||||
|
echo "==> Promoting $slot ($sha) to prod..."
|
||||||
|
set_image "prod" "$sha"
|
||||||
|
# Also stamp the :stable registry tag (pull and re-tag)
|
||||||
|
git_push "chore(neuron): promote $slot (${sha:0:12}) to prod"
|
||||||
|
wait_healthy "prod"
|
||||||
|
echo "==> prod is now running $sha"
|
||||||
|
echo "==> prod endpoint: https://neuron.neuralplatform.ai"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_rollback() {
|
||||||
|
local slot=${1:?Usage: rollback <slot> <sha>}
|
||||||
|
local sha=${2:?Usage: rollback <slot> <sha>}
|
||||||
|
echo "==> Rolling $slot back to $sha..."
|
||||||
|
set_image "$slot" "$sha"
|
||||||
|
git_push "chore(neuron): rollback $slot to ${sha:0:12}"
|
||||||
|
wait_healthy "$slot"
|
||||||
|
echo "==> $slot rolled back"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_status() {
|
||||||
|
echo "==> Neuron slot status"
|
||||||
|
echo ""
|
||||||
|
for slot in prod alpha beta gamma; do
|
||||||
|
local app="neuron"
|
||||||
|
[[ "$slot" != "prod" ]] && app="neuron-${slot}"
|
||||||
|
local sha
|
||||||
|
sha=$(current_sha "$slot" 2>/dev/null || echo "unknown")
|
||||||
|
local ready
|
||||||
|
ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \
|
||||||
|
--no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ')
|
||||||
|
printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready"
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
echo " Endpoints:"
|
||||||
|
echo " prod: https://neuron.neuralplatform.ai"
|
||||||
|
echo " alpha: https://neuron-alpha.neuralplatform.ai"
|
||||||
|
echo " beta: https://neuron-beta.neuralplatform.ai"
|
||||||
|
echo " gamma: https://neuron-gamma.neuralplatform.ai"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── self-improvement loop (autonomous) ────────────────────────────────────────
|
||||||
|
# When called with no args, prints the process for an autonomous agent to follow.
|
||||||
|
cmd_loop_process() {
|
||||||
|
cat << 'PROCESS'
|
||||||
|
NEURON SELF-IMPROVEMENT LOOP
|
||||||
|
=============================
|
||||||
|
|
||||||
|
Each iteration:
|
||||||
|
|
||||||
|
1. IDENTIFY — find an improvement opportunity in the codebase.
|
||||||
|
Sources: Neuron backlog, observed failures, performance gaps, code review.
|
||||||
|
|
||||||
|
2. IMPLEMENT — make the change on a git branch in neural-platform/neuron.
|
||||||
|
Commit message should describe the hypothesis being tested.
|
||||||
|
|
||||||
|
3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha.
|
||||||
|
Wait for CI to complete and alpha pod to be healthy.
|
||||||
|
|
||||||
|
4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB:
|
||||||
|
./scripts/neuron-self-improve.sh snapshot <slot>
|
||||||
|
This ensures the experiment runs against real data without touching prod.
|
||||||
|
|
||||||
|
5. EXPERIMENT — run the experiment against the alpha endpoint:
|
||||||
|
https://neuron-alpha.neuralplatform.ai
|
||||||
|
Use real workloads. Compare against prod or other slots.
|
||||||
|
Multiple experiments can run simultaneously across alpha/beta/gamma.
|
||||||
|
|
||||||
|
6. EVALUATE — compare results. Criteria:
|
||||||
|
- Does it behave correctly?
|
||||||
|
- Does it perform better (latency, quality, fewer errors)?
|
||||||
|
- Does it handle edge cases prod handles?
|
||||||
|
Schema changes are acceptable — they become migrations against prod on promote.
|
||||||
|
|
||||||
|
7. PROMOTE WINNER — if alpha wins:
|
||||||
|
./scripts/neuron-self-improve.sh promote alpha
|
||||||
|
This updates prod's image SHA in git → Argo CD deploys → prod is updated.
|
||||||
|
The old prod SHA is still in the registry — rollback is always available.
|
||||||
|
|
||||||
|
8. ROLLBACK — if something goes wrong:
|
||||||
|
./scripts/neuron-self-improve.sh rollback prod <previous-sha>
|
||||||
|
|
||||||
|
9. LOOP — go to step 1. Alpha is now free for the next experiment.
|
||||||
|
|
||||||
|
RULES
|
||||||
|
------
|
||||||
|
- Never skip the snapshot step before running an experiment against a slot.
|
||||||
|
- Prod data is never lost — slots only get copies.
|
||||||
|
- Schema changes in experiment code → Alembic auto-migrates the snapshot DB.
|
||||||
|
On promote, the migration runs against prod. Always verify migrations are
|
||||||
|
additive/non-destructive before promoting.
|
||||||
|
- All three experiment zones can run simultaneously with different hypotheses.
|
||||||
|
- The registry retains every SHA — any version can be deployed to any slot.
|
||||||
|
- This script handles git push; Argo CD handles k8s — never kubectl apply.
|
||||||
|
|
||||||
|
PROCESS
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── dispatch ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
case "${1:-loop}" in
|
||||||
|
snapshot) cmd_snapshot "${@:2}" ;;
|
||||||
|
deploy) cmd_deploy "${@:2}" ;;
|
||||||
|
promote) cmd_promote "${@:2}" ;;
|
||||||
|
rollback) cmd_rollback "${@:2}" ;;
|
||||||
|
status) cmd_status ;;
|
||||||
|
loop) cmd_loop_process ;;
|
||||||
|
*) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user