feat(neuron): gamma zone, all-slot ingress, snapshot job, self-improvement loop script
This commit is contained in:
Will Anderson
@@ -0,0 +1,89 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: neuron-gamma
|
||||
namespace: neuron
|
||||
labels:
|
||||
app: neuron-gamma
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-gamma
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: neuron-gamma
|
||||
spec:
|
||||
containers:
|
||||
- name: neuron
|
||||
image: registry.neuralplatform.ai/neural-platform/neuron:latest
|
||||
imagePullPolicy: Always
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- mkdir -p /root/.neuron && ln -sf /data/neuron.db /root/.neuron/neuron.db && python -m synapse platform serve --mcp-host 0.0.0.0 --mcp-port 8001 --webhook-port 3847
|
||||
ports:
|
||||
- name: mcp
|
||||
containerPort: 8001
|
||||
- name: webhook
|
||||
containerPort: 3847
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: neuron-config
|
||||
- secretRef:
|
||||
name: neuron-secrets
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: app-config
|
||||
mountPath: /app/config/default.yaml
|
||||
subPath: default.yaml
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 100m
|
||||
ephemeral-storage: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
cpu: 500m
|
||||
ephemeral-storage: 1Gi
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-gamma-data
|
||||
- name: app-config
|
||||
configMap:
|
||||
name: neuron-config
|
||||
items:
|
||||
- key: default.yaml
|
||||
path: default.yaml
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: neuron-gamma
|
||||
namespace: neuron
|
||||
spec:
|
||||
selector:
|
||||
app: neuron-gamma
|
||||
ports:
|
||||
- name: mcp
|
||||
port: 8001
|
||||
targetPort: 8001
|
||||
- name: webhook
|
||||
port: 3847
|
||||
targetPort: 3847
|
||||
type: ClusterIP
|
||||
@@ -12,6 +12,15 @@ spec:
|
||||
- secretName: neuron-tls
|
||||
hosts:
|
||||
- neuron.neuralplatform.ai
|
||||
- secretName: neuron-alpha-tls
|
||||
hosts:
|
||||
- neuron-alpha.neuralplatform.ai
|
||||
- secretName: neuron-beta-tls
|
||||
hosts:
|
||||
- neuron-beta.neuralplatform.ai
|
||||
- secretName: neuron-gamma-tls
|
||||
hosts:
|
||||
- neuron-gamma.neuralplatform.ai
|
||||
rules:
|
||||
- host: neuron.neuralplatform.ai
|
||||
http:
|
||||
@@ -23,3 +32,33 @@ spec:
|
||||
name: neuron
|
||||
port:
|
||||
number: 8001
|
||||
- host: neuron-alpha.neuralplatform.ai
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: neuron-alpha
|
||||
port:
|
||||
number: 8001
|
||||
- host: neuron-beta.neuralplatform.ai
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: neuron-beta
|
||||
port:
|
||||
number: 8001
|
||||
- host: neuron-gamma.neuralplatform.ai
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: neuron-gamma
|
||||
port:
|
||||
number: 8001
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: neuron-gamma-data
|
||||
namespace: neuron
|
||||
spec:
|
||||
storageClassName: local-path
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
@@ -0,0 +1,44 @@
|
||||
# Snapshot Job — copies prod DB into an experiment slot via sqlite3 .backup
|
||||
# Usage: set SLOT to alpha, beta, or gamma before applying
|
||||
#
|
||||
# SLOT=alpha && sed "s/SLOT/${SLOT}/g" snapshot-job.yaml | kubectl apply -f -
|
||||
#
|
||||
# The job completes and self-deletes after 5 minutes.
|
||||
# Always snapshot before deploying a new experiment to a slot.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: snapshot-prod-to-SLOT
|
||||
namespace: neuron
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 300
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: stop-check
|
||||
image: registry.neuralplatform.ai/neural-platform/neuron:latest
|
||||
command: ["/bin/sh", "-c", "echo 'Snapshotting prod DB to SLOT — ensure neuron-SLOT pod is stopped first'"]
|
||||
containers:
|
||||
- name: snapshot
|
||||
image: keinos/sqlite3:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "Starting backup from prod to SLOT..."
|
||||
sqlite3 /src/neuron.db ".backup /dst/neuron.db"
|
||||
echo "Done. Prod DB snapshotted to SLOT."
|
||||
ls -lh /dst/neuron.db
|
||||
volumeMounts:
|
||||
- name: src
|
||||
mountPath: /src
|
||||
readOnly: true
|
||||
- name: dst
|
||||
mountPath: /dst
|
||||
volumes:
|
||||
- name: src
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-data
|
||||
- name: dst
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-SLOT-data
|
||||
+28
-1
@@ -52,7 +52,7 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "legion" {
|
||||
tunnel_id = var.cloudflare_tunnel_id
|
||||
|
||||
config {
|
||||
# neuron.neuralplatform.ai — MCP server (SSE transport, requires CF-Access headers)
|
||||
# neuron.neuralplatform.ai — MCP server, stable prod
|
||||
ingress_rule {
|
||||
hostname = "neuron.neuralplatform.ai"
|
||||
service = "https://traefik.kube-system.svc:443"
|
||||
@@ -61,6 +61,33 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "legion" {
|
||||
}
|
||||
}
|
||||
|
||||
# neuron-alpha.neuralplatform.ai — experiment zone 1 (CI auto-deploys here)
|
||||
ingress_rule {
|
||||
hostname = "neuron-alpha.neuralplatform.ai"
|
||||
service = "https://traefik.kube-system.svc:443"
|
||||
origin_request {
|
||||
no_tls_verify = true
|
||||
}
|
||||
}
|
||||
|
||||
# neuron-beta.neuralplatform.ai — experiment zone 2
|
||||
ingress_rule {
|
||||
hostname = "neuron-beta.neuralplatform.ai"
|
||||
service = "https://traefik.kube-system.svc:443"
|
||||
origin_request {
|
||||
no_tls_verify = true
|
||||
}
|
||||
}
|
||||
|
||||
# neuron-gamma.neuralplatform.ai — experiment zone 3
|
||||
ingress_rule {
|
||||
hostname = "neuron-gamma.neuralplatform.ai"
|
||||
service = "https://traefik.kube-system.svc:443"
|
||||
origin_request {
|
||||
no_tls_verify = true
|
||||
}
|
||||
}
|
||||
|
||||
ingress_rule {
|
||||
hostname = "vault.neuralplatform.ai"
|
||||
service = "https://traefik.kube-system.svc:443"
|
||||
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env bash
|
||||
# neuron-self-improve.sh — self-improvement loop for Neuron
|
||||
#
|
||||
# Manages code experiments across alpha/beta/gamma zones and promotes winners to prod.
|
||||
# All cluster changes go through git → Argo CD. No direct kubectl mutations.
|
||||
#
|
||||
# Usage:
|
||||
# ./neuron-self-improve.sh snapshot <slot> # copy prod DB to slot
|
||||
# ./neuron-self-improve.sh deploy <slot> <sha> # point slot at a specific image
|
||||
# ./neuron-self-improve.sh promote <slot> # copy slot's image to prod
|
||||
# ./neuron-self-improve.sh status # show what's running in each zone
|
||||
# ./neuron-self-improve.sh rollback <slot> <sha> # revert slot to a previous image
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REGISTRY="registry.neuralplatform.ai/neural-platform/neuron"
|
||||
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
slot_manifest() {
|
||||
local slot=$1
|
||||
if [[ "$slot" == "prod" ]]; then
|
||||
echo "$INFRA_DIR/servers/legion/apps/neuron.yaml"
|
||||
else
|
||||
echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml"
|
||||
fi
|
||||
}
|
||||
|
||||
current_sha() {
|
||||
local slot=$1
|
||||
grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1
|
||||
}
|
||||
|
||||
set_image() {
|
||||
local slot=$1 sha=$2
|
||||
local manifest
|
||||
manifest="$(slot_manifest "$slot")"
|
||||
sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest"
|
||||
echo " set $slot → $sha"
|
||||
}
|
||||
|
||||
git_push() {
|
||||
local msg=$1
|
||||
cd "$INFRA_DIR"
|
||||
git add -A
|
||||
git commit -m "$msg"
|
||||
git push
|
||||
echo " pushed: $msg"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local slot=$1
|
||||
local app="neuron"
|
||||
[[ "$slot" != "prod" ]] && app="neuron-${slot}"
|
||||
echo " waiting for $slot to be healthy..."
|
||||
local i=0
|
||||
while [[ $i -lt 30 ]]; do
|
||||
local ready
|
||||
ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \
|
||||
--no-headers 2>/dev/null | grep "1/1" | wc -l)
|
||||
[[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0
|
||||
sleep 10
|
||||
((i++))
|
||||
done
|
||||
echo " WARNING: $slot did not become healthy within 5 minutes"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── commands ──────────────────────────────────────────────────────────────────
|
||||
|
||||
cmd_snapshot() {
|
||||
local slot=${1:?Usage: snapshot <slot>}
|
||||
echo "==> Snapshotting prod DB into $slot..."
|
||||
KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: snapshot-prod-to-${slot}
|
||||
namespace: neuron
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 300
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: snapshot
|
||||
image: keinos/sqlite3:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db
|
||||
volumeMounts:
|
||||
- name: src
|
||||
mountPath: /src
|
||||
readOnly: true
|
||||
- name: dst
|
||||
mountPath: /dst
|
||||
volumes:
|
||||
- name: src
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-data
|
||||
- name: dst
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-${slot}-data
|
||||
EOF
|
||||
echo " waiting for snapshot job..."
|
||||
KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \
|
||||
-n neuron --for=condition=complete --timeout=120s
|
||||
echo " snapshot complete"
|
||||
}
|
||||
|
||||
cmd_deploy() {
|
||||
local slot=${1:?Usage: deploy <slot> <sha>}
|
||||
local sha=${2:?Usage: deploy <slot> <sha>}
|
||||
echo "==> Deploying $sha to $slot..."
|
||||
set_image "$slot" "$sha"
|
||||
git_push "chore(neuron): deploy ${sha:0:12} to $slot"
|
||||
wait_healthy "$slot"
|
||||
echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai"
|
||||
}
|
||||
|
||||
cmd_promote() {
|
||||
local slot=${1:?Usage: promote <slot>}
|
||||
local sha
|
||||
sha=$(current_sha "$slot")
|
||||
[[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1
|
||||
echo "==> Promoting $slot ($sha) to prod..."
|
||||
set_image "prod" "$sha"
|
||||
# Also stamp the :stable registry tag (pull and re-tag)
|
||||
git_push "chore(neuron): promote $slot (${sha:0:12}) to prod"
|
||||
wait_healthy "prod"
|
||||
echo "==> prod is now running $sha"
|
||||
echo "==> prod endpoint: https://neuron.neuralplatform.ai"
|
||||
}
|
||||
|
||||
cmd_rollback() {
|
||||
local slot=${1:?Usage: rollback <slot> <sha>}
|
||||
local sha=${2:?Usage: rollback <slot> <sha>}
|
||||
echo "==> Rolling $slot back to $sha..."
|
||||
set_image "$slot" "$sha"
|
||||
git_push "chore(neuron): rollback $slot to ${sha:0:12}"
|
||||
wait_healthy "$slot"
|
||||
echo "==> $slot rolled back"
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
echo "==> Neuron slot status"
|
||||
echo ""
|
||||
for slot in prod alpha beta gamma; do
|
||||
local app="neuron"
|
||||
[[ "$slot" != "prod" ]] && app="neuron-${slot}"
|
||||
local sha
|
||||
sha=$(current_sha "$slot" 2>/dev/null || echo "unknown")
|
||||
local ready
|
||||
ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \
|
||||
--no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ')
|
||||
printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready"
|
||||
done
|
||||
echo ""
|
||||
echo " Endpoints:"
|
||||
echo " prod: https://neuron.neuralplatform.ai"
|
||||
echo " alpha: https://neuron-alpha.neuralplatform.ai"
|
||||
echo " beta: https://neuron-beta.neuralplatform.ai"
|
||||
echo " gamma: https://neuron-gamma.neuralplatform.ai"
|
||||
}
|
||||
|
||||
# ── self-improvement loop (autonomous) ────────────────────────────────────────
|
||||
# When called with no args, prints the process for an autonomous agent to follow.
|
||||
cmd_loop_process() {
|
||||
cat << 'PROCESS'
|
||||
NEURON SELF-IMPROVEMENT LOOP
|
||||
=============================
|
||||
|
||||
Each iteration:
|
||||
|
||||
1. IDENTIFY — find an improvement opportunity in the codebase.
|
||||
Sources: Neuron backlog, observed failures, performance gaps, code review.
|
||||
|
||||
2. IMPLEMENT — make the change on a git branch in neural-platform/neuron.
|
||||
Commit message should describe the hypothesis being tested.
|
||||
|
||||
3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha.
|
||||
Wait for CI to complete and alpha pod to be healthy.
|
||||
|
||||
4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB:
|
||||
./scripts/neuron-self-improve.sh snapshot <slot>
|
||||
This ensures the experiment runs against real data without touching prod.
|
||||
|
||||
5. EXPERIMENT — run the experiment against the alpha endpoint:
|
||||
https://neuron-alpha.neuralplatform.ai
|
||||
Use real workloads. Compare against prod or other slots.
|
||||
Multiple experiments can run simultaneously across alpha/beta/gamma.
|
||||
|
||||
6. EVALUATE — compare results. Criteria:
|
||||
- Does it behave correctly?
|
||||
- Does it perform better (latency, quality, fewer errors)?
|
||||
- Does it handle edge cases prod handles?
|
||||
Schema changes are acceptable — they become migrations against prod on promote.
|
||||
|
||||
7. PROMOTE WINNER — if alpha wins:
|
||||
./scripts/neuron-self-improve.sh promote alpha
|
||||
This updates prod's image SHA in git → Argo CD deploys → prod is updated.
|
||||
The old prod SHA is still in the registry — rollback is always available.
|
||||
|
||||
8. ROLLBACK — if something goes wrong:
|
||||
./scripts/neuron-self-improve.sh rollback prod <previous-sha>
|
||||
|
||||
9. LOOP — go to step 1. Alpha is now free for the next experiment.
|
||||
|
||||
RULES
|
||||
------
|
||||
- Never skip the snapshot step before running an experiment against a slot.
|
||||
- Prod data is never lost — slots only get copies.
|
||||
- Schema changes in experiment code → Alembic auto-migrates the snapshot DB.
|
||||
On promote, the migration runs against prod. Always verify migrations are
|
||||
additive/non-destructive before promoting.
|
||||
- All three experiment zones can run simultaneously with different hypotheses.
|
||||
- The registry retains every SHA — any version can be deployed to any slot.
|
||||
- This script handles git push; Argo CD handles k8s — never kubectl apply.
|
||||
|
||||
PROCESS
|
||||
}
|
||||
|
||||
# ── dispatch ──────────────────────────────────────────────────────────────────
|
||||
|
||||
case "${1:-loop}" in
|
||||
snapshot) cmd_snapshot "${@:2}" ;;
|
||||
deploy) cmd_deploy "${@:2}" ;;
|
||||
promote) cmd_promote "${@:2}" ;;
|
||||
rollback) cmd_rollback "${@:2}" ;;
|
||||
status) cmd_status ;;
|
||||
loop) cmd_loop_process ;;
|
||||
*) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user