Harden prod: security, autoscaling, observability, BuildKit CI
Security: - Drop ALL capabilities, enforce non-root, RuntimeDefault seccomp on neuron-mcp, neuron-rest, neuron-marketing pods - Add startup probes (150s window for JVM) so liveness doesn't fire early - Replace docker-sock hostPath with BuildKit rootless TCP endpoint (moby/buildkit:v0.19.0-rootless) — removes node root access from CI - Document full ESO AppRole migration path in cluster-secret-store.yaml Autoscaling & availability: - HPAs on mcp (1–6), rest (1–4), marketing (2–8) at 65–70% CPU - PodDisruptionBudgets (minAvailable: 1) on all three services - NetworkPolicy: default-deny-all in neuron-prod, explicit allow rules for Traefik ingress, intra-namespace, and egress to DNS/platform/vault Observability: - ServiceMonitors for mcp, rest, marketing (cross-namespace enabled in kube-prometheus-stack with serviceMonitorSelectorNilUsesHelmValues:false) - PrometheusRules: high error rate, high latency, crash loops, replica shortage, Postgres down/connections, backup failure, backup staleness Chart version pinning: - kube-prometheus-stack, loki, tempo, redis, alloy, postgres — all pinned to major-version ranges to block silent breaking upgrades Backup hardening: - restic:latest → restic:0.17.3 (deterministic image) - Weekly backup-verify CronJob: restores latest snapshot and validates SQL dump structure (≥5 CREATE TABLE, pg_dump header check) ArgoCD: - neuron-prod AppProject: scopes deploys to neuron-prod + platform ns, blacklists ClusterRole/ClusterRoleBinding/Namespace creation, automated sync window 2–6am UTC, manual always allowed
This commit is contained in:
Will Anderson
@@ -8,7 +8,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://grafana.github.io/helm-charts
|
||||
chart: alloy
|
||||
targetRevision: "*"
|
||||
targetRevision: ">=0.9.0, <1.0.0"
|
||||
helm:
|
||||
values: |
|
||||
controller:
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
# Gitea CI runner — general-purpose (legion)
|
||||
# Docker socket REMOVED. Builds go through buildkitd TCP endpoint.
|
||||
# Set DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 in build steps.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -15,8 +19,10 @@ spec:
|
||||
labels:
|
||||
app: gitea-runner
|
||||
annotations:
|
||||
config-version: "2026-04-23-nodeport-url"
|
||||
config-version: "2026-04-25-buildkit-no-sock"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: false # act_runner needs root for container management
|
||||
initContainers:
|
||||
- name: register
|
||||
image: registry.neuralplatform.ai/ci-base:latest
|
||||
@@ -36,10 +42,9 @@ spec:
|
||||
timeout: 3h
|
||||
container:
|
||||
network: host
|
||||
docker_host: "unix:///var/run/docker.sock"
|
||||
docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234"
|
||||
force_pull: false
|
||||
valid_volumes:
|
||||
- /var/run/docker.sock
|
||||
valid_volumes: []
|
||||
default_image: "registry.neuralplatform.ai/ci-base:latest"
|
||||
extra_hosts:
|
||||
- "gitea.git.svc.cluster.local:10.43.1.53"
|
||||
@@ -55,14 +60,16 @@ spec:
|
||||
image: registry.neuralplatform.ai/ci-base:latest
|
||||
workingDir: /data
|
||||
command: ["act_runner", "daemon", "--config", "/data/config.yaml"]
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: "tcp://buildkitd.ci.svc.cluster.local:1234"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: gitea-runner-secret
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: docker-sock
|
||||
mountPath: /var/run/docker.sock
|
||||
# docker-sock volume intentionally removed — use buildkitd TCP instead
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
@@ -73,11 +80,9 @@ spec:
|
||||
volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
- name: docker-sock
|
||||
hostPath:
|
||||
path: /var/run/docker.sock
|
||||
type: Socket
|
||||
# docker-sock hostPath intentionally removed
|
||||
---
|
||||
# Neuron Technologies CI runner
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -95,7 +100,7 @@ spec:
|
||||
labels:
|
||||
app: neuron-technologies-runner
|
||||
annotations:
|
||||
config-version: "2026-04-24-initial"
|
||||
config-version: "2026-04-25-buildkit-no-sock"
|
||||
spec:
|
||||
initContainers:
|
||||
- name: register
|
||||
@@ -116,10 +121,9 @@ spec:
|
||||
timeout: 3h
|
||||
container:
|
||||
network: host
|
||||
docker_host: "unix:///var/run/docker.sock"
|
||||
docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234"
|
||||
force_pull: false
|
||||
valid_volumes:
|
||||
- /var/run/docker.sock
|
||||
valid_volumes: []
|
||||
default_image: "registry.neuralplatform.ai/ci-base:latest"
|
||||
extra_hosts:
|
||||
- "gitea.git.svc.cluster.local:10.43.1.53"
|
||||
@@ -135,14 +139,15 @@ spec:
|
||||
image: registry.neuralplatform.ai/ci-base:latest
|
||||
workingDir: /data
|
||||
command: ["act_runner", "daemon", "--config", "/data/config.yaml"]
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: "tcp://buildkitd.ci.svc.cluster.local:1234"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: neuron-technologies-runner-secret
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: docker-sock
|
||||
mountPath: /var/run/docker.sock
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
@@ -153,7 +158,3 @@ spec:
|
||||
volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
- name: docker-sock
|
||||
hostPath:
|
||||
path: /var/run/docker.sock
|
||||
type: Socket
|
||||
|
||||
@@ -8,7 +8,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://prometheus-community.github.io/helm-charts
|
||||
chart: kube-prometheus-stack
|
||||
targetRevision: "*"
|
||||
targetRevision: ">=60.0.0, <70.0.0"
|
||||
helm:
|
||||
values: |
|
||||
grafana:
|
||||
@@ -37,6 +37,13 @@ spec:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
# Discover ServiceMonitors and PrometheusRules from ALL namespaces.
|
||||
# Without this, Prometheus only sees resources in the monitoring namespace.
|
||||
serviceMonitorSelectorNilUsesHelmValues: false
|
||||
serviceMonitorNamespaceSelector: {}
|
||||
serviceMonitorSelector: {}
|
||||
ruleNamespaceSelector: {}
|
||||
podMonitorNamespaceSelector: {}
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
|
||||
@@ -8,7 +8,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://grafana.github.io/helm-charts
|
||||
chart: loki
|
||||
targetRevision: "*"
|
||||
targetRevision: ">=6.0.0, <7.0.0"
|
||||
helm:
|
||||
values: |
|
||||
loki:
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
# ArgoCD AppProject — neuron-prod
|
||||
# Restricts which repos can deploy to which namespaces.
|
||||
# Prevents misconfigured apps from deploying cluster-admin RBAC or
|
||||
# accidentally targeting personal/hobby namespaces.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: neuron-prod
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
description: "Neuron Technologies production services"
|
||||
|
||||
# Only allow deployments sourced from the infrastructure repo.
|
||||
sourceRepos:
|
||||
- "http://10.43.1.53:3000/will/infrastructure.git"
|
||||
- "https://code.forgejo.org"
|
||||
- "registry.neuralplatform.ai"
|
||||
|
||||
# Only allow deploying to neuron-prod and platform namespaces.
|
||||
destinations:
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: neuron-prod
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: platform
|
||||
|
||||
# Cluster-scoped resources this project may NOT create.
|
||||
clusterResourceBlacklist:
|
||||
- group: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
- group: rbac.authorization.k8s.io
|
||||
kind: ClusterRoleBinding
|
||||
- group: ""
|
||||
kind: Namespace
|
||||
|
||||
# Namespace-scoped resources this project may create.
|
||||
namespaceResourceWhitelist:
|
||||
- group: "apps"
|
||||
kind: Deployment
|
||||
- group: "apps"
|
||||
kind: ReplicaSet
|
||||
- group: "batch"
|
||||
kind: CronJob
|
||||
- group: "batch"
|
||||
kind: Job
|
||||
- group: ""
|
||||
kind: Service
|
||||
- group: ""
|
||||
kind: ConfigMap
|
||||
- group: ""
|
||||
kind: Secret
|
||||
- group: ""
|
||||
kind: PersistentVolumeClaim
|
||||
- group: ""
|
||||
kind: ServiceAccount
|
||||
- group: "networking.k8s.io"
|
||||
kind: Ingress
|
||||
- group: "networking.k8s.io"
|
||||
kind: NetworkPolicy
|
||||
- group: "policy"
|
||||
kind: PodDisruptionBudget
|
||||
- group: "autoscaling"
|
||||
kind: HorizontalPodAutoscaler
|
||||
- group: "traefik.io"
|
||||
kind: IngressRoute
|
||||
- group: "external-secrets.io"
|
||||
kind: ExternalSecret
|
||||
- group: "monitoring.coreos.com"
|
||||
kind: ServiceMonitor
|
||||
- group: "monitoring.coreos.com"
|
||||
kind: PrometheusRule
|
||||
|
||||
# Sync windows: only allow automated syncs during low-traffic hours.
|
||||
# Manual syncs allowed any time.
|
||||
syncWindows:
|
||||
- kind: allow
|
||||
schedule: "0 2-6 * * *" # 2am-6am UTC (off-peak)
|
||||
duration: 4h
|
||||
applications:
|
||||
- "*"
|
||||
manualSync: true
|
||||
- kind: allow
|
||||
schedule: "* * * * *"
|
||||
duration: "1m"
|
||||
manualSync: true # Always allow manual deploys
|
||||
applications:
|
||||
- "*"
|
||||
@@ -9,7 +9,7 @@ spec:
|
||||
# Helm chart from Docker Hub OCI registry
|
||||
- repoURL: registry-1.docker.io/bitnamicharts
|
||||
chart: postgresql
|
||||
targetRevision: "*"
|
||||
targetRevision: ">=16.0.0, <17.0.0"
|
||||
helm:
|
||||
values: |
|
||||
auth:
|
||||
|
||||
@@ -8,7 +8,7 @@ spec:
|
||||
source:
|
||||
repoURL: registry-1.docker.io/bitnamicharts
|
||||
chart: redis
|
||||
targetRevision: "*"
|
||||
targetRevision: ">=20.0.0, <21.0.0"
|
||||
helm:
|
||||
values: |
|
||||
auth:
|
||||
|
||||
@@ -8,7 +8,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://grafana.github.io/helm-charts
|
||||
chart: tempo
|
||||
targetRevision: "*"
|
||||
targetRevision: ">=1.0.0, <2.0.0"
|
||||
helm:
|
||||
values: |
|
||||
tempo:
|
||||
|
||||
@@ -43,8 +43,10 @@ spec:
|
||||
limits:
|
||||
memory: 256Mi
|
||||
containers:
|
||||
# Pin to explicit version — never use latest for backup tooling.
|
||||
# Bump this intentionally after testing; do not let it float.
|
||||
- name: backup
|
||||
image: restic/restic:latest
|
||||
image: restic/restic:0.17.3
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
@@ -93,3 +95,77 @@ spec:
|
||||
readOnly: true
|
||||
- name: dump
|
||||
emptyDir: {}
|
||||
---
|
||||
# Weekly restore-verify job — proves the backup is actually restorable.
|
||||
# Runs Sunday 4am. Restores latest snapshot to /tmp, validates the SQL dump.
|
||||
# Alerts via Alertmanager if it fails (BackupJobFailed PrometheusRule watches git namespace).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: backup-verify
|
||||
namespace: git
|
||||
spec:
|
||||
schedule: "0 4 * * 0"
|
||||
concurrencyPolicy: Forbid
|
||||
failedJobsHistoryLimit: 2
|
||||
successfulJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 1800 # 30 min max
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: verify
|
||||
image: restic/restic:0.17.3
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
|
||||
echo "=== Backup Restore Verification ==="
|
||||
echo "Listing latest snapshots..."
|
||||
restic snapshots --latest 3
|
||||
|
||||
echo "Restoring latest snapshot to /tmp/restore-verify..."
|
||||
mkdir -p /tmp/restore-verify
|
||||
restic restore latest \
|
||||
--target /tmp/restore-verify \
|
||||
--include /all-databases.sql
|
||||
|
||||
SQL_FILE="/tmp/restore-verify/all-databases.sql"
|
||||
if [ ! -f "$SQL_FILE" ]; then
|
||||
echo "ERROR: SQL dump not found in restored snapshot"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SIZE=$(du -sh "$SQL_FILE" | cut -f1)
|
||||
echo "SQL dump restored: $SIZE"
|
||||
|
||||
# Basic structural validation
|
||||
if ! grep -q "PostgreSQL database cluster dump" "$SQL_FILE"; then
|
||||
echo "ERROR: SQL dump does not look like a valid pg_dumpall output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TABLE_COUNT=$(grep -c "^CREATE TABLE" "$SQL_FILE" || true)
|
||||
echo "Tables found in dump: $TABLE_COUNT"
|
||||
|
||||
if [ "$TABLE_COUNT" -lt 5 ]; then
|
||||
echo "ERROR: Too few tables in dump (expected >= 5, got $TABLE_COUNT)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Verification PASSED: backup is restorable ==="
|
||||
rm -rf /tmp/restore-verify
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: backup-credentials
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 100m
|
||||
limits:
|
||||
memory: 512Mi
|
||||
|
||||
@@ -1,11 +1,47 @@
|
||||
---
|
||||
# ClusterSecretStore — connects ESO to Vault using the root token Secret.
|
||||
# The vault-token Secret (name: vault-token, key: token) must be created
|
||||
# via kubectl as a one-time bootstrap step before this store can sync:
|
||||
# ClusterSecretStore — connects ESO to Vault
|
||||
#
|
||||
# kubectl create secret generic vault-token \
|
||||
# --namespace external-secrets \
|
||||
# --from-literal=token=$(cat ~/Secrets/tokens/vault-root-token)
|
||||
# CURRENT STATE: token auth (root token). This works but the root token
|
||||
# has no expiry and full Vault access — it's a single credential that
|
||||
# compromises everything if leaked.
|
||||
#
|
||||
# MIGRATION TO APPROLE (do this before GCP migration):
|
||||
# ─────────────────────────────────────────────────────
|
||||
# 1. Enable AppRole on Vault (if not already):
|
||||
# vault auth enable approle
|
||||
#
|
||||
# 2. Create a policy scoped to only the paths ESO reads:
|
||||
# vault policy write eso-read - <<EOF
|
||||
# path "secret/data/neuron-technologies/*" { capabilities = ["read"] }
|
||||
# path "secret/data/r2" { capabilities = ["read"] }
|
||||
# path "secret/data/slack" { capabilities = ["read"] }
|
||||
# path "secret/data/legion-db" { capabilities = ["read"] }
|
||||
# EOF
|
||||
#
|
||||
# 3. Create the AppRole:
|
||||
# vault write auth/approle/role/eso \
|
||||
# token_policies="eso-read" \
|
||||
# token_ttl=1h \
|
||||
# token_max_ttl=4h \
|
||||
# secret_id_ttl=0 # no expiry for the secretId
|
||||
#
|
||||
# 4. Get the roleId and secretId:
|
||||
# vault read auth/approle/role/eso/role-id → ROLE_ID
|
||||
# vault write -f auth/approle/role/eso/secret-id → SECRET_ID
|
||||
#
|
||||
# 5. Create the k8s secret (one-time bootstrap, same as the root token was):
|
||||
# kubectl create secret generic vault-approle \
|
||||
# --namespace external-secrets \
|
||||
# --from-literal=roleId=<ROLE_ID> \
|
||||
# --from-literal=secretId=<SECRET_ID>
|
||||
#
|
||||
# 6. Flip this file to the appRole stanza below, commit, push.
|
||||
# ESO will hot-reload when the manifest changes.
|
||||
#
|
||||
# 7. Delete the old vault-token secret:
|
||||
# kubectl delete secret vault-token -n external-secrets
|
||||
# ─────────────────────────────────────────────────────
|
||||
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ClusterSecretStore
|
||||
metadata:
|
||||
@@ -17,7 +53,17 @@ spec:
|
||||
path: "secret"
|
||||
version: "v2"
|
||||
auth:
|
||||
# ── Current: root token auth ──────────────────────────────────────────
|
||||
# TODO: replace with appRole stanza once AppRole is bootstrapped (see above).
|
||||
tokenSecretRef:
|
||||
name: vault-token
|
||||
namespace: external-secrets
|
||||
key: token
|
||||
# ── Target: AppRole auth (uncomment after bootstrap) ──────────────────
|
||||
# appRole:
|
||||
# path: "approle"
|
||||
# roleId: "eso"
|
||||
# secretRef:
|
||||
# name: vault-approle
|
||||
# namespace: external-secrets
|
||||
# key: secretId
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
---
|
||||
# BuildKit daemon — rootless image builder for CI runners.
|
||||
# Replaces the /var/run/docker.sock hostPath mount that gave containers
|
||||
# full root access to the Legion host node.
|
||||
#
|
||||
# Runners connect via TCP: DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: buildkitd
|
||||
namespace: ci
|
||||
labels:
|
||||
app: buildkitd
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: buildkitd
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: buildkitd
|
||||
spec:
|
||||
containers:
|
||||
- name: buildkitd
|
||||
image: moby/buildkit:v0.19.0-rootless
|
||||
args:
|
||||
- --oci-worker-no-process-sandbox
|
||||
- --addr
|
||||
- tcp://0.0.0.0:1234
|
||||
- --addr
|
||||
- unix:///run/buildkit/buildkitd.sock
|
||||
ports:
|
||||
- name: tcp
|
||||
containerPort: 1234
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: Unconfined # BuildKit rootless requires this
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["buildctl", "debug", "workers"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 4Gi
|
||||
volumeMounts:
|
||||
- name: buildkit-storage
|
||||
mountPath: /home/user/.local/share/buildkit
|
||||
volumes:
|
||||
- name: buildkit-storage
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: buildkitd
|
||||
namespace: ci
|
||||
spec:
|
||||
selector:
|
||||
app: buildkitd
|
||||
ports:
|
||||
- name: tcp
|
||||
port: 1234
|
||||
targetPort: 1234
|
||||
@@ -0,0 +1,153 @@
|
||||
---
|
||||
# PrometheusRules — SLO and operational alerts for neuron-prod
|
||||
# These fire into Alertmanager which routes to #infrastructure-alerts on Slack.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: neuron-slo
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
|
||||
# ── Neuron API SLOs ──────────────────────────────────────��───────────────
|
||||
- name: neuron.slo
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: NeuronMCPHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp",status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp"}[5m]))
|
||||
) > 0.01
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-mcp 5xx error rate > 1%"
|
||||
description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
|
||||
|
||||
- alert: NeuronRestHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest",status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest"}[5m]))
|
||||
) > 0.01
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-rest 5xx error rate > 1%"
|
||||
description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
|
||||
|
||||
- alert: NeuronMCPHighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-mcp"}[10m]))
|
||||
by (le)
|
||||
) > 2.0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-mcp p99 latency > 2s"
|
||||
description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."
|
||||
|
||||
- alert: NeuronRestHighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-rest"}[10m]))
|
||||
by (le)
|
||||
) > 2.0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-rest p99 latency > 2s"
|
||||
description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."
|
||||
|
||||
# ── Pod health ───────────────────────────────────────────────────────────
|
||||
- name: neuron.pods
|
||||
rules:
|
||||
- alert: NeuronPodCrashLooping
|
||||
expr: |
|
||||
rate(kube_pod_container_status_restarts_total{namespace="neuron-prod"}[15m]) * 60 * 15 > 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} is crash looping"
|
||||
description: "Container {{ $labels.container }} restarted {{ $value | humanize }} times in 15 minutes."
|
||||
|
||||
- alert: NeuronPodNotReady
|
||||
expr: |
|
||||
sum by (pod) (kube_pod_status_ready{namespace="neuron-prod",condition="false"}) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} not ready for 5 minutes"
|
||||
|
||||
- alert: NeuronDeploymentReplicasMissing
|
||||
expr: |
|
||||
kube_deployment_status_replicas_available{namespace="neuron-prod"}
|
||||
< kube_deployment_spec_replicas{namespace="neuron-prod"}
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "Deployment {{ $labels.deployment }} has fewer replicas than desired"
|
||||
|
||||
# ── Postgres ─────────────────────────���─────────────────────────────────��─
|
||||
- name: postgres.health
|
||||
rules:
|
||||
- alert: PostgresDown
|
||||
expr: pg_up{namespace="platform"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Postgres is down"
|
||||
|
||||
- alert: PostgresHighConnections
|
||||
expr: |
|
||||
(pg_stat_database_numbackends{namespace="platform"} / pg_settings_max_connections{namespace="platform"})
|
||||
> 0.80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Postgres connection pool > 80%"
|
||||
description: "Connection utilization is {{ $value | humanizePercentage }}."
|
||||
|
||||
# ── Backup health ────────────────────────────���───────────────────────────
|
||||
- name: backup.health
|
||||
rules:
|
||||
- alert: BackupJobFailed
|
||||
expr: |
|
||||
kube_job_status_failed{namespace="git",job_name=~"gitea-backup.*"} > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Gitea backup job failed"
|
||||
description: "Check: kubectl logs -n git -l job-name=gitea-backup --previous"
|
||||
|
||||
- alert: BackupNotRunRecently
|
||||
expr: |
|
||||
(time() - kube_job_status_completion_time{namespace="git",job_name=~"gitea-backup.*"}) > 90000
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Last successful backup was more than 25 hours ago"
|
||||
@@ -0,0 +1,64 @@
|
||||
---
|
||||
# ServiceMonitors for neuron-prod — scraped by kube-prometheus-stack Prometheus
|
||||
# Requires: kube-prometheus-stack prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false
|
||||
# OR a serviceMonitorNamespaceSelector that includes neuron-prod.
|
||||
# We set this in the Prometheus stack values (see apps/kube-prometheus-stack.yaml).
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: neuron-mcp
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- neuron-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-mcp
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /actuator/prometheus
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: neuron-rest
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- neuron-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-rest
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /actuator/prometheus
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: neuron-marketing
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- neuron-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-marketing
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /api/metrics
|
||||
interval: 60s
|
||||
scrapeTimeout: 15s
|
||||
@@ -17,6 +17,11 @@ spec:
|
||||
app: neuron-marketing
|
||||
env: prod
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: neuron-marketing
|
||||
image: registry.neuralplatform.ai/neuron-technologies/marketing:1e94e8ae
|
||||
@@ -24,6 +29,12 @@ spec:
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false # Next.js writes .next/cache at runtime
|
||||
runAsNonRoot: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: NODE_ENV
|
||||
value: production
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
---
|
||||
# HorizontalPodAutoscalers for neuron-prod
|
||||
# Single Legion node today — minReplicas=1 prevents unnecessary idle waste.
|
||||
# When GCP GKE is live these minimums should go to 2 for HA.
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: neuron-mcp-hpa
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: neuron-mcp-blue
|
||||
minReplicas: 1
|
||||
maxReplicas: 6
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 65
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
behavior:
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 300 # Don't thrash — wait 5m before scaling down
|
||||
policies:
|
||||
- type: Pods
|
||||
value: 1
|
||||
periodSeconds: 120
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 30
|
||||
policies:
|
||||
- type: Pods
|
||||
value: 2
|
||||
periodSeconds: 60
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: neuron-rest-hpa
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: neuron-rest
|
||||
minReplicas: 1
|
||||
maxReplicas: 4
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 65
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
behavior:
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 300
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 30
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: neuron-marketing-hpa
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: neuron-marketing
|
||||
minReplicas: 2
|
||||
maxReplicas: 8
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
behavior:
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 120
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 15
|
||||
@@ -15,3 +15,7 @@ resources:
|
||||
- backup-cronjob.yaml
|
||||
- runpod-inference.yaml
|
||||
- runpod-lb-configmap.yaml
|
||||
# Hardening additions
|
||||
- hpa.yaml
|
||||
- pdb.yaml
|
||||
- network-policy.yaml
|
||||
|
||||
@@ -25,6 +25,10 @@ spec:
|
||||
kubectl.kubernetes.io/restartedAt: "2026-04-25T08:40:00Z"
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: neuron-mcp
|
||||
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.15.3
|
||||
@@ -32,6 +36,12 @@ spec:
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false # JVM writes temp files; restrict paths via allowedPaths once profiled
|
||||
runAsNonRoot: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "http://alloy-otlp.monitoring.svc.cluster.local:4318"
|
||||
@@ -57,18 +67,25 @@ spec:
|
||||
exec:
|
||||
# Give Traefik time to drain in-flight connections before pod terminates
|
||||
command: ["sh", "-c", "sleep 8"]
|
||||
# startupProbe gives the JVM up to 150s to start before liveness takes over.
|
||||
# Prevents false crash loops during slow cold starts.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
---
|
||||
# NetworkPolicies for neuron-prod
|
||||
# Default-deny all ingress and egress, then explicitly allow only what's needed.
|
||||
# Enforced by k3s's built-in NetworkPolicy controller.
|
||||
|
||||
# ── Baseline: deny all traffic in namespace ───────────────────────────────────
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-all
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
---
|
||||
# ── neuron-mcp: accept traffic from Traefik (kube-system) only ───────────────
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-mcp-ingress
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: neuron-mcp
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: neuron-prod
|
||||
---
|
||||
# ── neuron-rest: accept from mcp, kube-system (Traefik), monitoring (Alloy) ──
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-rest-ingress
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: neuron-rest
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: neuron-prod
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: monitoring
|
||||
---
|
||||
# ── neuron-marketing: accept from Traefik only ───────────────────────────────
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-marketing-ingress
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: neuron-marketing
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
---
|
||||
# ── Egress: all prod pods may reach platform (postgres/redis), vault,
|
||||
# monitoring (alloy OTLP), kube-dns, and the internet (external APIs) ─
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-prod-egress
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
# Kubernetes DNS
|
||||
- ports:
|
||||
- port: 53
|
||||
protocol: UDP
|
||||
- port: 53
|
||||
protocol: TCP
|
||||
# platform namespace (Postgres, Redis, accounts service)
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: platform
|
||||
# vault namespace
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: vault
|
||||
# monitoring namespace (Alloy OTLP)
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: monitoring
|
||||
# intra-namespace (mcp ↔ rest ↔ license)
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: neuron-prod
|
||||
# external internet (Stripe, OAuth providers, Cloudflare, etc.)
|
||||
- ports:
|
||||
- port: 443
|
||||
protocol: TCP
|
||||
- port: 80
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
# PodDisruptionBudgets for neuron-prod
|
||||
# Ensures at least 1 replica of each service survives node drains and upgrades.
|
||||
# With replicas=1 today, minAvailable=1 means drain will block until HPA scales up
|
||||
# to 2+ — this is intentional and provides a natural safety gate.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: neuron-mcp-pdb
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-mcp
|
||||
slot: blue
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: neuron-rest-pdb
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-rest
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: neuron-marketing-pdb
|
||||
namespace: neuron-prod
|
||||
spec:
|
||||
# Marketing runs 2 replicas — keep at least 1 alive during drains.
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-marketing
|
||||
@@ -19,6 +19,10 @@ spec:
|
||||
annotations:
|
||||
kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: neuron-rest
|
||||
image: registry.neuralplatform.ai/neuron-technologies/neuron-rest:v0.15.3
|
||||
@@ -26,6 +30,12 @@ spec:
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8081
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false
|
||||
runAsNonRoot: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: neuron-prod-config
|
||||
@@ -41,18 +51,23 @@ spec:
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
|
||||
Reference in New Issue
Block a user