will.anderson/infrastructure
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden prod: security, autoscaling, observability, BuildKit CI

Browse Source 
Security:
- Drop ALL capabilities, enforce non-root, RuntimeDefault seccomp on
  neuron-mcp, neuron-rest, neuron-marketing pods
- Add startup probes (150s window for JVM) so liveness doesn't fire early
- Replace docker-sock hostPath with BuildKit rootless TCP endpoint
  (moby/buildkit:v0.19.0-rootless) — removes node root access from CI
- Document full ESO AppRole migration path in cluster-secret-store.yaml

Autoscaling & availability:
- HPAs on mcp (1–6), rest (1–4), marketing (2–8) at 65–70% CPU
- PodDisruptionBudgets (minAvailable: 1) on all three services
- NetworkPolicy: default-deny-all in neuron-prod, explicit allow rules
  for Traefik ingress, intra-namespace, and egress to DNS/platform/vault

Observability:
- ServiceMonitors for mcp, rest, marketing (cross-namespace enabled in
  kube-prometheus-stack with serviceMonitorSelectorNilUsesHelmValues:false)
- PrometheusRules: high error rate, high latency, crash loops, replica
  shortage, Postgres down/connections, backup failure, backup staleness

Chart version pinning:
- kube-prometheus-stack, loki, tempo, redis, alloy, postgres — all pinned
  to major-version ranges to block silent breaking upgrades

Backup hardening:
- restic:latest → restic:0.17.3 (deterministic image)
- Weekly backup-verify CronJob: restores latest snapshot and validates
  SQL dump structure (≥5 CREATE TABLE, pg_dump header check)

ArgoCD:
- neuron-prod AppProject: scopes deploys to neuron-prod + platform ns,
  blacklists ClusterRole/ClusterRoleBinding/Namespace creation,
  automated sync window 2–6am UTC, manual always allowed
This commit is contained in:
Will Anderson
2026-04-25 22:34:59 -05:00
parent 8fd3d12907
commit 93358505fc
20 changed files with 852 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
servers/legion/apps/alloy.yaml
+1 -1
View File
@@ -8,7 +8,7 @@ spec:
source:
repoURL: https://grafana.github.io/helm-charts
chart: alloy
targetRevision: "*"
targetRevision: ">=0.9.0, <1.0.0"
helm:
values: |
controller:
servers/legion/apps/gitea-runner.yaml
+21 -20
View File
@@ -1,3 +1,7 @@
---
# Gitea CI runner — general-purpose (legion)
# Docker socket REMOVED. Builds go through buildkitd TCP endpoint.
# Set DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 in build steps.
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -15,8 +19,10 @@ spec:
labels:
app: gitea-runner
annotations:
config-version: "2026-04-23-nodeport-url"
config-version: "2026-04-25-buildkit-no-sock"
spec:
securityContext:
runAsNonRoot: false # act_runner needs root for container management
initContainers:
- name: register
image: registry.neuralplatform.ai/ci-base:latest
@@ -36,10 +42,9 @@ spec:
timeout: 3h
container:
network: host
docker_host: "unix:///var/run/docker.sock"
docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234"
force_pull: false
valid_volumes:
- /var/run/docker.sock
valid_volumes: []
default_image: "registry.neuralplatform.ai/ci-base:latest"
extra_hosts:
- "gitea.git.svc.cluster.local:10.43.1.53"
@@ -55,14 +60,16 @@ spec:
image: registry.neuralplatform.ai/ci-base:latest
workingDir: /data
command: ["act_runner", "daemon", "--config", "/data/config.yaml"]
env:
- name: DOCKER_HOST
value: "tcp://buildkitd.ci.svc.cluster.local:1234"
envFrom:
- secretRef:
name: gitea-runner-secret
volumeMounts:
- name: data
mountPath: /data
- name: docker-sock
mountPath: /var/run/docker.sock
# docker-sock volume intentionally removed — use buildkitd TCP instead
resources:
requests:
memory: 512Mi
@@ -73,11 +80,9 @@ spec:
volumes:
- name: data
emptyDir: {}
- name: docker-sock
hostPath:
path: /var/run/docker.sock
type: Socket
# docker-sock hostPath intentionally removed
---
# Neuron Technologies CI runner
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -95,7 +100,7 @@ spec:
labels:
app: neuron-technologies-runner
annotations:
config-version: "2026-04-24-initial"
config-version: "2026-04-25-buildkit-no-sock"
spec:
initContainers:
- name: register
@@ -116,10 +121,9 @@ spec:
timeout: 3h
container:
network: host
docker_host: "unix:///var/run/docker.sock"
docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234"
force_pull: false
valid_volumes:
- /var/run/docker.sock
valid_volumes: []
default_image: "registry.neuralplatform.ai/ci-base:latest"
extra_hosts:
- "gitea.git.svc.cluster.local:10.43.1.53"
@@ -135,14 +139,15 @@ spec:
image: registry.neuralplatform.ai/ci-base:latest
workingDir: /data
command: ["act_runner", "daemon", "--config", "/data/config.yaml"]
env:
- name: DOCKER_HOST
value: "tcp://buildkitd.ci.svc.cluster.local:1234"
envFrom:
- secretRef:
name: neuron-technologies-runner-secret
volumeMounts:
- name: data
mountPath: /data
- name: docker-sock
mountPath: /var/run/docker.sock
resources:
requests:
memory: 512Mi
@@ -153,7 +158,3 @@ spec:
volumes:
- name: data
emptyDir: {}
- name: docker-sock
hostPath:
path: /var/run/docker.sock
type: Socket
servers/legion/apps/kube-prometheus-stack.yaml
+8 -1
View File
@@ -8,7 +8,7 @@ spec:
source:
repoURL: https://prometheus-community.github.io/helm-charts
chart: kube-prometheus-stack
targetRevision: "*"
targetRevision: ">=60.0.0, <70.0.0"
helm:
values: |
grafana:
@@ -37,6 +37,13 @@ spec:
cert-manager.io/cluster-issuer: letsencrypt-prod
prometheus:
prometheusSpec:
# Discover ServiceMonitors and PrometheusRules from ALL namespaces.
# Without this, Prometheus only sees resources in the monitoring namespace.
serviceMonitorSelectorNilUsesHelmValues: false
serviceMonitorNamespaceSelector: {}
serviceMonitorSelector: {}
ruleNamespaceSelector: {}
podMonitorNamespaceSelector: {}
storageSpec:
volumeClaimTemplate:
spec:
servers/legion/apps/loki.yaml
+1 -1
View File
@@ -8,7 +8,7 @@ spec:
source:
repoURL: https://grafana.github.io/helm-charts
chart: loki
targetRevision: "*"
targetRevision: ">=6.0.0, <7.0.0"
helm:
values: |
loki:
servers/legion/apps/neuron-prod-project.yaml
+89
View File
@@ -0,0 +1,89 @@
---
# ArgoCD AppProject — neuron-prod
# Restricts which repos can deploy to which namespaces.
# Prevents misconfigured apps from deploying cluster-admin RBAC or
# accidentally targeting personal/hobby namespaces.
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: neuron-prod
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
description: "Neuron Technologies production services"
# Only allow deployments sourced from the infrastructure repo.
sourceRepos:
- "http://10.43.1.53:3000/will/infrastructure.git"
- "https://code.forgejo.org"
- "registry.neuralplatform.ai"
# Only allow deploying to neuron-prod and platform namespaces.
destinations:
- server: https://kubernetes.default.svc
namespace: neuron-prod
- server: https://kubernetes.default.svc
namespace: platform
# Cluster-scoped resources this project may NOT create.
clusterResourceBlacklist:
- group: rbac.authorization.k8s.io
kind: ClusterRole
- group: rbac.authorization.k8s.io
kind: ClusterRoleBinding
- group: ""
kind: Namespace
# Namespace-scoped resources this project may create.
namespaceResourceWhitelist:
- group: "apps"
kind: Deployment
- group: "apps"
kind: ReplicaSet
- group: "batch"
kind: CronJob
- group: "batch"
kind: Job
- group: ""
kind: Service
- group: ""
kind: ConfigMap
- group: ""
kind: Secret
- group: ""
kind: PersistentVolumeClaim
- group: ""
kind: ServiceAccount
- group: "networking.k8s.io"
kind: Ingress
- group: "networking.k8s.io"
kind: NetworkPolicy
- group: "policy"
kind: PodDisruptionBudget
- group: "autoscaling"
kind: HorizontalPodAutoscaler
- group: "traefik.io"
kind: IngressRoute
- group: "external-secrets.io"
kind: ExternalSecret
- group: "monitoring.coreos.com"
kind: ServiceMonitor
- group: "monitoring.coreos.com"
kind: PrometheusRule
# Sync windows: only allow automated syncs during low-traffic hours.
# Manual syncs allowed any time.
syncWindows:
- kind: allow
schedule: "0 2-6 * * *" # 2am-6am UTC (off-peak)
duration: 4h
applications:
- "*"
manualSync: true
- kind: allow
schedule: "* * * * *"
duration: "1m"
manualSync: true # Always allow manual deploys
applications:
- "*"
servers/legion/apps/postgres.yaml
+1 -1
View File
@@ -9,7 +9,7 @@ spec:
# Helm chart from Docker Hub OCI registry
- repoURL: registry-1.docker.io/bitnamicharts
chart: postgresql
targetRevision: "*"
targetRevision: ">=16.0.0, <17.0.0"
helm:
values: |
auth:
servers/legion/apps/redis.yaml
+1 -1
View File
@@ -8,7 +8,7 @@ spec:
source:
repoURL: registry-1.docker.io/bitnamicharts
chart: redis
targetRevision: "*"
targetRevision: ">=20.0.0, <21.0.0"
helm:
values: |
auth:
servers/legion/apps/tempo.yaml
+1 -1
View File
@@ -8,7 +8,7 @@ spec:
source:
repoURL: https://grafana.github.io/helm-charts
chart: tempo
targetRevision: "*"
targetRevision: ">=1.0.0, <2.0.0"
helm:
values: |
tempo: