Harden prod: security, autoscaling, observability, BuildKit CI
Security: - Drop ALL capabilities, enforce non-root, RuntimeDefault seccomp on neuron-mcp, neuron-rest, neuron-marketing pods - Add startup probes (150s window for JVM) so liveness doesn't fire early - Replace docker-sock hostPath with BuildKit rootless TCP endpoint (moby/buildkit:v0.19.0-rootless) — removes node root access from CI - Document full ESO AppRole migration path in cluster-secret-store.yaml Autoscaling & availability: - HPAs on mcp (1–6), rest (1–4), marketing (2–8) at 65–70% CPU - PodDisruptionBudgets (minAvailable: 1) on all three services - NetworkPolicy: default-deny-all in neuron-prod, explicit allow rules for Traefik ingress, intra-namespace, and egress to DNS/platform/vault Observability: - ServiceMonitors for mcp, rest, marketing (cross-namespace enabled in kube-prometheus-stack with serviceMonitorSelectorNilUsesHelmValues:false) - PrometheusRules: high error rate, high latency, crash loops, replica shortage, Postgres down/connections, backup failure, backup staleness Chart version pinning: - kube-prometheus-stack, loki, tempo, redis, alloy, postgres — all pinned to major-version ranges to block silent breaking upgrades Backup hardening: - restic:latest → restic:0.17.3 (deterministic image) - Weekly backup-verify CronJob: restores latest snapshot and validates SQL dump structure (≥5 CREATE TABLE, pg_dump header check) ArgoCD: - neuron-prod AppProject: scopes deploys to neuron-prod + platform ns, blacklists ClusterRole/ClusterRoleBinding/Namespace creation, automated sync window 2–6am UTC, manual always allowed
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
---
|
||||
# ArgoCD AppProject — neuron-prod
|
||||
# Restricts which repos can deploy to which namespaces.
|
||||
# Prevents misconfigured apps from deploying cluster-admin RBAC or
|
||||
# accidentally targeting personal/hobby namespaces.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: neuron-prod
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
description: "Neuron Technologies production services"
|
||||
|
||||
# Only allow deployments sourced from the infrastructure repo.
|
||||
sourceRepos:
|
||||
- "http://10.43.1.53:3000/will/infrastructure.git"
|
||||
- "https://code.forgejo.org"
|
||||
- "registry.neuralplatform.ai"
|
||||
|
||||
# Only allow deploying to neuron-prod and platform namespaces.
|
||||
destinations:
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: neuron-prod
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: platform
|
||||
|
||||
# Cluster-scoped resources this project may NOT create.
|
||||
clusterResourceBlacklist:
|
||||
- group: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
- group: rbac.authorization.k8s.io
|
||||
kind: ClusterRoleBinding
|
||||
- group: ""
|
||||
kind: Namespace
|
||||
|
||||
# Namespace-scoped resources this project may create.
|
||||
namespaceResourceWhitelist:
|
||||
- group: "apps"
|
||||
kind: Deployment
|
||||
- group: "apps"
|
||||
kind: ReplicaSet
|
||||
- group: "batch"
|
||||
kind: CronJob
|
||||
- group: "batch"
|
||||
kind: Job
|
||||
- group: ""
|
||||
kind: Service
|
||||
- group: ""
|
||||
kind: ConfigMap
|
||||
- group: ""
|
||||
kind: Secret
|
||||
- group: ""
|
||||
kind: PersistentVolumeClaim
|
||||
- group: ""
|
||||
kind: ServiceAccount
|
||||
- group: "networking.k8s.io"
|
||||
kind: Ingress
|
||||
- group: "networking.k8s.io"
|
||||
kind: NetworkPolicy
|
||||
- group: "policy"
|
||||
kind: PodDisruptionBudget
|
||||
- group: "autoscaling"
|
||||
kind: HorizontalPodAutoscaler
|
||||
- group: "traefik.io"
|
||||
kind: IngressRoute
|
||||
- group: "external-secrets.io"
|
||||
kind: ExternalSecret
|
||||
- group: "monitoring.coreos.com"
|
||||
kind: ServiceMonitor
|
||||
- group: "monitoring.coreos.com"
|
||||
kind: PrometheusRule
|
||||
|
||||
# Sync windows: only allow automated syncs during low-traffic hours.
|
||||
# Manual syncs allowed any time.
|
||||
syncWindows:
|
||||
- kind: allow
|
||||
schedule: "0 2-6 * * *" # 2am-6am UTC (off-peak)
|
||||
duration: 4h
|
||||
applications:
|
||||
- "*"
|
||||
manualSync: true
|
||||
- kind: allow
|
||||
schedule: "* * * * *"
|
||||
duration: "1m"
|
||||
manualSync: true # Always allow manual deploys
|
||||
applications:
|
||||
- "*"
|
||||
Reference in New Issue
Block a user