Will Anderson
93358505fc
Security: - Drop ALL capabilities, enforce non-root, RuntimeDefault seccomp on neuron-mcp, neuron-rest, neuron-marketing pods - Add startup probes (150s window for JVM) so liveness doesn't fire early - Replace docker-sock hostPath with BuildKit rootless TCP endpoint (moby/buildkit:v0.19.0-rootless) — removes node root access from CI - Document full ESO AppRole migration path in cluster-secret-store.yaml Autoscaling & availability: - HPAs on mcp (1–6), rest (1–4), marketing (2–8) at 65–70% CPU - PodDisruptionBudgets (minAvailable: 1) on all three services - NetworkPolicy: default-deny-all in neuron-prod, explicit allow rules for Traefik ingress, intra-namespace, and egress to DNS/platform/vault Observability: - ServiceMonitors for mcp, rest, marketing (cross-namespace enabled in kube-prometheus-stack with serviceMonitorSelectorNilUsesHelmValues:false) - PrometheusRules: high error rate, high latency, crash loops, replica shortage, Postgres down/connections, backup failure, backup staleness Chart version pinning: - kube-prometheus-stack, loki, tempo, redis, alloy, postgres — all pinned to major-version ranges to block silent breaking upgrades Backup hardening: - restic:latest → restic:0.17.3 (deterministic image) - Weekly backup-verify CronJob: restores latest snapshot and validates SQL dump structure (≥5 CREATE TABLE, pg_dump header check) ArgoCD: - neuron-prod AppProject: scopes deploys to neuron-prod + platform ns, blacklists ClusterRole/ClusterRoleBinding/Namespace creation, automated sync window 2–6am UTC, manual always allowed
114 lines
4.0 KiB
YAML
114 lines
4.0 KiB
YAML
apiVersion: argoproj.io/v1alpha1
|
|
kind: Application
|
|
metadata:
|
|
name: kube-prometheus-stack
|
|
namespace: argocd
|
|
spec:
|
|
project: default
|
|
source:
|
|
repoURL: https://prometheus-community.github.io/helm-charts
|
|
chart: kube-prometheus-stack
|
|
targetRevision: ">=60.0.0, <70.0.0"
|
|
helm:
|
|
values: |
|
|
grafana:
|
|
admin:
|
|
existingSecret: grafana-admin-secret
|
|
passwordKey: admin-password
|
|
persistence:
|
|
enabled: true
|
|
size: 5Gi
|
|
sidecar:
|
|
datasources:
|
|
defaultDatasourceEnabled: false
|
|
initChownData:
|
|
enabled: false
|
|
ingress:
|
|
enabled: true
|
|
ingressClassName: traefik
|
|
hosts:
|
|
- grafana.neuralplatform.ai
|
|
tls:
|
|
- secretName: grafana-tls
|
|
hosts:
|
|
- grafana.neuralplatform.ai
|
|
annotations:
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
cert-manager.io/cluster-issuer: letsencrypt-prod
|
|
prometheus:
|
|
prometheusSpec:
|
|
# Discover ServiceMonitors and PrometheusRules from ALL namespaces.
|
|
# Without this, Prometheus only sees resources in the monitoring namespace.
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
serviceMonitorNamespaceSelector: {}
|
|
serviceMonitorSelector: {}
|
|
ruleNamespaceSelector: {}
|
|
podMonitorNamespaceSelector: {}
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
kubeEtcd:
|
|
enabled: false
|
|
kubeControllerManager:
|
|
enabled: false
|
|
kubeScheduler:
|
|
enabled: false
|
|
kubeProxy:
|
|
enabled: false
|
|
alertmanager:
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
inhibit_rules:
|
|
- source_matchers: [severity="critical"]
|
|
target_matchers: [severity=~"warning|info"]
|
|
equal: [namespace, alertname]
|
|
- source_matchers: [severity="warning"]
|
|
target_matchers: [severity="info"]
|
|
equal: [namespace, alertname]
|
|
receivers:
|
|
- name: "null"
|
|
- name: slack-alerts
|
|
slack_configs:
|
|
- channel: "#infrastructure-alerts"
|
|
api_url: "https://slack.com/api/chat.postMessage"
|
|
http_config:
|
|
authorization:
|
|
type: Bearer
|
|
credentials_file: /etc/alertmanager/secrets/slack-bot-token/bot_token
|
|
send_resolved: true
|
|
title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
text: |
|
|
{{ range .Alerts }}
|
|
*{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ else }}{{ .Labels.alertname }}{{ end }}*{{ if .Labels.severity }} · {{ .Labels.severity }}{{ end }}
|
|
{{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
|
|
{{ end }}
|
|
footer: "Legion · neuralplatform.ai"
|
|
route:
|
|
group_by: [namespace, alertname]
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 4h
|
|
receiver: slack-alerts
|
|
routes:
|
|
- matchers: [alertname="Watchdog"]
|
|
receiver: "null"
|
|
- matchers: [severity="info"]
|
|
receiver: "null"
|
|
alertmanagerSpec:
|
|
secrets:
|
|
- slack-bot-token
|
|
destination:
|
|
server: https://kubernetes.default.svc
|
|
namespace: monitoring
|
|
syncPolicy:
|
|
automated:
|
|
prune: true
|
|
selfHeal: true
|
|
syncOptions:
|
|
- CreateNamespace=true
|
|
- ServerSideApply=true
|