diff --git a/servers/gcp/artifact-registry.tf b/servers/gcp/artifact-registry.tf new file mode 100644 index 0000000..2791ccd --- /dev/null +++ b/servers/gcp/artifact-registry.tf @@ -0,0 +1,96 @@ +# ── Artifact Registry ───────────────────────────────────────────────────────── +# One repository per service. Images are pushed here from Legion CI runners +# (the only compiled artifacts that cross the boundary to GCP). +# Source code and intelligence stay on Legion — only the built container +# crosses out. + +resource "google_artifact_registry_repository" "marketing" { + location = "us-central1" + repository_id = "neuron-marketing" + description = "Marketing site (Next.js) Docker images" + format = "DOCKER" + project = var.project_id + + cleanup_policies { + id = "keep-last-10" + action = "KEEP" + most_recent_versions { + keep_count = 10 + } + } +} + +resource "google_artifact_registry_repository" "accounts" { + location = "us-central1" + repository_id = "neuron-accounts" + description = "Accounts service (Go) Docker images" + format = "DOCKER" + project = var.project_id + + cleanup_policies { + id = "keep-last-10" + action = "KEEP" + most_recent_versions { + keep_count = 10 + } + } +} + +resource "google_artifact_registry_repository" "api" { + location = "us-central1" + repository_id = "neuron-api" + description = "REST API (neuron-rest, Go) Docker images" + format = "DOCKER" + project = var.project_id + + cleanup_policies { + id = "keep-last-10" + action = "KEEP" + most_recent_versions { + keep_count = 10 + } + } +} + +# ── IAM: CI runner pushes images ────────────────────────────────────────────── +# The CI service account (from Legion's Gitea runner) needs Artifact Registry +# writer access so it can push compiled images after each build. + +resource "google_service_account" "ci_pusher" { + account_id = "neuron-ci-pusher" + display_name = "Neuron CI Image Pusher" + description = "Legion CI runner uses this SA to push compiled images to Artifact Registry" + project = var.project_id +} + +resource "google_artifact_registry_repository_iam_member" "ci_marketing" { + project = var.project_id + location = "us-central1" + repository = google_artifact_registry_repository.marketing.name + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.ci_pusher.email}" +} + +resource "google_artifact_registry_repository_iam_member" "ci_accounts" { + project = var.project_id + location = "us-central1" + repository = google_artifact_registry_repository.accounts.name + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.ci_pusher.email}" +} + +resource "google_artifact_registry_repository_iam_member" "ci_api" { + project = var.project_id + location = "us-central1" + repository = google_artifact_registry_repository.api.name + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.ci_pusher.email}" +} + +# ── CI pusher JSON key (download once, store in Vault/Secret Manager) ───────── +# After `terraform apply`, create a key and save it to GCP Secret Manager: +# gcloud iam service-accounts keys create /tmp/ci-pusher.json \ +# --iam-account=$(terraform output -raw ci_pusher_email) +# gcloud secrets create ci-gcp-sa-key --data-file=/tmp/ci-pusher.json +# rm /tmp/ci-pusher.json +# Then set it as a repo secret in Gitea: GCP_SA_KEY diff --git a/servers/gcp/bootstrap.sh b/servers/gcp/bootstrap.sh new file mode 100755 index 0000000..297a7ae --- /dev/null +++ b/servers/gcp/bootstrap.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# ── GCP Bootstrap — run once after terraform apply ──────────────────────────── +# Sets up secrets and credentials that Terraform can't create automatically. +# +# Prerequisites: +# - GOOGLE_APPLICATION_CREDENTIALS or `gcloud auth application-default login` +# - GCP project access +# - Vault running (for reading existing secrets) +# - terraform apply has run successfully (creates SA, secrets) +# +# Usage: +# cd servers/gcp +# terraform apply +# bash bootstrap.sh + +set -euo pipefail + +PROJECT="neuron-785695" +VAULT_ADDR="${VAULT_ADDR:-https://vault.neuralplatform.ai}" + +echo "==> Bootstrapping GCP secrets and CI credentials" + +# ── 1. Populate Stripe secrets in GCP Secret Manager ───────────────────────── +echo "--> Pulling Stripe secrets from Vault..." +MARKETING_SECRETS=$(curl -sf \ + -H "X-Vault-Token: ${VAULT_TOKEN}" \ + "${VAULT_ADDR}/v1/secret/data/neuron-technologies/marketing") + +populate_secret() { + local secret_id="$1" + local vault_key="$2" + local value + value=$(echo "${MARKETING_SECRETS}" | python3 -c " +import json,sys; d=json.load(sys.stdin)['data']['data']; print(d.get('${vault_key}','')) +") + if [ -z "${value}" ]; then + echo " WARN: vault key '${vault_key}' not found — skipping ${secret_id}" + return + fi + # Check if secret already has a version + if gcloud secrets versions list "${secret_id}" --project="${PROJECT}" \ + --format="value(name)" 2>/dev/null | grep -q .; then + echo " SKIP: ${secret_id} already has a version" + else + echo "${value}" | gcloud secrets versions add "${secret_id}" \ + --project="${PROJECT}" \ + --data-file=- + echo " OK: ${secret_id}" + fi +} + +populate_secret "stripe-secret-key" "stripe_secret_key" +populate_secret "stripe-webhook-secret" "stripe_webhook_secret" +populate_secret "stripe-price-professional" "stripe_price_professional" +populate_secret "stripe-price-founding" "stripe_price_founding" + +# ── 2. License admin token ──────────────────────────────────────────────────── +echo "--> Populating license-admin-token..." +LICENSE_TOKEN=$(curl -sf \ + -H "X-Vault-Token: ${VAULT_TOKEN}" \ + "${VAULT_ADDR}/v1/secret/data/neuron-technologies/marketing" \ + | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['data'].get('license_admin_token',''))") +if [ -n "${LICENSE_TOKEN}" ]; then + echo "${LICENSE_TOKEN}" | gcloud secrets versions add "license-admin-token" \ + --project="${PROJECT}" --data-file=- 2>/dev/null || echo " SKIP: already exists" + echo " OK: license-admin-token" +fi + +# ── 3. Create CI service account JSON key ──────────────────────────────────── +CI_SA_EMAIL=$(terraform output -raw ci_pusher_email) +echo "--> Creating CI SA key for: ${CI_SA_EMAIL}" +KEY_FILE="/tmp/neuron-ci-pusher.json" +gcloud iam service-accounts keys create "${KEY_FILE}" \ + --iam-account="${CI_SA_EMAIL}" \ + --project="${PROJECT}" +echo " SA key written to: ${KEY_FILE}" +echo "" +echo " !! Add this as a Gitea secret GCP_SA_KEY in the neuron-technologies/neuron repo:" +echo " tea secret set --repo neuron-technologies/neuron GCP_SA_KEY < ${KEY_FILE}" +echo " Then delete the key file:" +echo " rm ${KEY_FILE}" +echo "" + +# ── 4. Summary ──────────────────────────────────────────────────────────────── +echo "==> Bootstrap complete" +echo "" +echo "Next steps:" +echo " 1. Set Gitea secret: tea secret set --repo neuron-technologies/neuron GCP_SA_KEY < /tmp/neuron-ci-pusher.json" +echo " 2. Delete key file: rm /tmp/neuron-ci-pusher.json" +echo " 3. Point DNS at: $(terraform output -raw prod_lb_ip)" +echo " A records (proxied=true in Cloudflare):" +echo " neurontechnologies.ai → prod_lb_ip" +echo " www.neurontechnologies.ai → prod_lb_ip" +echo " api.neurontechnologies.ai → prod_lb_ip" +echo " accounts.neurontechnologies.ai → prod_lb_ip" +echo " 4. Run a marketing CI build on main to push first image to GCP AR" +echo " 5. Check SSL cert provisioning: gcloud compute ssl-certificates list --project=${PROJECT}" diff --git a/servers/gcp/cloud-run-accounts.tf b/servers/gcp/cloud-run-accounts.tf new file mode 100644 index 0000000..257a8f7 --- /dev/null +++ b/servers/gcp/cloud-run-accounts.tf @@ -0,0 +1,443 @@ +# ── Accounts Service — Cloud Run ────────────────────────────────────────────── +# Handles auth, billing, subscriptions, marketplace. +# Connects to Cloud SQL via built-in Auth Proxy (Unix socket volume mount). +# All three regions share the same Cloud SQL instance (us-central1). + +locals { + accounts_labels = { + "managed-by" = "terraform" + "service" = "neuron-accounts" + } + sql_instance = google_sql_database_instance.main.connection_name +} + +# ── Prod — us-central1 ──────────────────────────────────────────────────────── + +resource "google_cloud_run_v2_service" "accounts_us" { + name = "accounts-prod-us" + location = "us-central1" + project = var.project_id + ingress = "INGRESS_TRAFFIC_ALL" + labels = local.accounts_labels + + template { + service_account = google_service_account.accounts.email + + scaling { + min_instance_count = 1 + max_instance_count = 50 + } + + containers { + image = local.accounts_image + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + cpu_idle = true + } + + env { + name = "ENV" + value = "production" + } + env { + name = "PORT" + value = "8080" + } + + env { + name = "ACCOUNTS_DATABASE_URL" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.accounts_database_url.secret_id + version = "latest" + } + } + } + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.jwt_secret.secret_id + version = "latest" + } + } + } + env { + name = "STRIPE_SECRET_KEY" + value_source { + secret_key_ref { + secret = "stripe-secret-key" + version = "latest" + } + } + } + env { + name = "STRIPE_WEBHOOK_SECRET" + value_source { + secret_key_ref { + secret = "stripe-webhook-secret" + version = "latest" + } + } + } + env { + name = "STRIPE_PRICE_PROFESSIONAL" + value_source { + secret_key_ref { + secret = "stripe-price-professional" + version = "latest" + } + } + } + env { + name = "STRIPE_PRICE_FOUNDING" + value_source { + secret_key_ref { + secret = "stripe-price-founding" + version = "latest" + } + } + } + + ports { + container_port = 8080 + name = "http1" + } + + startup_probe { + http_get { + path = "/health" + port = 8080 + } + initial_delay_seconds = 2 + timeout_seconds = 5 + period_seconds = 5 + failure_threshold = 10 + } + + liveness_probe { + http_get { + path = "/health" + port = 8080 + } + timeout_seconds = 5 + period_seconds = 30 + failure_threshold = 3 + } + + volume_mounts { + name = "cloudsql" + mount_path = "/cloudsql" + } + } + + volumes { + name = "cloudsql" + cloud_sql_instance { + instances = [local.sql_instance] + } + } + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [ + google_project_iam_member.accounts_secret_accessor, + google_project_iam_member.accounts_sql_client, + google_secret_manager_secret_version.accounts_database_url, + google_secret_manager_secret_version.jwt_secret, + ] +} + +# ── Prod — europe-west1 ─────────────────────────────────────────────────────── + +resource "google_cloud_run_v2_service" "accounts_eu" { + name = "accounts-prod-eu" + location = "europe-west1" + project = var.project_id + ingress = "INGRESS_TRAFFIC_ALL" + labels = local.accounts_labels + + template { + service_account = google_service_account.accounts.email + + scaling { + min_instance_count = 1 + max_instance_count = 50 + } + + containers { + image = local.accounts_image + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + cpu_idle = true + } + + env { + name = "ENV" + value = "production" + } + env { + name = "PORT" + value = "8080" + } + + env { + name = "ACCOUNTS_DATABASE_URL" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.accounts_database_url.secret_id + version = "latest" + } + } + } + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.jwt_secret.secret_id + version = "latest" + } + } + } + env { + name = "STRIPE_SECRET_KEY" + value_source { + secret_key_ref { + secret = "stripe-secret-key" + version = "latest" + } + } + } + env { + name = "STRIPE_WEBHOOK_SECRET" + value_source { + secret_key_ref { + secret = "stripe-webhook-secret" + version = "latest" + } + } + } + env { + name = "STRIPE_PRICE_PROFESSIONAL" + value_source { + secret_key_ref { + secret = "stripe-price-professional" + version = "latest" + } + } + } + env { + name = "STRIPE_PRICE_FOUNDING" + value_source { + secret_key_ref { + secret = "stripe-price-founding" + version = "latest" + } + } + } + + ports { + container_port = 8080 + name = "http1" + } + + startup_probe { + http_get { + path = "/health" + port = 8080 + } + initial_delay_seconds = 2 + timeout_seconds = 5 + period_seconds = 5 + failure_threshold = 10 + } + + liveness_probe { + http_get { + path = "/health" + port = 8080 + } + timeout_seconds = 5 + period_seconds = 30 + failure_threshold = 3 + } + + volume_mounts { + name = "cloudsql" + mount_path = "/cloudsql" + } + } + + volumes { + name = "cloudsql" + cloud_sql_instance { + instances = [local.sql_instance] + } + } + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [ + google_project_iam_member.accounts_secret_accessor, + google_project_iam_member.accounts_sql_client, + ] +} + +# ── Prod — asia-northeast1 ──────────────────────────────────────────────────── + +resource "google_cloud_run_v2_service" "accounts_apac" { + name = "accounts-prod-apac" + location = "asia-northeast1" + project = var.project_id + ingress = "INGRESS_TRAFFIC_ALL" + labels = local.accounts_labels + + template { + service_account = google_service_account.accounts.email + + scaling { + min_instance_count = 1 + max_instance_count = 50 + } + + containers { + image = local.accounts_image + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + cpu_idle = true + } + + env { + name = "ENV" + value = "production" + } + env { + name = "PORT" + value = "8080" + } + + env { + name = "ACCOUNTS_DATABASE_URL" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.accounts_database_url.secret_id + version = "latest" + } + } + } + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.jwt_secret.secret_id + version = "latest" + } + } + } + env { + name = "STRIPE_SECRET_KEY" + value_source { + secret_key_ref { + secret = "stripe-secret-key" + version = "latest" + } + } + } + env { + name = "STRIPE_WEBHOOK_SECRET" + value_source { + secret_key_ref { + secret = "stripe-webhook-secret" + version = "latest" + } + } + } + env { + name = "STRIPE_PRICE_PROFESSIONAL" + value_source { + secret_key_ref { + secret = "stripe-price-professional" + version = "latest" + } + } + } + env { + name = "STRIPE_PRICE_FOUNDING" + value_source { + secret_key_ref { + secret = "stripe-price-founding" + version = "latest" + } + } + } + + ports { + container_port = 8080 + name = "http1" + } + + startup_probe { + http_get { + path = "/health" + port = 8080 + } + initial_delay_seconds = 2 + timeout_seconds = 5 + period_seconds = 5 + failure_threshold = 10 + } + + liveness_probe { + http_get { + path = "/health" + port = 8080 + } + timeout_seconds = 5 + period_seconds = 30 + failure_threshold = 3 + } + + volume_mounts { + name = "cloudsql" + mount_path = "/cloudsql" + } + } + + volumes { + name = "cloudsql" + cloud_sql_instance { + instances = [local.sql_instance] + } + } + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [ + google_project_iam_member.accounts_secret_accessor, + google_project_iam_member.accounts_sql_client, + ] +} diff --git a/servers/gcp/cloud-run-api.tf b/servers/gcp/cloud-run-api.tf new file mode 100644 index 0000000..d956916 --- /dev/null +++ b/servers/gcp/cloud-run-api.tf @@ -0,0 +1,308 @@ +# ── REST API (neuron-rest) — Cloud Run ──────────────────────────────────────── +# Stateless API layer. Validates JWTs from accounts service. +# No database connection — reads user context from accounts service. +# cpu_idle=false: always-hot, no cold-start latency for API calls. + +locals { + api_labels = { + "managed-by" = "terraform" + "service" = "neuron-api" + } +} + +# ── Prod — us-central1 ──────────────────────────────────────────────────────── + +resource "google_cloud_run_v2_service" "api_us" { + name = "api-prod-us" + location = "us-central1" + project = var.project_id + ingress = "INGRESS_TRAFFIC_ALL" + labels = local.api_labels + + template { + service_account = google_service_account.api.email + + scaling { + min_instance_count = 1 + max_instance_count = 100 + } + + containers { + image = local.api_image + + resources { + limits = { + cpu = "2" + memory = "1Gi" + } + cpu_idle = false + } + + env { + name = "ENV" + value = "production" + } + env { + name = "PORT" + value = "8080" + } + env { + name = "ACCOUNTS_SERVICE_URL" + value = "https://accounts.neurontechnologies.ai" + } + + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.jwt_secret.secret_id + version = "latest" + } + } + } + env { + name = "LICENSE_ADMIN_TOKEN" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.license_admin_token.secret_id + version = "latest" + } + } + } + + ports { + container_port = 8080 + name = "http1" + } + + startup_probe { + http_get { + path = "/health" + port = 8080 + } + initial_delay_seconds = 2 + timeout_seconds = 5 + period_seconds = 5 + failure_threshold = 6 + } + + liveness_probe { + http_get { + path = "/health" + port = 8080 + } + timeout_seconds = 5 + period_seconds = 30 + failure_threshold = 3 + } + } + + max_instance_request_concurrency = 1000 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [ + google_project_iam_member.api_secret_accessor, + google_secret_manager_secret_version.jwt_secret, + ] +} + +# ── Prod — europe-west1 ─────────────────────────────────────────────────────── + +resource "google_cloud_run_v2_service" "api_eu" { + name = "api-prod-eu" + location = "europe-west1" + project = var.project_id + ingress = "INGRESS_TRAFFIC_ALL" + labels = local.api_labels + + template { + service_account = google_service_account.api.email + + scaling { + min_instance_count = 1 + max_instance_count = 100 + } + + containers { + image = local.api_image + + resources { + limits = { + cpu = "2" + memory = "1Gi" + } + cpu_idle = false + } + + env { + name = "ENV" + value = "production" + } + env { + name = "PORT" + value = "8080" + } + env { + name = "ACCOUNTS_SERVICE_URL" + value = "https://accounts.neurontechnologies.ai" + } + + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.jwt_secret.secret_id + version = "latest" + } + } + } + env { + name = "LICENSE_ADMIN_TOKEN" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.license_admin_token.secret_id + version = "latest" + } + } + } + + ports { + container_port = 8080 + name = "http1" + } + + startup_probe { + http_get { + path = "/health" + port = 8080 + } + initial_delay_seconds = 2 + timeout_seconds = 5 + period_seconds = 5 + failure_threshold = 6 + } + + liveness_probe { + http_get { + path = "/health" + port = 8080 + } + timeout_seconds = 5 + period_seconds = 30 + failure_threshold = 3 + } + } + + max_instance_request_concurrency = 1000 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_iam_member.api_secret_accessor] +} + +# ── Prod — asia-northeast1 ──────────────────────────────────────────────────── + +resource "google_cloud_run_v2_service" "api_apac" { + name = "api-prod-apac" + location = "asia-northeast1" + project = var.project_id + ingress = "INGRESS_TRAFFIC_ALL" + labels = local.api_labels + + template { + service_account = google_service_account.api.email + + scaling { + min_instance_count = 1 + max_instance_count = 100 + } + + containers { + image = local.api_image + + resources { + limits = { + cpu = "2" + memory = "1Gi" + } + cpu_idle = false + } + + env { + name = "ENV" + value = "production" + } + env { + name = "PORT" + value = "8080" + } + env { + name = "ACCOUNTS_SERVICE_URL" + value = "https://accounts.neurontechnologies.ai" + } + + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.jwt_secret.secret_id + version = "latest" + } + } + } + env { + name = "LICENSE_ADMIN_TOKEN" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.license_admin_token.secret_id + version = "latest" + } + } + } + + ports { + container_port = 8080 + name = "http1" + } + + startup_probe { + http_get { + path = "/health" + port = 8080 + } + initial_delay_seconds = 2 + timeout_seconds = 5 + period_seconds = 5 + failure_threshold = 6 + } + + liveness_probe { + http_get { + path = "/health" + port = 8080 + } + timeout_seconds = 5 + period_seconds = 30 + failure_threshold = 3 + } + } + + max_instance_request_concurrency = 1000 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_iam_member.api_secret_accessor] +} diff --git a/servers/gcp/cloud-sql.tf b/servers/gcp/cloud-sql.tf new file mode 100644 index 0000000..f4c42eb --- /dev/null +++ b/servers/gcp/cloud-sql.tf @@ -0,0 +1,150 @@ +# ── Cloud SQL — PostgreSQL 15 ───────────────────────────────────────────────── +# Single instance in us-central1 for the accounts service. +# Cloud Run services connect via the built-in Cloud SQL Auth Proxy +# (no direct IP exposure, encrypted, IAM-authenticated). +# +# Sizing: db-g1-small (1 shared vCPU, 1.7 GB) for launch. Scale up to +# db-n1-standard-2 once traffic warrants it — zero-downtime restart. + +resource "google_sql_database_instance" "main" { + name = "neuron-prod-pg15" + database_version = "POSTGRES_15" + region = "us-central1" + project = var.project_id + + # Prevent accidental deletion via terraform destroy + deletion_protection = true + + settings { + tier = "db-g1-small" + availability_type = "ZONAL" # Upgrade to REGIONAL (HA) once > 1k users + + backup_configuration { + enabled = true + point_in_time_recovery_enabled = true + start_time = "03:00" # 3am UTC — before backup window on Legion + transaction_log_retention_days = 7 + backup_retention_settings { + retained_backups = 14 + } + } + + maintenance_window { + day = 7 # Sunday + hour = 4 # 4am UTC + update_track = "stable" + } + + ip_configuration { + # No public IP — Cloud Run uses the Auth Proxy via private service connect + # Flip ipv4_enabled=true + authorized_networks if you ever need direct access + # for migrations/seeding from a bastion. + ipv4_enabled = true # Required for Cloud Run Auth Proxy (until VPC SC configured) + ssl_mode = "ENCRYPTED_ONLY" + } + + database_flags { + name = "max_connections" + value = "100" # Cloud Run can burst many instances; cap connections early + } + + database_flags { + name = "log_min_duration_statement" + value = "500" # Log queries taking > 500ms + } + + insights_config { + query_insights_enabled = true + query_string_length = 1024 + record_application_tags = true + record_client_address = false + } + } +} + +# ── Databases ───────────────────────────────────────────────────────────────── + +resource "google_sql_database" "accounts" { + name = "accounts" + instance = google_sql_database_instance.main.name + project = var.project_id +} + +# ── Users ───────────────────────────────────────────────────────────────────── +# Passwords are stored in Secret Manager. The accounts service reads them +# at runtime via the ACCOUNTS_DATABASE_URL secret. +# Cloud IAM DB users are the preferred path long-term (no password rotation); +# using password auth here for compatibility with pgx/stdlib. + +resource "google_sql_user" "accounts" { + name = "accounts" + instance = google_sql_database_instance.main.name + project = var.project_id + password = random_password.accounts_db.result +} + +resource "random_password" "accounts_db" { + length = 32 + special = true +} + +# ── Secret Manager — Database URL ───────────────────────────────────────────── +# Stored as a full DSN so the app just reads one env var. +# Uses /cloudsql/ Unix socket path — Cloud Run Auth Proxy mounts it there. + +resource "google_secret_manager_secret" "accounts_database_url" { + secret_id = "accounts-database-url" + project = var.project_id + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "accounts_database_url" { + secret = google_secret_manager_secret.accounts_database_url.id + secret_data = "host=/cloudsql/${google_sql_database_instance.main.connection_name} user=accounts password=${random_password.accounts_db.result} dbname=accounts sslmode=disable" +} + +# ── Secret Manager — JWT signing key ───────────────────────────────────────── +# Shared between accounts (issues tokens) and neuron-rest (validates tokens). + +resource "random_password" "jwt_secret" { + length = 64 + special = false # URL-safe; used as HMAC key +} + +resource "google_secret_manager_secret" "jwt_secret" { + secret_id = "jwt-signing-key" + project = var.project_id + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "jwt_secret" { + secret = google_secret_manager_secret.jwt_secret.id + secret_data = random_password.jwt_secret.result +} + +# ── Secret Manager — License admin token ───────────────────────────────────── +# Used by neuron-rest to authorize license validation calls. +# Currently held in Vault under neuron-technologies/marketing.license_admin_token. +# Create this secret manually from Vault: +# gcloud secrets create license-admin-token --data-file=<(vault kv get \ +# -field=license_admin_token secret/neuron-technologies/marketing) + +resource "google_secret_manager_secret" "license_admin_token" { + secret_id = "license-admin-token" + project = var.project_id + + replication { + auto {} + } + # Version populated manually or via bootstrap script — not managed by Terraform + # to avoid exposing the value in state. + lifecycle { + ignore_changes = [id] + } +} diff --git a/servers/gcp/load-balancer-backends.tf b/servers/gcp/load-balancer-backends.tf new file mode 100644 index 0000000..a7ab25e --- /dev/null +++ b/servers/gcp/load-balancer-backends.tf @@ -0,0 +1,119 @@ +# ── Serverless NEGs — Accounts ──────────────────────────────────────────────── + +resource "google_compute_region_network_endpoint_group" "accounts_us" { + name = "accounts-neg-us" + network_endpoint_type = "SERVERLESS" + region = "us-central1" + project = var.project_id + cloud_run { service = google_cloud_run_v2_service.accounts_us.name } +} + +resource "google_compute_region_network_endpoint_group" "accounts_eu" { + name = "accounts-neg-eu" + network_endpoint_type = "SERVERLESS" + region = "europe-west1" + project = var.project_id + cloud_run { service = google_cloud_run_v2_service.accounts_eu.name } +} + +resource "google_compute_region_network_endpoint_group" "accounts_apac" { + name = "accounts-neg-apac" + network_endpoint_type = "SERVERLESS" + region = "asia-northeast1" + project = var.project_id + cloud_run { service = google_cloud_run_v2_service.accounts_apac.name } +} + +# ── Serverless NEGs — API ───────────────────────────────────────────────────── + +resource "google_compute_region_network_endpoint_group" "api_us" { + name = "api-neg-us" + network_endpoint_type = "SERVERLESS" + region = "us-central1" + project = var.project_id + cloud_run { service = google_cloud_run_v2_service.api_us.name } +} + +resource "google_compute_region_network_endpoint_group" "api_eu" { + name = "api-neg-eu" + network_endpoint_type = "SERVERLESS" + region = "europe-west1" + project = var.project_id + cloud_run { service = google_cloud_run_v2_service.api_eu.name } +} + +resource "google_compute_region_network_endpoint_group" "api_apac" { + name = "api-neg-apac" + network_endpoint_type = "SERVERLESS" + region = "asia-northeast1" + project = var.project_id + cloud_run { service = google_cloud_run_v2_service.api_apac.name } +} + +# ── Backend Service — Accounts ──────────────────────────────────────────────── + +resource "google_compute_backend_service" "accounts" { + name = "accounts-backend-prod" + project = var.project_id + load_balancing_scheme = "EXTERNAL_MANAGED" + protocol = "HTTPS" + timeout_sec = 30 + + security_policy = google_compute_security_policy.marketing.self_link + + # No CDN for accounts — responses are user-specific and must not be cached + enable_cdn = false + + backend { group = google_compute_region_network_endpoint_group.accounts_us.self_link } + backend { group = google_compute_region_network_endpoint_group.accounts_eu.self_link } + backend { group = google_compute_region_network_endpoint_group.accounts_apac.self_link } + + log_config { + enable = true + sample_rate = 1.0 + } +} + +# ── Backend Service — API ───────────────────────────────────────────────────── + +resource "google_compute_backend_service" "api" { + name = "api-backend-prod" + project = var.project_id + load_balancing_scheme = "EXTERNAL_MANAGED" + protocol = "HTTPS" + timeout_sec = 60 # API calls may be longer (AI inference routing) + + security_policy = google_compute_security_policy.marketing.self_link + + enable_cdn = false # API responses are per-request; no CDN benefit + + backend { group = google_compute_region_network_endpoint_group.api_us.self_link } + backend { group = google_compute_region_network_endpoint_group.api_eu.self_link } + backend { group = google_compute_region_network_endpoint_group.api_apac.self_link } + + log_config { + enable = true + sample_rate = 1.0 + } +} + +# ── SSL Certs — accounts and api subdomains ─────────────────────────────────── +# Added to the existing prod HTTPS proxy alongside the marketing cert. +# DNS must point to the same prod global IP (marketing-ip-prod) for +# provisioning to succeed. + +resource "google_compute_managed_ssl_certificate" "accounts" { + name = "accounts-cert-prod" + project = var.project_id + managed { + domains = ["accounts.neurontechnologies.ai"] + } +} + +resource "google_compute_managed_ssl_certificate" "api" { + name = "api-cert-prod" + project = var.project_id + managed { + domains = ["api.neurontechnologies.ai"] + } +} diff --git a/servers/gcp/load-balancer.tf b/servers/gcp/load-balancer.tf index 9a6c345..0aa21aa 100644 --- a/servers/gcp/load-balancer.tf +++ b/servers/gcp/load-balancer.tf @@ -223,21 +223,59 @@ resource "google_compute_managed_ssl_certificate" "stage" { } } -# ── Prod URL Map ────────────────────────────────────────────────────────────── +# ── Prod URL Map — host-based routing ──────────────────────────────────────── +# One global IP handles all three services via Host header routing. +# - neurontechnologies.ai / www. → marketing (default) +# - api.neurontechnologies.ai → API backend +# - accounts.neurontechnologies.ai → accounts backend resource "google_compute_url_map" "prod" { name = "marketing-urlmap-prod" project = var.project_id default_service = google_compute_backend_service.prod.self_link + + host_rule { + hosts = ["neurontechnologies.ai", "www.neurontechnologies.ai"] + path_matcher = "marketing" + } + + host_rule { + hosts = ["api.neurontechnologies.ai"] + path_matcher = "api" + } + + host_rule { + hosts = ["accounts.neurontechnologies.ai"] + path_matcher = "accounts" + } + + path_matcher { + name = "marketing" + default_service = google_compute_backend_service.prod.self_link + } + + path_matcher { + name = "api" + default_service = google_compute_backend_service.api.self_link + } + + path_matcher { + name = "accounts" + default_service = google_compute_backend_service.accounts.self_link + } } # ── Prod HTTPS Target Proxy ─────────────────────────────────────────────────── resource "google_compute_target_https_proxy" "prod" { - name = "marketing-https-proxy-prod" - project = var.project_id - url_map = google_compute_url_map.prod.self_link - ssl_certificates = [google_compute_managed_ssl_certificate.prod.self_link] + name = "marketing-https-proxy-prod" + project = var.project_id + url_map = google_compute_url_map.prod.self_link + ssl_certificates = [ + google_compute_managed_ssl_certificate.prod.self_link, + google_compute_managed_ssl_certificate.accounts.self_link, + google_compute_managed_ssl_certificate.api.self_link, + ] } # ── Prod HTTP → HTTPS redirect ──────────────────────────────────────────────── diff --git a/servers/gcp/main.tf b/servers/gcp/main.tf index 702fab2..47a109c 100644 --- a/servers/gcp/main.tf +++ b/servers/gcp/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/google-beta" version = "~> 5.0" } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } } backend "s3" { diff --git a/servers/gcp/outputs.tf b/servers/gcp/outputs.tf index 5613db7..1ab4933 100644 --- a/servers/gcp/outputs.tf +++ b/servers/gcp/outputs.tf @@ -1,44 +1,96 @@ # ── Outputs ─────────────────────────────────────────────────────────────────── -# After `terraform apply`, use these IPs to update Cloudflare DNS A records. +# After `terraform apply`, use prod_lb_ip for Cloudflare DNS A records. # -# Prod: neurontechnologies.ai + www.neurontechnologies.ai → prod_lb_ip -# Stage: stage.neurontechnologies.ai → stage_lb_ip +# All three services share the same global anycast IP (prod_lb_ip). +# Set these A records in Cloudflare (proxied=true for CDN + DDoS): +# neurontechnologies.ai → prod_lb_ip +# www.neurontechnologies.ai → prod_lb_ip +# api.neurontechnologies.ai → prod_lb_ip +# accounts.neurontechnologies.ai → prod_lb_ip # -# In the Legion Terraform (dns-neurontechnologies.tf), replace the tunnel CNAMEs -# for neurontechnologies.ai, www, and stage with A records pointing to these IPs. -# Set proxied=true in Cloudflare to keep CDN/DDoS protection layered in front. +# Stage: stage.neurontechnologies.ai → stage_lb_ip (separate IP) output "prod_lb_ip" { - description = "Global anycast IP for the production load balancer (neurontechnologies.ai + www)" + description = "Global anycast IP for all prod services (marketing, accounts, api)" value = google_compute_global_address.prod.address } output "stage_lb_ip" { - description = "Global anycast IP for the staging load balancer (stage.neurontechnologies.ai)" + description = "Global anycast IP for the staging load balancer" value = google_compute_global_address.stage.address } output "prod_ssl_cert_name" { - description = "Name of the Google-managed SSL cert for prod (check provisioning status in GCP console)" + description = "Marketing SSL cert (check provisioning status in GCP console)" value = google_compute_managed_ssl_certificate.prod.name } +output "accounts_ssl_cert_name" { + description = "Accounts SSL cert (check provisioning status in GCP console)" + value = google_compute_managed_ssl_certificate.accounts.name +} + +output "api_ssl_cert_name" { + description = "API SSL cert (check provisioning status in GCP console)" + value = google_compute_managed_ssl_certificate.api.name +} + output "stage_ssl_cert_name" { - description = "Name of the Google-managed SSL cert for stage" + description = "Stage SSL cert" value = google_compute_managed_ssl_certificate.stage.name } output "marketing_service_account_email" { - description = "Service account email for Cloud Run services" + description = "Marketing Cloud Run SA" value = google_service_account.marketing.email } +output "accounts_service_account_email" { + description = "Accounts Cloud Run SA" + value = google_service_account.accounts.email +} + +output "api_service_account_email" { + description = "API Cloud Run SA" + value = google_service_account.api.email +} + +output "ci_pusher_email" { + description = "CI SA email — use to create the JSON key for GCP_SA_KEY Gitea secret" + value = google_service_account.ci_pusher.email +} + +output "cloud_sql_connection_name" { + description = "Cloud SQL instance connection name — use in Cloud Run volume mounts" + value = google_sql_database_instance.main.connection_name +} + +output "cloud_sql_instance_ip" { + description = "Cloud SQL public IP (for bastion/migration access — disable when not needed)" + value = google_sql_database_instance.main.public_ip_address +} + output "cloud_run_services" { - description = "Cloud Run service URLs per region" + description = "Cloud Run service URLs per service and region" value = { - prod_us = google_cloud_run_v2_service.prod_us.uri - prod_eu = google_cloud_run_v2_service.prod_eu.uri - prod_apac = google_cloud_run_v2_service.prod_apac.uri - stage = google_cloud_run_v2_service.stage.uri + marketing_us = google_cloud_run_v2_service.prod_us.uri + marketing_eu = google_cloud_run_v2_service.prod_eu.uri + marketing_apac = google_cloud_run_v2_service.prod_apac.uri + marketing_stage = google_cloud_run_v2_service.stage.uri + accounts_us = google_cloud_run_v2_service.accounts_us.uri + accounts_eu = google_cloud_run_v2_service.accounts_eu.uri + accounts_apac = google_cloud_run_v2_service.accounts_apac.uri + api_us = google_cloud_run_v2_service.api_us.uri + api_eu = google_cloud_run_v2_service.api_eu.uri + api_apac = google_cloud_run_v2_service.api_apac.uri + } +} + +output "artifact_registry_urls" { + description = "Docker image base URLs for each service" + value = { + marketing = "us-central1-docker.pkg.dev/${var.project_id}/neuron-marketing/marketing" + accounts = "us-central1-docker.pkg.dev/${var.project_id}/neuron-accounts/accounts" + api = "us-central1-docker.pkg.dev/${var.project_id}/neuron-api/api" } } diff --git a/servers/gcp/service-account.tf b/servers/gcp/service-account.tf index 70fcc90..1c0692e 100644 --- a/servers/gcp/service-account.tf +++ b/servers/gcp/service-account.tf @@ -1,13 +1,27 @@ -# ── Service Account ─────────────────────────────────────────────────────────── +# ── Service Accounts ────────────────────────────────────────────────────────── resource "google_service_account" "marketing" { account_id = "neuron-marketing-sa" display_name = "Neuron Marketing Cloud Run SA" description = "Service account for the neurontechnologies.ai marketing site on Cloud Run" + project = var.project_id } -# ── IAM — Secret Manager accessor ───────────────────────────────────────────── -# Grant the SA access to read all marketing secrets from Secret Manager. +resource "google_service_account" "accounts" { + account_id = "neuron-accounts-sa" + display_name = "Neuron Accounts Cloud Run SA" + description = "Service account for the accounts service (auth, billing, marketplace)" + project = var.project_id +} + +resource "google_service_account" "api" { + account_id = "neuron-api-sa" + display_name = "Neuron REST API Cloud Run SA" + description = "Service account for the neuron-rest API service" + project = var.project_id +} + +# ── IAM — Secret Manager ────────────────────────────────────────────────────── resource "google_project_iam_member" "marketing_secret_accessor" { project = var.project_id @@ -15,9 +29,32 @@ resource "google_project_iam_member" "marketing_secret_accessor" { member = "serviceAccount:${google_service_account.marketing.email}" } -# ── IAM — Public (unauthenticated) invocation for prod services ─────────────── -# Cloud Run v2: allow-unauthenticated is set per-service via google_cloud_run_v2_service_iam_member. +resource "google_project_iam_member" "accounts_secret_accessor" { + project = var.project_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.accounts.email}" +} +resource "google_project_iam_member" "api_secret_accessor" { + project = var.project_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.api.email}" +} + +# ── IAM — Cloud SQL ─────────────────────────────────────────────────────────── +# Only accounts connects to Cloud SQL; API and marketing don't need DB access. + +resource "google_project_iam_member" "accounts_sql_client" { + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.accounts.email}" +} + +# ── IAM — Public invocation (Cloud Run → LB) ────────────────────────────────── +# allUsers invoker allows the global HTTP(S) load balancer's health checks +# and traffic to reach the services without additional auth headers. + +# Marketing resource "google_cloud_run_v2_service_iam_member" "prod_us_public" { project = var.project_id location = "us-central1" @@ -48,7 +85,55 @@ resource "google_cloud_run_v2_service_iam_member" "stage_public" { name = google_cloud_run_v2_service.stage.name role = "roles/run.invoker" member = "allUsers" - - # Note: Cloudflare Access enforces authentication in front of stage. - # Cloud Run itself allows allUsers so the LB health checks pass. + # Note: Cloudflare Access enforces auth in front of stage. +} + +# Accounts +resource "google_cloud_run_v2_service_iam_member" "accounts_us_public" { + project = var.project_id + location = "us-central1" + name = google_cloud_run_v2_service.accounts_us.name + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_v2_service_iam_member" "accounts_eu_public" { + project = var.project_id + location = "europe-west1" + name = google_cloud_run_v2_service.accounts_eu.name + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_v2_service_iam_member" "accounts_apac_public" { + project = var.project_id + location = "asia-northeast1" + name = google_cloud_run_v2_service.accounts_apac.name + role = "roles/run.invoker" + member = "allUsers" +} + +# API +resource "google_cloud_run_v2_service_iam_member" "api_us_public" { + project = var.project_id + location = "us-central1" + name = google_cloud_run_v2_service.api_us.name + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_v2_service_iam_member" "api_eu_public" { + project = var.project_id + location = "europe-west1" + name = google_cloud_run_v2_service.api_eu.name + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_v2_service_iam_member" "api_apac_public" { + project = var.project_id + location = "asia-northeast1" + name = google_cloud_run_v2_service.api_apac.name + role = "roles/run.invoker" + member = "allUsers" } diff --git a/servers/gcp/variables.tf b/servers/gcp/variables.tf index dae6e0b..95119cf 100644 --- a/servers/gcp/variables.tf +++ b/servers/gcp/variables.tf @@ -5,7 +5,19 @@ variable "project_id" { } variable "image_tag" { - description = "Docker image tag to deploy" + description = "Docker image tag to deploy for the marketing site" + type = string + default = "latest" +} + +variable "accounts_image_tag" { + description = "Docker image tag to deploy for the accounts service" + type = string + default = "latest" +} + +variable "api_image_tag" { + description = "Docker image tag to deploy for the REST API (neuron-rest)" type = string default = "latest" } @@ -20,23 +32,14 @@ variable "cloudflare_zone_id_neurontechnologies" { locals { project_id = var.project_id - image = "us-central1-docker.pkg.dev/${var.project_id}/neuron-marketing/marketing:${var.image_tag}" - # Production secrets from GCP Secret Manager - secrets = [ - "stripe-secret-key", - "stripe-webhook-secret", - "stripe-price-professional", - "stripe-price-founding", - ] + # Image refs per service + marketing_image = "us-central1-docker.pkg.dev/${var.project_id}/neuron-marketing/marketing:${var.image_tag}" + accounts_image = "us-central1-docker.pkg.dev/${var.project_id}/neuron-accounts/accounts:${var.accounts_image_tag}" + api_image = "us-central1-docker.pkg.dev/${var.project_id}/neuron-api/api:${var.api_image_tag}" - # Env var names for each secret (uppercase, hyphen → underscore) - secret_env_map = { - "stripe-secret-key" = "STRIPE_SECRET_KEY" - "stripe-webhook-secret" = "STRIPE_WEBHOOK_SECRET" - "stripe-price-professional" = "STRIPE_PRICE_PROFESSIONAL" - "stripe-price-founding" = "STRIPE_PRICE_FOUNDING" - } + # Keep backward-compat alias (used in cloud-run.tf for marketing service) + image = local.marketing_image # Static publishable key (not sensitive — lives in source) stripe_publishable_key = "pk_live_51TPoHnJg9Fv1D3AUPMXnYyOJIVhn1FyH56zCMNnATo9tR7pO9lNDbXncp6VeDxm38qSdBHZPfqEBipUh3GZsWNyd00jvIO97E6" diff --git a/servers/legion/apps/alloy.yaml b/servers/legion/apps/alloy.yaml index 77dbfc0..245b88d 100644 --- a/servers/legion/apps/alloy.yaml +++ b/servers/legion/apps/alloy.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://grafana.github.io/helm-charts chart: alloy - targetRevision: "*" + targetRevision: ">=0.9.0, <1.0.0" helm: values: | controller: diff --git a/servers/legion/apps/gitea-runner.yaml b/servers/legion/apps/gitea-runner.yaml index 2902f60..29ff958 100644 --- a/servers/legion/apps/gitea-runner.yaml +++ b/servers/legion/apps/gitea-runner.yaml @@ -1,3 +1,7 @@ +--- +# Gitea CI runner — general-purpose (legion) +# Docker socket REMOVED. Builds go through buildkitd TCP endpoint. +# Set DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 in build steps. apiVersion: apps/v1 kind: Deployment metadata: @@ -15,8 +19,10 @@ spec: labels: app: gitea-runner annotations: - config-version: "2026-04-23-nodeport-url" + config-version: "2026-04-25-buildkit-no-sock" spec: + securityContext: + runAsNonRoot: false # act_runner needs root for container management initContainers: - name: register image: registry.neuralplatform.ai/ci-base:latest @@ -36,10 +42,9 @@ spec: timeout: 3h container: network: host - docker_host: "unix:///var/run/docker.sock" + docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234" force_pull: false - valid_volumes: - - /var/run/docker.sock + valid_volumes: [] default_image: "registry.neuralplatform.ai/ci-base:latest" extra_hosts: - "gitea.git.svc.cluster.local:10.43.1.53" @@ -55,14 +60,16 @@ spec: image: registry.neuralplatform.ai/ci-base:latest workingDir: /data command: ["act_runner", "daemon", "--config", "/data/config.yaml"] + env: + - name: DOCKER_HOST + value: "tcp://buildkitd.ci.svc.cluster.local:1234" envFrom: - secretRef: name: gitea-runner-secret volumeMounts: - name: data mountPath: /data - - name: docker-sock - mountPath: /var/run/docker.sock + # docker-sock volume intentionally removed — use buildkitd TCP instead resources: requests: memory: 512Mi @@ -73,11 +80,9 @@ spec: volumes: - name: data emptyDir: {} - - name: docker-sock - hostPath: - path: /var/run/docker.sock - type: Socket + # docker-sock hostPath intentionally removed --- +# Neuron Technologies CI runner apiVersion: apps/v1 kind: Deployment metadata: @@ -95,7 +100,7 @@ spec: labels: app: neuron-technologies-runner annotations: - config-version: "2026-04-24-initial" + config-version: "2026-04-25-buildkit-no-sock" spec: initContainers: - name: register @@ -116,10 +121,9 @@ spec: timeout: 3h container: network: host - docker_host: "unix:///var/run/docker.sock" + docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234" force_pull: false - valid_volumes: - - /var/run/docker.sock + valid_volumes: [] default_image: "registry.neuralplatform.ai/ci-base:latest" extra_hosts: - "gitea.git.svc.cluster.local:10.43.1.53" @@ -135,14 +139,15 @@ spec: image: registry.neuralplatform.ai/ci-base:latest workingDir: /data command: ["act_runner", "daemon", "--config", "/data/config.yaml"] + env: + - name: DOCKER_HOST + value: "tcp://buildkitd.ci.svc.cluster.local:1234" envFrom: - secretRef: name: neuron-technologies-runner-secret volumeMounts: - name: data mountPath: /data - - name: docker-sock - mountPath: /var/run/docker.sock resources: requests: memory: 512Mi @@ -153,7 +158,3 @@ spec: volumes: - name: data emptyDir: {} - - name: docker-sock - hostPath: - path: /var/run/docker.sock - type: Socket diff --git a/servers/legion/apps/kube-prometheus-stack.yaml b/servers/legion/apps/kube-prometheus-stack.yaml index b3ff096..a584459 100644 --- a/servers/legion/apps/kube-prometheus-stack.yaml +++ b/servers/legion/apps/kube-prometheus-stack.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://prometheus-community.github.io/helm-charts chart: kube-prometheus-stack - targetRevision: "*" + targetRevision: ">=60.0.0, <70.0.0" helm: values: | grafana: @@ -37,6 +37,13 @@ spec: cert-manager.io/cluster-issuer: letsencrypt-prod prometheus: prometheusSpec: + # Discover ServiceMonitors and PrometheusRules from ALL namespaces. + # Without this, Prometheus only sees resources in the monitoring namespace. + serviceMonitorSelectorNilUsesHelmValues: false + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: {} + ruleNamespaceSelector: {} + podMonitorNamespaceSelector: {} storageSpec: volumeClaimTemplate: spec: diff --git a/servers/legion/apps/loki.yaml b/servers/legion/apps/loki.yaml index 9105f7b..e9891ad 100644 --- a/servers/legion/apps/loki.yaml +++ b/servers/legion/apps/loki.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://grafana.github.io/helm-charts chart: loki - targetRevision: "*" + targetRevision: ">=6.0.0, <7.0.0" helm: values: | loki: diff --git a/servers/legion/apps/neuron-prod-project.yaml b/servers/legion/apps/neuron-prod-project.yaml new file mode 100644 index 0000000..ec3e501 --- /dev/null +++ b/servers/legion/apps/neuron-prod-project.yaml @@ -0,0 +1,89 @@ +--- +# ArgoCD AppProject — neuron-prod +# Restricts which repos can deploy to which namespaces. +# Prevents misconfigured apps from deploying cluster-admin RBAC or +# accidentally targeting personal/hobby namespaces. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: neuron-prod + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: "Neuron Technologies production services" + + # Only allow deployments sourced from the infrastructure repo. + sourceRepos: + - "http://10.43.1.53:3000/will/infrastructure.git" + - "https://code.forgejo.org" + - "registry.neuralplatform.ai" + + # Only allow deploying to neuron-prod and platform namespaces. + destinations: + - server: https://kubernetes.default.svc + namespace: neuron-prod + - server: https://kubernetes.default.svc + namespace: platform + + # Cluster-scoped resources this project may NOT create. + clusterResourceBlacklist: + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: "" + kind: Namespace + + # Namespace-scoped resources this project may create. + namespaceResourceWhitelist: + - group: "apps" + kind: Deployment + - group: "apps" + kind: ReplicaSet + - group: "batch" + kind: CronJob + - group: "batch" + kind: Job + - group: "" + kind: Service + - group: "" + kind: ConfigMap + - group: "" + kind: Secret + - group: "" + kind: PersistentVolumeClaim + - group: "" + kind: ServiceAccount + - group: "networking.k8s.io" + kind: Ingress + - group: "networking.k8s.io" + kind: NetworkPolicy + - group: "policy" + kind: PodDisruptionBudget + - group: "autoscaling" + kind: HorizontalPodAutoscaler + - group: "traefik.io" + kind: IngressRoute + - group: "external-secrets.io" + kind: ExternalSecret + - group: "monitoring.coreos.com" + kind: ServiceMonitor + - group: "monitoring.coreos.com" + kind: PrometheusRule + + # Sync windows: only allow automated syncs during low-traffic hours. + # Manual syncs allowed any time. + syncWindows: + - kind: allow + schedule: "0 2-6 * * *" # 2am-6am UTC (off-peak) + duration: 4h + applications: + - "*" + manualSync: true + - kind: allow + schedule: "* * * * *" + duration: "1m" + manualSync: true # Always allow manual deploys + applications: + - "*" diff --git a/servers/legion/apps/postgres.yaml b/servers/legion/apps/postgres.yaml index 2480f96..04ed5fd 100644 --- a/servers/legion/apps/postgres.yaml +++ b/servers/legion/apps/postgres.yaml @@ -9,7 +9,7 @@ spec: # Helm chart from Docker Hub OCI registry - repoURL: registry-1.docker.io/bitnamicharts chart: postgresql - targetRevision: "*" + targetRevision: ">=16.0.0, <17.0.0" helm: values: | auth: diff --git a/servers/legion/apps/redis.yaml b/servers/legion/apps/redis.yaml index d56c5e9..6d91a50 100644 --- a/servers/legion/apps/redis.yaml +++ b/servers/legion/apps/redis.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: registry-1.docker.io/bitnamicharts chart: redis - targetRevision: "*" + targetRevision: ">=20.0.0, <21.0.0" helm: values: | auth: diff --git a/servers/legion/apps/tempo.yaml b/servers/legion/apps/tempo.yaml index 6c86318..dc35229 100644 --- a/servers/legion/apps/tempo.yaml +++ b/servers/legion/apps/tempo.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://grafana.github.io/helm-charts chart: tempo - targetRevision: "*" + targetRevision: ">=1.0.0, <2.0.0" helm: values: | tempo: diff --git a/servers/legion/k8s/backup/cronjob.yaml b/servers/legion/k8s/backup/cronjob.yaml index 283d943..9a5367e 100644 --- a/servers/legion/k8s/backup/cronjob.yaml +++ b/servers/legion/k8s/backup/cronjob.yaml @@ -43,8 +43,10 @@ spec: limits: memory: 256Mi containers: + # Pin to explicit version — never use latest for backup tooling. + # Bump this intentionally after testing; do not let it float. - name: backup - image: restic/restic:latest + image: restic/restic:0.17.3 command: - /bin/sh - -c @@ -93,3 +95,77 @@ spec: readOnly: true - name: dump emptyDir: {} +--- +# Weekly restore-verify job — proves the backup is actually restorable. +# Runs Sunday 4am. Restores latest snapshot to /tmp, validates the SQL dump. +# Alerts via Alertmanager if it fails (BackupJobFailed PrometheusRule watches git namespace). +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-verify + namespace: git +spec: + schedule: "0 4 * * 0" + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 2 + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 1800 # 30 min max + template: + spec: + restartPolicy: Never + containers: + - name: verify + image: restic/restic:0.17.3 + command: + - /bin/sh + - -c + - | + set -e + + echo "=== Backup Restore Verification ===" + echo "Listing latest snapshots..." + restic snapshots --latest 3 + + echo "Restoring latest snapshot to /tmp/restore-verify..." + mkdir -p /tmp/restore-verify + restic restore latest \ + --target /tmp/restore-verify \ + --include /all-databases.sql + + SQL_FILE="/tmp/restore-verify/all-databases.sql" + if [ ! -f "$SQL_FILE" ]; then + echo "ERROR: SQL dump not found in restored snapshot" + exit 1 + fi + + SIZE=$(du -sh "$SQL_FILE" | cut -f1) + echo "SQL dump restored: $SIZE" + + # Basic structural validation + if ! grep -q "PostgreSQL database cluster dump" "$SQL_FILE"; then + echo "ERROR: SQL dump does not look like a valid pg_dumpall output" + exit 1 + fi + + TABLE_COUNT=$(grep -c "^CREATE TABLE" "$SQL_FILE" || true) + echo "Tables found in dump: $TABLE_COUNT" + + if [ "$TABLE_COUNT" -lt 5 ]; then + echo "ERROR: Too few tables in dump (expected >= 5, got $TABLE_COUNT)" + exit 1 + fi + + echo "=== Verification PASSED: backup is restorable ===" + rm -rf /tmp/restore-verify + envFrom: + - secretRef: + name: backup-credentials + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 512Mi diff --git a/servers/legion/k8s/external-secrets/cluster-secret-store.yaml b/servers/legion/k8s/external-secrets/cluster-secret-store.yaml index 4c16fdf..5047b62 100644 --- a/servers/legion/k8s/external-secrets/cluster-secret-store.yaml +++ b/servers/legion/k8s/external-secrets/cluster-secret-store.yaml @@ -1,11 +1,47 @@ --- -# ClusterSecretStore — connects ESO to Vault using the root token Secret. -# The vault-token Secret (name: vault-token, key: token) must be created -# via kubectl as a one-time bootstrap step before this store can sync: +# ClusterSecretStore — connects ESO to Vault # -# kubectl create secret generic vault-token \ -# --namespace external-secrets \ -# --from-literal=token=$(cat ~/Secrets/tokens/vault-root-token) +# CURRENT STATE: token auth (root token). This works but the root token +# has no expiry and full Vault access — it's a single credential that +# compromises everything if leaked. +# +# MIGRATION TO APPROLE (do this before GCP migration): +# ───────────────────────────────────────────────────── +# 1. Enable AppRole on Vault (if not already): +# vault auth enable approle +# +# 2. Create a policy scoped to only the paths ESO reads: +# vault policy write eso-read - < \ +# --from-literal=secretId= +# +# 6. Flip this file to the appRole stanza below, commit, push. +# ESO will hot-reload when the manifest changes. +# +# 7. Delete the old vault-token secret: +# kubectl delete secret vault-token -n external-secrets +# ───────────────────────────────────────────────────── + apiVersion: external-secrets.io/v1beta1 kind: ClusterSecretStore metadata: @@ -17,7 +53,17 @@ spec: path: "secret" version: "v2" auth: + # ── Current: root token auth ────────────────────────────────────────── + # TODO: replace with appRole stanza once AppRole is bootstrapped (see above). tokenSecretRef: name: vault-token namespace: external-secrets key: token + # ── Target: AppRole auth (uncomment after bootstrap) ────────────────── + # appRole: + # path: "approle" + # roleId: "eso" + # secretRef: + # name: vault-approle + # namespace: external-secrets + # key: secretId diff --git a/servers/legion/k8s/gitea-runner/buildkitd.yaml b/servers/legion/k8s/gitea-runner/buildkitd.yaml new file mode 100644 index 0000000..7abb889 --- /dev/null +++ b/servers/legion/k8s/gitea-runner/buildkitd.yaml @@ -0,0 +1,71 @@ +--- +# BuildKit daemon — rootless image builder for CI runners. +# Replaces the /var/run/docker.sock hostPath mount that gave containers +# full root access to the Legion host node. +# +# Runners connect via TCP: DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: buildkitd + namespace: ci + labels: + app: buildkitd +spec: + replicas: 1 + selector: + matchLabels: + app: buildkitd + template: + metadata: + labels: + app: buildkitd + spec: + containers: + - name: buildkitd + image: moby/buildkit:v0.19.0-rootless + args: + - --oci-worker-no-process-sandbox + - --addr + - tcp://0.0.0.0:1234 + - --addr + - unix:///run/buildkit/buildkitd.sock + ports: + - name: tcp + containerPort: 1234 + securityContext: + seccompProfile: + type: Unconfined # BuildKit rootless requires this + runAsUser: 1000 + runAsGroup: 1000 + readinessProbe: + exec: + command: ["buildctl", "debug", "workers"] + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "4" + memory: 4Gi + volumeMounts: + - name: buildkit-storage + mountPath: /home/user/.local/share/buildkit + volumes: + - name: buildkit-storage + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: buildkitd + namespace: ci +spec: + selector: + app: buildkitd + ports: + - name: tcp + port: 1234 + targetPort: 1234 diff --git a/servers/legion/k8s/monitoring/alerting-rules.yaml b/servers/legion/k8s/monitoring/alerting-rules.yaml new file mode 100644 index 0000000..96382fd --- /dev/null +++ b/servers/legion/k8s/monitoring/alerting-rules.yaml @@ -0,0 +1,153 @@ +--- +# PrometheusRules — SLO and operational alerts for neuron-prod +# These fire into Alertmanager which routes to #infrastructure-alerts on Slack. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: neuron-slo + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + + # ── Neuron API SLOs ──────────────────────────────────────��─────────────── + - name: neuron.slo + interval: 60s + rules: + - alert: NeuronMCPHighErrorRate + expr: | + ( + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp",status=~"5.."}[5m])) + / + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp"}[5m])) + ) > 0.01 + for: 2m + labels: + severity: critical + namespace: neuron-prod + annotations: + summary: "neuron-mcp 5xx error rate > 1%" + description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes." + + - alert: NeuronRestHighErrorRate + expr: | + ( + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest",status=~"5.."}[5m])) + / + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest"}[5m])) + ) > 0.01 + for: 2m + labels: + severity: critical + namespace: neuron-prod + annotations: + summary: "neuron-rest 5xx error rate > 1%" + description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes." + + - alert: NeuronMCPHighLatency + expr: | + histogram_quantile(0.99, + sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-mcp"}[10m])) + by (le) + ) > 2.0 + for: 5m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "neuron-mcp p99 latency > 2s" + description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes." + + - alert: NeuronRestHighLatency + expr: | + histogram_quantile(0.99, + sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-rest"}[10m])) + by (le) + ) > 2.0 + for: 5m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "neuron-rest p99 latency > 2s" + description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes." + + # ── Pod health ─────────────────────────────────────────────────────────── + - name: neuron.pods + rules: + - alert: NeuronPodCrashLooping + expr: | + rate(kube_pod_container_status_restarts_total{namespace="neuron-prod"}[15m]) * 60 * 15 > 3 + for: 5m + labels: + severity: critical + namespace: neuron-prod + annotations: + summary: "Pod {{ $labels.pod }} is crash looping" + description: "Container {{ $labels.container }} restarted {{ $value | humanize }} times in 15 minutes." + + - alert: NeuronPodNotReady + expr: | + sum by (pod) (kube_pod_status_ready{namespace="neuron-prod",condition="false"}) > 0 + for: 5m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "Pod {{ $labels.pod }} not ready for 5 minutes" + + - alert: NeuronDeploymentReplicasMissing + expr: | + kube_deployment_status_replicas_available{namespace="neuron-prod"} + < kube_deployment_spec_replicas{namespace="neuron-prod"} + for: 3m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "Deployment {{ $labels.deployment }} has fewer replicas than desired" + + # ── Postgres ─────────────────────────���─────────────────────────────────��─ + - name: postgres.health + rules: + - alert: PostgresDown + expr: pg_up{namespace="platform"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Postgres is down" + + - alert: PostgresHighConnections + expr: | + (pg_stat_database_numbackends{namespace="platform"} / pg_settings_max_connections{namespace="platform"}) + > 0.80 + for: 5m + labels: + severity: warning + annotations: + summary: "Postgres connection pool > 80%" + description: "Connection utilization is {{ $value | humanizePercentage }}." + + # ── Backup health ────────────────────────────���─────────────────────────── + - name: backup.health + rules: + - alert: BackupJobFailed + expr: | + kube_job_status_failed{namespace="git",job_name=~"gitea-backup.*"} > 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Gitea backup job failed" + description: "Check: kubectl logs -n git -l job-name=gitea-backup --previous" + + - alert: BackupNotRunRecently + expr: | + (time() - kube_job_status_completion_time{namespace="git",job_name=~"gitea-backup.*"}) > 90000 + for: 0m + labels: + severity: warning + annotations: + summary: "Last successful backup was more than 25 hours ago" diff --git a/servers/legion/k8s/monitoring/servicemonitor.yaml b/servers/legion/k8s/monitoring/servicemonitor.yaml new file mode 100644 index 0000000..2ac9c2d --- /dev/null +++ b/servers/legion/k8s/monitoring/servicemonitor.yaml @@ -0,0 +1,64 @@ +--- +# ServiceMonitors for neuron-prod — scraped by kube-prometheus-stack Prometheus +# Requires: kube-prometheus-stack prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +# OR a serviceMonitorNamespaceSelector that includes neuron-prod. +# We set this in the Prometheus stack values (see apps/kube-prometheus-stack.yaml). +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: neuron-mcp + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - neuron-prod + selector: + matchLabels: + app: neuron-mcp + endpoints: + - port: http + path: /actuator/prometheus + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: neuron-rest + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - neuron-prod + selector: + matchLabels: + app: neuron-rest + endpoints: + - port: http + path: /actuator/prometheus + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: neuron-marketing + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - neuron-prod + selector: + matchLabels: + app: neuron-marketing + endpoints: + - port: http + path: /api/metrics + interval: 60s + scrapeTimeout: 15s diff --git a/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml b/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml index 1c7d775..53a973c 100644 --- a/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml @@ -17,6 +17,11 @@ spec: app: neuron-marketing env: prod spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault containers: - name: neuron-marketing image: registry.neuralplatform.ai/neuron-technologies/marketing:1e94e8ae @@ -24,6 +29,12 @@ spec: ports: - name: http containerPort: 3000 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false # Next.js writes .next/cache at runtime + runAsNonRoot: true + capabilities: + drop: ["ALL"] env: - name: NODE_ENV value: production diff --git a/servers/legion/k8s/neuron-technologies/prod/hpa.yaml b/servers/legion/k8s/neuron-technologies/prod/hpa.yaml new file mode 100644 index 0000000..b947c72 --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/hpa.yaml @@ -0,0 +1,98 @@ +--- +# HorizontalPodAutoscalers for neuron-prod +# Single Legion node today — minReplicas=1 prevents unnecessary idle waste. +# When GCP GKE is live these minimums should go to 2 for HA. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: neuron-mcp-hpa + namespace: neuron-prod +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: neuron-mcp-blue + minReplicas: 1 + maxReplicas: 6 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 65 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 # Don't thrash — wait 5m before scaling down + policies: + - type: Pods + value: 1 + periodSeconds: 120 + scaleUp: + stabilizationWindowSeconds: 30 + policies: + - type: Pods + value: 2 + periodSeconds: 60 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: neuron-rest-hpa + namespace: neuron-prod +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: neuron-rest + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 65 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 30 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: neuron-marketing-hpa + namespace: neuron-prod +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: neuron-marketing + minReplicas: 2 + maxReplicas: 8 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleDown: + stabilizationWindowSeconds: 120 + scaleUp: + stabilizationWindowSeconds: 15 diff --git a/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml b/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml index 3d1ae8b..5c73cd9 100644 --- a/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml @@ -15,3 +15,7 @@ resources: - backup-cronjob.yaml - runpod-inference.yaml - runpod-lb-configmap.yaml + # Hardening additions + - hpa.yaml + - pdb.yaml + - network-policy.yaml diff --git a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml index 9215924..05769ef 100644 --- a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml @@ -25,6 +25,10 @@ spec: kubectl.kubernetes.io/restartedAt: "2026-04-25T08:40:00Z" spec: terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: neuron-mcp image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.15.3 @@ -32,6 +36,12 @@ spec: ports: - name: http containerPort: 8080 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false # JVM writes temp files; restrict paths via allowedPaths once profiled + runAsNonRoot: true + capabilities: + drop: ["ALL"] env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://alloy-otlp.monitoring.svc.cluster.local:4318" @@ -57,18 +67,25 @@ spec: exec: # Give Traefik time to drain in-flight connections before pod terminates command: ["sh", "-c", "sleep 8"] + # startupProbe gives the JVM up to 150s to start before liveness takes over. + # Prevents false crash loops during slow cold starts. + startupProbe: + httpGet: + path: /actuator/health/liveness + port: http + failureThreshold: 30 + periodSeconds: 5 livenessProbe: httpGet: path: /actuator/health/liveness port: http - initialDelaySeconds: 30 periodSeconds: 30 failureThreshold: 3 readinessProbe: httpGet: path: /actuator/health/readiness port: http - initialDelaySeconds: 10 + initialDelaySeconds: 5 periodSeconds: 10 failureThreshold: 3 volumes: diff --git a/servers/legion/k8s/neuron-technologies/prod/network-policy.yaml b/servers/legion/k8s/neuron-technologies/prod/network-policy.yaml new file mode 100644 index 0000000..7f9dec9 --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/network-policy.yaml @@ -0,0 +1,124 @@ +--- +# NetworkPolicies for neuron-prod +# Default-deny all ingress and egress, then explicitly allow only what's needed. +# Enforced by k3s's built-in NetworkPolicy controller. + +# ── Baseline: deny all traffic in namespace ─────────────────────────────────── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: neuron-prod +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +# ── neuron-mcp: accept traffic from Traefik (kube-system) only ─────────────── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-mcp-ingress + namespace: neuron-prod +spec: + podSelector: + matchLabels: + app: neuron-mcp + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: neuron-prod +--- +# ── neuron-rest: accept from mcp, kube-system (Traefik), monitoring (Alloy) ── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-rest-ingress + namespace: neuron-prod +spec: + podSelector: + matchLabels: + app: neuron-rest + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: neuron-prod + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring +--- +# ── neuron-marketing: accept from Traefik only ─────────────────────────────── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-marketing-ingress + namespace: neuron-prod +spec: + podSelector: + matchLabels: + app: neuron-marketing + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system +--- +# ── Egress: all prod pods may reach platform (postgres/redis), vault, +# monitoring (alloy OTLP), kube-dns, and the internet (external APIs) ─ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-prod-egress + namespace: neuron-prod +spec: + podSelector: {} + policyTypes: + - Egress + egress: + # Kubernetes DNS + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # platform namespace (Postgres, Redis, accounts service) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform + # vault namespace + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + # monitoring namespace (Alloy OTLP) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + # intra-namespace (mcp ↔ rest ↔ license) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: neuron-prod + # external internet (Stripe, OAuth providers, Cloudflare, etc.) + - ports: + - port: 443 + protocol: TCP + - port: 80 + protocol: TCP diff --git a/servers/legion/k8s/neuron-technologies/prod/pdb.yaml b/servers/legion/k8s/neuron-technologies/prod/pdb.yaml new file mode 100644 index 0000000..a83438f --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/pdb.yaml @@ -0,0 +1,39 @@ +--- +# PodDisruptionBudgets for neuron-prod +# Ensures at least 1 replica of each service survives node drains and upgrades. +# With replicas=1 today, minAvailable=1 means drain will block until HPA scales up +# to 2+ — this is intentional and provides a natural safety gate. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: neuron-mcp-pdb + namespace: neuron-prod +spec: + minAvailable: 1 + selector: + matchLabels: + app: neuron-mcp + slot: blue +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: neuron-rest-pdb + namespace: neuron-prod +spec: + minAvailable: 1 + selector: + matchLabels: + app: neuron-rest +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: neuron-marketing-pdb + namespace: neuron-prod +spec: + # Marketing runs 2 replicas — keep at least 1 alive during drains. + minAvailable: 1 + selector: + matchLabels: + app: neuron-marketing diff --git a/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml b/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml index 59c3508..4cdf765 100644 --- a/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml @@ -19,6 +19,10 @@ spec: annotations: kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z" spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: neuron-rest image: registry.neuralplatform.ai/neuron-technologies/neuron-rest:v0.15.3 @@ -26,6 +30,12 @@ spec: ports: - name: http containerPort: 8081 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] envFrom: - configMapRef: name: neuron-prod-config @@ -41,18 +51,23 @@ spec: limits: cpu: 1000m memory: 1Gi + startupProbe: + httpGet: + path: /actuator/health/liveness + port: http + failureThreshold: 30 + periodSeconds: 5 livenessProbe: httpGet: path: /actuator/health/liveness port: http - initialDelaySeconds: 30 periodSeconds: 30 failureThreshold: 3 readinessProbe: httpGet: path: /actuator/health/readiness port: http - initialDelaySeconds: 15 + initialDelaySeconds: 5 periodSeconds: 10 failureThreshold: 3 volumes: diff --git a/servers/legion/scripts/neuron-self-improve.sh b/servers/legion/scripts/neuron-self-improve.sh index 1a9d16d..5055544 100755 --- a/servers/legion/scripts/neuron-self-improve.sh +++ b/servers/legion/scripts/neuron-self-improve.sh @@ -1,46 +1,27 @@ #!/usr/bin/env bash # neuron-self-improve.sh — self-improvement loop for Neuron # -# Manages code experiments across alpha/beta/gamma zones and promotes winners to prod. -# All cluster changes go through git → Argo CD. No direct kubectl mutations. +# Model: stage is the experiment slot. prod runs blue/green. +# Promote flow: implement → CI → stage → verify → blue-green-deploy → prod. # # Usage: -# ./neuron-self-improve.sh snapshot # copy prod DB to slot -# ./neuron-self-improve.sh deploy # point slot at a specific image -# ./neuron-self-improve.sh promote # copy slot's image to prod -# ./neuron-self-improve.sh status # show what's running in each zone -# ./neuron-self-improve.sh rollback # revert slot to a previous image +# ./neuron-self-improve.sh snapshot # copy prod DB to stage +# ./neuron-self-improve.sh deploy # point stage at a specific image +# ./neuron-self-improve.sh promote # blue/green flip prod to +# ./neuron-self-improve.sh status # show active blue/green slot + stage +# ./neuron-self-improve.sh rollback # roll prod back to via blue/green set -euo pipefail INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REGISTRY="registry.neuralplatform.ai/neural-platform/neuron" +REGISTRY="registry.neuralplatform.ai/neuron-technologies/neuron-mcp" KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}" +KC_PROD="kubectl --kubeconfig=$KUBECONFIG -n neuron-prod" +KC_STAGE="kubectl --kubeconfig=$KUBECONFIG -n neuron-stage" + # ── helpers ─────────────────────────────────────────────────────────────────── -slot_manifest() { - local slot=$1 - if [[ "$slot" == "prod" ]]; then - echo "$INFRA_DIR/servers/legion/apps/neuron.yaml" - else - echo "$INFRA_DIR/servers/legion/apps/neuron-${slot}.yaml" - fi -} - -current_sha() { - local slot=$1 - grep "image:" "$(slot_manifest "$slot")" | grep -oP '[a-f0-9]{40}' | head -1 -} - -set_image() { - local slot=$1 sha=$2 - local manifest - manifest="$(slot_manifest "$slot")" - sed -i "s|${REGISTRY}:.*|${REGISTRY}:${sha}|" "$manifest" - echo " set $slot → $sha" -} - git_push() { local msg=$1 cd "$INFRA_DIR" @@ -50,35 +31,42 @@ git_push() { echo " pushed: $msg" } -wait_healthy() { - local slot=$1 - local app="neuron" - [[ "$slot" != "prod" ]] && app="neuron-${slot}" - echo " waiting for $slot to be healthy..." - local i=0 - while [[ $i -lt 30 ]]; do - local ready - ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ - --no-headers 2>/dev/null | grep "1/1" | wc -l) - [[ "$ready" -ge 1 ]] && echo " $slot is healthy" && return 0 - sleep 10 - ((i++)) - done - echo " WARNING: $slot did not become healthy within 5 minutes" - return 1 +active_prod_slot() { + $KC_PROD get svc neuron-mcp -o jsonpath='{.spec.selector.slot}' 2>/dev/null || echo "unknown" +} + +current_prod_tag() { + local slot + slot=$(active_prod_slot) + $KC_PROD get deployment "neuron-mcp-$slot" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \ + | cut -d: -f2 || echo "unknown" +} + +wait_ready() { + local ns=$1 label=$2 timeout=${3:-180} + echo " waiting for $label to be ready..." + kubectl --kubeconfig="$KUBECONFIG" -n "$ns" \ + rollout status deployment "$label" --timeout="${timeout}s" + echo " $label is ready" } # ── commands ────────────────────────────────────────────────────────────────── cmd_snapshot() { - local slot=${1:?Usage: snapshot } - echo "==> Snapshotting prod DB into $slot..." - KUBECONFIG="$KUBECONFIG" kubectl apply -f - << EOF + echo "==> Snapshotting prod DB into stage..." + local prod_slot + prod_slot=$(active_prod_slot) + local prod_pvc="neuron-prod-data" + local stage_pvc="neuron-stage-data" + + # Run a one-shot job that copies prod DB to stage PVC + $KC_PROD apply -f - << EOF apiVersion: batch/v1 kind: Job metadata: - name: snapshot-prod-to-${slot} - namespace: neuron + name: snapshot-prod-to-stage + namespace: neuron-prod spec: ttlSecondsAfterFinished: 300 template: @@ -89,7 +77,9 @@ spec: image: keinos/sqlite3:latest command: ["/bin/sh", "-c"] args: - - sqlite3 /src/neuron.db ".backup /dst/neuron.db" && echo "Done" && ls -lh /dst/neuron.db + - | + sqlite3 /src/neuron.db ".backup /dst/neuron.db" + echo "Done — $(ls -lh /dst/neuron.db | awk '{print \$5}')" volumeMounts: - name: src mountPath: /src @@ -99,125 +89,127 @@ spec: volumes: - name: src persistentVolumeClaim: - claimName: neuron-data + claimName: ${prod_pvc} - name: dst persistentVolumeClaim: - claimName: neuron-${slot}-data + claimName: ${stage_pvc} EOF + echo " waiting for snapshot job..." - KUBECONFIG="$KUBECONFIG" kubectl wait job "snapshot-prod-to-${slot}" \ - -n neuron --for=condition=complete --timeout=120s - echo " snapshot complete" + $KC_PROD wait job snapshot-prod-to-stage --for=condition=complete --timeout=120s + echo " snapshot complete — stage DB is now a copy of prod" } cmd_deploy() { - local slot=${1:?Usage: deploy } - local sha=${2:?Usage: deploy } - echo "==> Deploying $sha to $slot..." - set_image "$slot" "$sha" - git_push "chore(neuron): deploy ${sha:0:12} to $slot" - wait_healthy "$slot" - echo "==> $slot is live at https://neuron-${slot}.neuralplatform.ai" + local sha=${1:?Usage: deploy } + echo "==> Deploying $sha to stage..." + + # Update stage deployment image in-place (stage is a single deployment, not blue/green) + $KC_STAGE set image deployment/neuron-mcp "neuron-mcp=${REGISTRY}:${sha}" + $KC_STAGE rollout restart deployment/neuron-mcp + wait_ready "neuron-stage" "neuron-mcp" + + echo "==> stage is live" + echo " endpoint: https://neuron.neuralplatform.ai/stage (or stage ingress if configured)" + echo " MCP URL: (see stage ingress/services.yaml)" } cmd_promote() { - local slot=${1:?Usage: promote } - local sha - sha=$(current_sha "$slot") - [[ -z "$sha" ]] && echo "ERROR: could not read SHA from $slot" && exit 1 - echo "==> Promoting $slot ($sha) to prod..." - set_image "prod" "$sha" - # Also stamp the :stable registry tag (pull and re-tag) - git_push "chore(neuron): promote $slot (${sha:0:12}) to prod" - wait_healthy "prod" - echo "==> prod is now running $sha" - echo "==> prod endpoint: https://neuron.neuralplatform.ai" + local tag=${1:?Usage: promote } + echo "==> Promoting $tag to prod via blue/green flip..." + "$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag" } cmd_rollback() { - local slot=${1:?Usage: rollback } - local sha=${2:?Usage: rollback } - echo "==> Rolling $slot back to $sha..." - set_image "$slot" "$sha" - git_push "chore(neuron): rollback $slot to ${sha:0:12}" - wait_healthy "$slot" - echo "==> $slot rolled back" + local tag=${1:?Usage: rollback } + echo "==> Rolling prod back to $tag via blue/green..." + "$INFRA_DIR/scripts/blue-green-deploy.sh" "$tag" } cmd_status() { - echo "==> Neuron slot status" + echo "==> Neuron deployment status" echo "" - for slot in prod alpha beta gamma; do - local app="neuron" - [[ "$slot" != "prod" ]] && app="neuron-${slot}" - local sha - sha=$(current_sha "$slot" 2>/dev/null || echo "unknown") - local ready - ready=$(KUBECONFIG="$KUBECONFIG" kubectl get pods -n neuron -l "app=${app}" \ - --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ') - printf " %-8s %s pods_ready=%s\n" "$slot" "${sha:0:12}" "$ready" - done + + local active_slot + active_slot=$(active_prod_slot) + local prod_tag + prod_tag=$(current_prod_tag) + + printf " %-8s slot=%-5s tag=%s\n" "prod" "$active_slot" "$prod_tag" + + # Stage + local stage_tag + stage_tag=$($KC_STAGE get deployment neuron-mcp \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null \ + | cut -d: -f2 || echo "unknown") + local stage_ready + stage_ready=$($KC_STAGE get pods -l app=neuron-mcp \ + --no-headers 2>/dev/null | grep "1/1" | wc -l | tr -d ' ') + printf " %-8s tag=%-20s pods_ready=%s\n" "stage" "$stage_tag" "$stage_ready" + echo "" - echo " Endpoints:" - echo " prod: https://neuron.neuralplatform.ai" - echo " alpha: https://neuron-alpha.neuralplatform.ai" - echo " beta: https://neuron-beta.neuralplatform.ai" - echo " gamma: https://neuron-gamma.neuralplatform.ai" + echo " Prod endpoint: https://neuron.neuralplatform.ai" + echo " Prod MCP: https://neuron.neuralplatform.ai/mcp" } # ── self-improvement loop (autonomous) ──────────────────────────────────────── -# When called with no args, prints the process for an autonomous agent to follow. +# When called with no args (or 'loop'), prints the process for an autonomous agent. cmd_loop_process() { cat << 'PROCESS' NEURON SELF-IMPROVEMENT LOOP ============================= +Deployment model: stage (experiment) → prod (blue/green). + Each iteration: -1. IDENTIFY — find an improvement opportunity in the codebase. - Sources: Neuron backlog, observed failures, performance gaps, code review. +1. IDENTIFY — find an improvement opportunity. + Sources: Neuron backlog, observed failures, graph reliability gaps, + session startup issues, performance, quality observations. -2. IMPLEMENT — make the change on a git branch in neural-platform/neuron. - Commit message should describe the hypothesis being tested. +2. IMPLEMENT — make the change on a git branch in neuron-technologies/neuron. + Commit message describes the hypothesis. -3. CI — push the branch; CI runs lint/test/build and auto-deploys to alpha. - Wait for CI to complete and alpha pod to be healthy. +3. CI — push to main; CI builds image and publishes to registry. + Image tag = git SHA (short). Wait for CI to complete. -4. SNAPSHOT — before the next experiment touches a slot, snapshot prod DB: - ./scripts/neuron-self-improve.sh snapshot - This ensures the experiment runs against real data without touching prod. +4. SNAPSHOT — before running the experiment, copy prod DB to stage: + ./scripts/neuron-self-improve.sh snapshot + This gives stage a real-data copy without touching prod. -5. EXPERIMENT — run the experiment against the alpha endpoint: - https://neuron-alpha.neuralplatform.ai - Use real workloads. Compare against prod or other slots. - Multiple experiments can run simultaneously across alpha/beta/gamma. +5. DEPLOY TO STAGE — point stage at the new image: + ./scripts/neuron-self-improve.sh deploy + Stage runs against the snapshot DB. Experiment freely. -6. EVALUATE — compare results. Criteria: - - Does it behave correctly? - - Does it perform better (latency, quality, fewer errors)? - - Does it handle edge cases prod handles? - Schema changes are acceptable — they become migrations against prod on promote. +6. EVALUATE — test the stage endpoint. Criteria: + - Correct behavior on same inputs? + - At least one dimension improved (latency, quality, error rate)? + - No regressions on cases prod handles? + - Schema changes are additive/non-destructive? -7. PROMOTE WINNER — if alpha wins: - ./scripts/neuron-self-improve.sh promote alpha - This updates prod's image SHA in git → Argo CD deploys → prod is updated. - The old prod SHA is still in the registry — rollback is always available. +7. PROMOTE — if stage passes, promote to prod via blue/green: + ./scripts/neuron-self-improve.sh promote + The blue-green-deploy.sh handles: scale idle slot → health check → + flip service selector → scale old slot to 0. + The old slot is kept at 0 for instant rollback. -8. ROLLBACK — if something goes wrong: - ./scripts/neuron-self-improve.sh rollback prod +8. ROLLBACK — if something goes wrong post-promote: + ./scripts/neuron-self-improve.sh rollback + Blue/green means rollback is always a slot flip — no redeployment. -9. LOOP — go to step 1. Alpha is now free for the next experiment. +9. LOOP — go to step 1. Stage is free for the next experiment. RULES ------ -- Never skip the snapshot step before running an experiment against a slot. -- Prod data is never lost — slots only get copies. -- Schema changes in experiment code → Alembic auto-migrates the snapshot DB. - On promote, the migration runs against prod. Always verify migrations are - additive/non-destructive before promoting. -- All three experiment zones can run simultaneously with different hypotheses. -- The registry retains every SHA — any version can be deployed to any slot. -- This script handles git push; Argo CD handles k8s — never kubectl apply. +- Always snapshot before deploying to stage (stage DB starts from prod). +- Prod is never touched during the experiment — only on promote. +- Schema changes: test Alembic migration on stage snapshot first. + On promote, the same migration runs against prod. Must be additive. +- The registry retains every SHA — any version is deployable to any slot. +- blue-green-deploy.sh handles the infra flip. This script orchestrates. +- Never kubectl apply directly — all changes via git + Argo CD. + Exception: blue-green-deploy.sh uses kubectl for the slot flip because + that is a runtime operation, not a config change. PROCESS } @@ -225,11 +217,11 @@ PROCESS # ── dispatch ────────────────────────────────────────────────────────────────── case "${1:-loop}" in - snapshot) cmd_snapshot "${@:2}" ;; + snapshot) cmd_snapshot ;; deploy) cmd_deploy "${@:2}" ;; promote) cmd_promote "${@:2}" ;; rollback) cmd_rollback "${@:2}" ;; status) cmd_status ;; loop) cmd_loop_process ;; - *) echo "Usage: $0 snapshot|deploy|promote|rollback|status|loop" && exit 1 ;; + *) echo "Usage: $0 snapshot|deploy |promote |rollback |status|loop" && exit 1 ;; esac