deploy soma inference gateway with neuron model on ai.neurontechnologies.ai

servers/legion/dns-neurontechnologies.tf

@@ -70,6 +70,15 @@ resource "cloudflare_record" "nt_tunnel_alpha" {
   ttl     = 1
 }
 
+resource "cloudflare_record" "nt_tunnel_ai" {
+  zone_id = local.zone_neurontechnologies_ai
+  name    = "ai"
+  type    = "CNAME"
+  content = "${var.cloudflare_tunnel_id}.cfargotunnel.com"
+  proxied = true
+  ttl     = 1
+}
+
 resource "cloudflare_record" "nt_tunnel_beta" {
   zone_id = local.zone_neurontechnologies_ai
   name    = "beta"

servers/legion/k8s/neuron-technologies/prod/kustomization.yaml

@@ -20,3 +20,9 @@ resources:
   - hpa.yaml
   - pdb.yaml
   - network-policy.yaml
+  # Soma inference gateway
+  - soma-externalsecret.yaml
+  - soma-configmap.yaml
+  - soma-deployment.yaml
+  - soma-service.yaml
+  - soma-ingressroute.yaml

servers/legion/k8s/neuron-technologies/prod/soma-configmap.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: soma-config
+  namespace: neuron-prod
+data:
+  soma.toml: |
+    [soma]
+    name = "neuron-prod"
+    version = "0.1.0"
+    region_primary = "us-central1"
+
+    [api]
+    port = 8080
+    default_rate_limit_rpm = 500
+    burst_rate_limit_rpm = 1000
+
+    [routing]
+    anti_concentration_limit = 0.60
+    idle_drain_minutes = 15
+    pre_warm_load_threshold = 0.70
+    cost_oracle_poll_seconds = 60
+
+    [warm_pool]
+    llm_min_warm = 1
+    image_gen_min_warm = 1
+    video_min_warm = 1
+
+    [providers]
+    priority = ["legion"]
+
+    [providers.gcp]
+    enabled = false
+    project_id = ""
+    zones = []
+    service_account_key_path = ""
+
+    [providers.runpod]
+    enabled = false
+    api_key_env = "RUNPOD_API_KEY"
+    preferred_gpu = "H100_SXM"
+
+    [providers.legion]
+    enabled = true
+    host = "legion.neuralplatform.ai"
+    ssh_key_env = "LEGION_SSH_KEY"
+    cost_per_hour = 0.40
+
+    [providers.aws]
+    enabled = false
+    region = "us-east-1"
+    access_key_env = "AWS_ACCESS_KEY_ID"
+    secret_key_env = "AWS_SECRET_ACCESS_KEY"
+
+    [providers.azure]
+    enabled = false
+    subscription_id = ""
+
+    [storage]
+    primary = "r2"
+    replicate_to = []
+
+    [storage.gcs]
+    project = ""
+    buckets = []
+
+    [storage.r2]
+    account_id_env = "CF_R2_ACCOUNT_ID"
+    access_key_env = "CF_R2_ACCESS_KEY"
+
+    [identity]
+    vault_addr = "https://vault.neuralplatform.ai"
+    vault_token_env = "VAULT_TOKEN"
+    secret_rotation_days = 90
+
+    [inference.llm]
+    default_model = "neuron-technologies/Neuron"
+    default_backend = "huggingface"
+
+    [inference.image_gen]
+    default_model = "lustify"
+    default_backend = "sd-forge"
+    default_width = 1024
+    default_height = 1024
+
+    [email]
+    provider = "smtp"
+    from_address = "noreply@neurontechnologies.ai"
+    from_name = "Neuron Technologies"
+    smtp_host = "smtp.postmarkapp.com"
+    smtp_port_env = "SMTP_PORT"
+
+    [telemetry]
+    otlp_endpoint = "http://alloy.monitoring.svc.cluster.local:4318"
+    otlp_tenant = "legion"
+    log_level = "info"

servers/legion/k8s/neuron-technologies/prod/soma-deployment.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: soma
+  namespace: neuron-prod
+  labels:
+    app: soma
+    env: prod
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: soma
+  template:
+    metadata:
+      labels:
+        app: soma
+        env: prod
+    spec:
+      securityContext:
+        runAsUser: 1000
+        fsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: soma
+          image: registry.neuralplatform.ai/soma:latest
+          imagePullPolicy: Always
+          ports:
+            - name: http
+              containerPort: 8080
+          env:
+            - name: SOMA_CONFIG_PATH
+              value: /etc/soma/soma.toml
+          envFrom:
+            - secretRef:
+                name: soma-secrets
+          volumeMounts:
+            - name: config
+              mountPath: /etc/soma
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 500m
+              memory: 512Mi
+            limits:
+              cpu: 2000m
+              memory: 1Gi
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
+      volumes:
+        - name: config
+          configMap:
+            name: soma-config

servers/legion/k8s/neuron-technologies/prod/soma-externalsecret.yaml

@@ -0,0 +1,22 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: soma-secrets
+  namespace: neuron-prod
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vault
+    kind: ClusterSecretStore
+  target:
+    name: soma-secrets
+    creationPolicy: Owner
+  data:
+    - secretKey: SOMA_OPERATOR_KEY
+      remoteRef:
+        key: secret/data/soma
+        property: operator_key
+    - secretKey: HF_TOKEN
+      remoteRef:
+        key: secret/data/soma
+        property: hf_token

servers/legion/k8s/neuron-technologies/prod/soma-ingressroute.yaml

@@ -0,0 +1,18 @@
+# Traefik IngressRoute: ai.neurontechnologies.ai → soma service
+# Exposes soma as the Neuron AI inference gateway.
+# Endpoint: https://ai.neurontechnologies.ai/v1/chat/completions
+
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: soma-ai
+  namespace: neuron-prod
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`ai.neurontechnologies.ai`)
+      kind: Rule
+      services:
+        - name: soma
+          port: 8080

servers/legion/k8s/neuron-technologies/prod/soma-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: soma
+  namespace: neuron-prod
+  labels:
+    app: soma
+spec:
+  selector:
+    app: soma
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080
+  type: ClusterIP