el/.gitea/workflows/sdk-release.yaml

name: El SDK Release

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  build-and-release:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: lang

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Enforce source branch (main <- stage only)
        if: github.event_name == 'pull_request'
        run: |
          SOURCE="${GITHUB_HEAD_REF}"
          if [ "${SOURCE}" != "stage" ]; then
            echo "ERROR: Main branch only accepts PRs from 'stage'. Source was: '${SOURCE}'"
            exit 1
          fi
          echo "Source branch check passed: ${SOURCE} -> main"

      # Guards must run from the REPO ROOT — override the job's
      # defaults.run.working-directory: lang
      - name: Guard - single canonical runtime source
        working-directory: ${{ github.workspace }}
        run: bash scripts/check-single-runtime.sh

      - name: Guard - el_runtime.c growth budget
        working-directory: ${{ github.workspace }}
        run: bash scripts/check-runtime-growth.sh

      - name: Install build dependencies
        run: |
          apt-get update -qq
          apt-get install -y gcc libcurl4-openssl-dev

      # Seed: use the committed linux-amd64 binary as the bootstrap
      - name: Bootstrap from committed linux binary (seed)
        run: |
          chmod +x dist/platform/elc-linux-amd64
          echo "seed elc (committed linux-amd64 binary)"
          dist/platform/elc-linux-amd64 --version || true

      # Gen2: use seed to self-host compile the El compiler
      - name: Self-host compile El compiler (gen2)
        run: |
          mkdir -p dist/platform
          dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
          gcc -O2 \
            -I runtime \
            dist/elc-gen2.c \
            $(../scripts/el-runtime-sources.sh runtime) \
            -lcurl -lssl -lcrypto -lpthread -lm \
            -o dist/platform/elc
          chmod +x dist/platform/elc
          echo "gen2 (self-hosted) elc built"
          dist/platform/elc --version || true

      # Build elb binary
      - name: Build elb
        run: |
          mkdir -p dist/bin
          dist/platform/elc elb.el > dist/elb.c
          gcc -O2 \
            -I runtime \
            dist/elb.c \
            $(../scripts/el-runtime-sources.sh runtime) \
            -lcurl -lssl -lcrypto -lpthread -lm \
            -o dist/bin/elb
          chmod +x dist/bin/elb
          echo "elb built"

      # Build epm binary using elb (epm lives at repo root, not inside lang/)
      - name: Build epm
        run: |
          ABS_ELB="$(pwd)/dist/bin/elb"
          ABS_ELC="$(pwd)/dist/platform/elc"
          ABS_RUNTIME="$(pwd)/runtime"
          ABS_OUT="$(pwd)/dist/bin"
          (cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
          chmod +x dist/bin/epm
          echo "epm built"

      # Build el-install binary using elb
      - name: Build el-install
        run: |
          ABS_ELB="$(pwd)/dist/bin/elb"
          ABS_ELC="$(pwd)/dist/platform/elc"
          ABS_RUNTIME="$(pwd)/runtime"
          ABS_OUT="$(pwd)/dist/bin"
          (cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
          chmod +x dist/bin/el-install
          echo "el-install built"

      - name: Run tests - text
        run: |
          ELC="$(pwd)/dist/platform/elc" \
          EL_HOME="$(pwd)" \
          bash tests/text/run.sh

      - name: Run tests - calendar
        run: |
          ELC="$(pwd)/dist/platform/elc" \
          EL_HOME="$(pwd)" \
          bash tests/calendar/run.sh

      - name: Run tests - time
        run: |
          ELC="$(pwd)/dist/platform/elc" \
          EL_HOME="$(pwd)" \
          bash tests/time/run.sh

      - name: Run tests - html_sanitizer
        run: |
          ELC="$(pwd)/dist/platform/elc" \
          EL_HOME="$(pwd)" \
          bash tests/html_sanitizer/run.sh

      # Native El test suites (elc --test, compile-link-run)
      - name: Run tests - native (core)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
          /tmp/el_native_core

      - name: Run tests - native (text)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
          /tmp/el_native_text

      - name: Run tests - native (string)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
          /tmp/el_native_string

      - name: Run tests - native (math)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
          /tmp/el_native_math

      - name: Run tests - native (state)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
          /tmp/el_native_state

      - name: Run tests - native (time)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
          /tmp/el_native_time

      - name: Run tests - native (json)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
          /tmp/el_native_json

      - name: Run tests - native (env)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
          /tmp/el_native_env

      - name: Run tests - native (fs)
        run: |
          set -euo pipefail
          ELC="$(pwd)/dist/platform/elc"
          RUNTIME="$(pwd)/runtime"
          "$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
          gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
            -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
          /tmp/el_native_fs

      # Bundle the SDK tarball - runs from the repo root to reference lang/ paths correctly
      - name: Bundle SDK tarball
        if: github.event_name == 'push'
        working-directory: ${{ github.workspace }}
        run: |
          mkdir -p dist/sdk/bin dist/sdk/runtime
          cp lang/dist/platform/elc          dist/sdk/bin/elc
          cp lang/dist/bin/elb               dist/sdk/bin/elb
          cp lang/dist/bin/epm               dist/sdk/bin/epm
          # Ship the WHOLE runtime link set, not el_runtime.c alone. el_runtime.c
          # #includes six engram headers and calls into all six sibling .c files,
          # so an SDK carrying only el_runtime.c{,.h} + engram_store.c{,.h} cannot
          # link — downstream `ld` fails on engram_ground_json, eg_find_relation,
          # cog_assert_two_axis and friends. lang/runtime/SOURCES is the source of
          # truth; --check makes a missing file fail the release loudly.
          for f in $(scripts/el-runtime-sources.sh --check) \
                   $(scripts/el-runtime-sources.sh --headers --check); do
            cp "lang/runtime/${f}" dist/sdk/runtime/
          done
          cp lang/runtime/SOURCES            dist/sdk/runtime/
          cp lang/runtime/*.el               dist/sdk/runtime/
          tar -czf dist/el-sdk-latest.tar.gz -C dist/sdk .
          echo "SDK tarball bundled: dist/el-sdk-latest.tar.gz"
          ls -lh dist/el-sdk-latest.tar.gz

      # Publish / update the `latest` release with all SDK assets
      - name: Publish latest release
        if: github.event_name == 'push'
        working-directory: ${{ github.workspace }}
        env:
          GITEA_TOKEN: ${{ secrets.GIT_TOKEN }}
          GITEA_API: https://git.neuralplatform.ai/api/v1
          REPO: neuron-technologies/el
        run: |
          EXISTING_ID=$(curl -sf \
            -H "Authorization: token ${GITEA_TOKEN}" \
            "${GITEA_API}/repos/${REPO}/releases/tags/latest" \
            | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['id'])" 2>/dev/null || true)

          if [ -n "${EXISTING_ID}" ]; then
            echo "Deleting existing release id=${EXISTING_ID}"
            curl -sf -X DELETE \
              -H "Authorization: token ${GITEA_TOKEN}" \
              "${GITEA_API}/repos/${REPO}/releases/${EXISTING_ID}"
          fi

          curl -sf -X DELETE \
            -H "Authorization: token ${GITEA_TOKEN}" \
            "${GITEA_API}/repos/${REPO}/tags/latest" || true

          RELEASE_ID=$(curl -sf -X POST \
            -H "Authorization: token ${GITEA_TOKEN}" \
            -H "Content-Type: application/json" \
            "${GITEA_API}/repos/${REPO}/releases" \
            -d "{
              \"tag_name\": \"latest\",
              \"name\": \"El SDK (latest)\",
              \"body\": \"Latest El SDK build from commit ${GITHUB_SHA}.\nBuilt $(date -u +%Y-%m-%dT%H:%M:%SZ).\",
              \"draft\": false,
              \"prerelease\": false
            }" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")

          echo "Created release id=${RELEASE_ID}"

          upload_asset() {
            local filepath="$1"
            local name="$2"
            echo "Uploading ${name}..."
            curl -sf -X POST \
              -H "Authorization: token ${GITEA_TOKEN}" \
              -F "attachment=@${filepath};filename=${name}" \
              "${GITEA_API}/repos/${REPO}/releases/${RELEASE_ID}/assets"
          }

          # Per-file assets (downstream CI needs these individually).
          # lang/install.sh downloads every one of these by name — the list is
          # lang/runtime/SOURCES. Shipping el_runtime.c alone produced a lib/
          # that could not link; that is the bug this loop closes.
          upload_asset lang/dist/platform/elc                elc
          for f in $(scripts/el-runtime-sources.sh --check) \
                   $(scripts/el-runtime-sources.sh --headers --check); do
            upload_asset "lang/runtime/${f}" "${f}"
          done
          upload_asset lang/runtime/SOURCES SOURCES

          # SDK bundle and installer binary
          upload_asset dist/el-sdk-latest.tar.gz             el-sdk-latest.tar.gz
          upload_asset lang/dist/bin/el-install              el-install

          echo "Release published successfully"

      - name: Publish El SDK to Artifact Registry (prod)
        if: github.event_name == 'push'
        env:
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
        run: |
          # Fail loudly: previously this step had no `set -e`, so an auth or
          # upload failure was swallowed (step exited 0 on the trailing echo)
          # and the SDK silently never published. Surface failures now.
          set -euo pipefail
          if [ -z "${GCP_SA_KEY:-}" ]; then
            echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
            exit 1
          fi
          echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
          apt-get install -y -qq apt-transport-https ca-certificates curl
          echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
          apt-get update -qq && apt-get install -y google-cloud-cli
          gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
          gcloud config set project neuron-785695
          echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"

          VERSION="${GITHUB_SHA:0:8}"

          gcloud artifacts generic upload \
            --repository=foundation-prod \
            --location=us-central1 \
            --project=neuron-785695 \
            --package=el-elc \
            --version="${VERSION}" \
            --source=dist/platform/elc

          gcloud artifacts generic upload \
            --repository=foundation-prod \
            --location=us-central1 \
            --project=neuron-785695 \
            --package=el-elb \
            --version="${VERSION}" \
            --source=dist/bin/elb

          gcloud artifacts generic upload \
            --repository=foundation-prod \
            --location=us-central1 \
            --project=neuron-785695 \
            --package=el-runtime-c \
            --version="${VERSION}" \
            --source=runtime/el_runtime.c

          gcloud artifacts generic upload \
            --repository=foundation-prod \
            --location=us-central1 \
            --project=neuron-785695 \
            --package=el-runtime-h \
            --version="${VERSION}" \
            --source=runtime/el_runtime.h

          gcloud artifacts generic upload \
            --repository=foundation-prod \
            --location=us-central1 \
            --project=neuron-785695 \
            --package=el-runtime-js \
            --version="${VERSION}" \
            --source=runtime/el_runtime.js

          # el-runtime-src — the COMPLETE runtime link set as one tarball.
          #
          # The el-runtime-c / el-runtime-h packages above are single files and are
          # kept for backward compatibility with consumers that already pull them,
          # but they are NOT sufficient to link: el_runtime.c calls into six engram
          # sibling translation units. New consumers should pull el-runtime-src and
          # link everything named in its SOURCES file.
          tar -czf /tmp/el-runtime-src.tar.gz \
            -C runtime SOURCES \
            $(../scripts/el-runtime-sources.sh --check) \
            $(../scripts/el-runtime-sources.sh --headers --check)

          gcloud artifacts generic upload \
            --repository=foundation-prod \
            --location=us-central1 \
            --project=neuron-785695 \
            --package=el-runtime-src \
            --version="${VERSION}" \
            --source=/tmp/el-runtime-src.tar.gz

          echo "Published El SDK version=${VERSION} to foundation-prod"
          # Keep key alive for the ci-base rebuild step below
          # (deleted in that step after docker push)

      - name: Rebuild ci-base with fresh El SDK
        # Patches ci-base:latest in-place: pulls the existing image (which has all
        # system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
        # built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
        #
        # continue-on-error: this is a CI-cache optimization, NOT the release
        # artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
        # runner where DinD/Docker availability is fragile. A failure here must
        # never block or redden the job — the SDK publish above is the deliverable.
        continue-on-error: true
        if: github.event_name == 'push'
        env:
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
        run: |
          set -euo pipefail
          CI_BASE="us-central1-docker.pkg.dev/neuron-785695/neuron-ci/ci-base"
          SHA="${GITHUB_SHA:0:8}"

          echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
          gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
          gcloud config set project neuron-785695
          gcloud auth configure-docker us-central1-docker.pkg.dev --quiet

          # Pull existing ci-base (system deps stay cached in the base layer)
          docker pull "${CI_BASE}:latest"

          # Inline Dockerfile — only replaces the El SDK layer
          cat > /tmp/Dockerfile.ci-base-patch << 'EOF'
          ARG BASE
          FROM ${BASE}
          COPY dist/platform/elc                 /opt/el/dist/platform/elc
          COPY dist/bin/elb                       /opt/el/dist/bin/elb
          # Whole runtime link set — el_runtime.c alone does not link (it calls
          # into the six engram sibling TUs). See lang/runtime/SOURCES.
          COPY runtime/ /opt/el/runtime/
          COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
          RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
          EOF

          docker build \
            --build-arg BASE="${CI_BASE}:latest" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            -f /tmp/Dockerfile.ci-base-patch \
            -t "${CI_BASE}:latest" \
            -t "${CI_BASE}:${SHA}" \
            .

          docker push "${CI_BASE}:latest"
          docker push "${CI_BASE}:${SHA}"

          echo "ci-base rebuilt: ${CI_BASE}:latest (${SHA})"
          rm -f /tmp/gcp-key.json

      - name: Dispatch el-sdk-updated to downstream repos
        if: github.event_name == 'push'
        env:
          GITEA_TOKEN: ${{ secrets.GIT_TOKEN }}
          GITEA_API: https://git.neuralplatform.ai/api/v1
        run: |
          for repo in neuron-technologies/forge neuron-technologies/neuron-web; do
            curl -sf -X POST \
              -H "Authorization: token ${GITEA_TOKEN}" \
              -H "Content-Type: application/json" \
              "${GITEA_API}/repos/${repo}/dispatches" \
              -d "{
                \"type\": \"el-sdk-updated\",
                \"inputs\": {\"el_version\": \"latest\", \"commit\": \"${GITHUB_SHA}\"}
              }" && echo "Dispatched to ${repo}" || echo "Warning: dispatch to ${repo} failed"
          done