diff --git a/.gitea/workflows/ci-dev.yaml b/.gitea/workflows/ci-dev.yaml index b1e1d7b..d4194b8 100644 --- a/.gitea/workflows/ci-dev.yaml +++ b/.gitea/workflows/ci-dev.yaml @@ -214,9 +214,18 @@ jobs: env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} run: | + # Fail loudly: previously this step had no `set -e`, so an auth or + # upload failure was swallowed (step exited 0 on the trailing echo) + # and the SDK silently never published. Surface failures now. + set -euo pipefail + if [ -z "${GCP_SA_KEY:-}" ]; then + echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2 + exit 1 + fi echo "${GCP_SA_KEY}" > /tmp/gcp-key.json gcloud auth activate-service-account --key-file=/tmp/gcp-key.json gcloud config set project neuron-785695 + echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)" VERSION="${GITHUB_SHA:0:8}" @@ -268,6 +277,12 @@ jobs: # Patches ci-base:dev in-place: pulls the existing image (which has all # system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly # built El SDK on top. Keeps the full ci-base rebuild fast and incremental. + # + # continue-on-error: this is a CI-cache optimization, NOT the release + # artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE + # runner where DinD/Docker availability is fragile. A failure here must + # never block or redden the job — the SDK publish above is the deliverable. + continue-on-error: true if: github.event_name == 'push' env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} diff --git a/.gitea/workflows/ci-stage.yaml b/.gitea/workflows/ci-stage.yaml index cd1d9e8..3280050 100644 --- a/.gitea/workflows/ci-stage.yaml +++ b/.gitea/workflows/ci-stage.yaml @@ -212,12 +212,21 @@ jobs: env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} run: | + # Fail loudly: previously this step had no `set -e`, so an auth or + # upload failure was swallowed (step exited 0 on the trailing echo) + # and the SDK silently never published. Surface failures now. + set -euo pipefail + if [ -z "${GCP_SA_KEY:-}" ]; then + echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2 + exit 1 + fi echo "${GCP_SA_KEY}" > /tmp/gcp-key.json apt-get install -y -qq apt-transport-https ca-certificates curl echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list apt-get update -qq && apt-get install -y google-cloud-cli gcloud auth activate-service-account --key-file=/tmp/gcp-key.json gcloud config set project neuron-785695 + echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)" VERSION="${GITHUB_SHA:0:8}" @@ -253,6 +262,12 @@ jobs: # Patches ci-base:stage in-place: pulls the existing image (which has all # system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly # built El SDK on top. Keeps the full ci-base rebuild fast and incremental. + # + # continue-on-error: this is a CI-cache optimization, NOT the release + # artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE + # runner where DinD/Docker availability is fragile. A failure here must + # never block or redden the job — the SDK publish above is the deliverable. + continue-on-error: true if: github.event_name == 'push' env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} diff --git a/.gitea/workflows/sdk-release.yaml b/.gitea/workflows/sdk-release.yaml index 1d14870..d8eb0c7 100644 --- a/.gitea/workflows/sdk-release.yaml +++ b/.gitea/workflows/sdk-release.yaml @@ -292,12 +292,21 @@ jobs: env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} run: | + # Fail loudly: previously this step had no `set -e`, so an auth or + # upload failure was swallowed (step exited 0 on the trailing echo) + # and the SDK silently never published. Surface failures now. + set -euo pipefail + if [ -z "${GCP_SA_KEY:-}" ]; then + echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2 + exit 1 + fi echo "${GCP_SA_KEY}" > /tmp/gcp-key.json apt-get install -y -qq apt-transport-https ca-certificates curl echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list apt-get update -qq && apt-get install -y google-cloud-cli gcloud auth activate-service-account --key-file=/tmp/gcp-key.json gcloud config set project neuron-785695 + echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)" VERSION="${GITHUB_SHA:0:8}" @@ -349,6 +358,12 @@ jobs: # Patches ci-base:latest in-place: pulls the existing image (which has all # system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly # built El SDK on top. Keeps the full ci-base rebuild fast and incremental. + # + # continue-on-error: this is a CI-cache optimization, NOT the release + # artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE + # runner where DinD/Docker availability is fragile. A failure here must + # never block or redden the job — the SDK publish above is the deliverable. + continue-on-error: true if: github.event_name == 'push' env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} diff --git a/README.md b/README.md new file mode 100644 index 0000000..ef76ffa --- /dev/null +++ b/README.md @@ -0,0 +1,154 @@ +# El + +**A self-hosting, statically-typed language that compiles to C — built around a graph-native runtime instead of a database driver.** + +El is the execution substrate for the Neuron agent runtime, the DHARMA network, and the Engram knowledge graph. This repository is the monorepo for the whole stack: the language itself, the graph memory engine it's built to talk to natively, and the tools (package manager, IDE, UI framework, diagramming) built on top of it. + +--- + +## Why El exists + +Every other language treats persistent, associative state as something you reach for through a driver — a SQL client, an ORM, a Redis library bolted on from outside. El inverts that: graph operations (`engram_*`) are runtime primitives, on the same footing as string or list operations. There is no separate database driver because the database is not separate. + +El has four defining properties: + +1. **Self-hosting compiler.** The compiler (`lexer.el`, `parser.el`, `codegen.el`, `compiler.el`) is written in El. It compiles El source to C, which `cc` compiles against a fixed runtime into a native binary. A Rust genesis compiler bootstrapped the first iteration; the self-hosted binary at `lang/dist/platform/elc` has been the canonical compiler ever since — every binary in `dist/platform/` was produced by an earlier version of itself compiling `el-compiler/src/`. The chain is auditable: source is the ground truth, not the binary. See [lang/BOOTSTRAP.md](lang/BOOTSTRAP.md) for the full recovery path if that binary is ever lost. +2. **C compilation target.** Every compiled program is plain C11. Every El value is `el_val_t` (`int64_t`); strings are heap pointers cast through it. Functions become C functions; top-level statements become `main()`. +3. **Graph-native runtime.** The runtime provides first-class graph operations over an in-process Engram store — no separate DB driver, no ORM. +4. **DHARMA-aware identity.** A `cgi` block declares a program's DHARMA identity at compile time. The runtime resolves identity before user code runs, so `dharma_*` calls have a stable principal and channel surface throughout. + +--- + +## Architecture map + +``` + ┌─────────────┐ + │ lang │ El compiler + C runtime + │ (El itself) │ everything below is written in it, + └──────┬──────┘ or compiles down through it + │ + ┌─────────────┼─────────────┐ + │ │ │ + ┌──────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐ + │ engram │ │ epm │ │ ide │ + │ graph/mem │ │ package │ │ editor + │ + │ substrate │ │ manager │ │ LSP │ + └──────┬─────┘ └───────────┘ └───────────┘ + │ + ┌───────┼────────────────┬─────────────────────┐ + │ │ │ │ +┌─────▼───┐ ┌─▼──────────┐ ┌──▼──────────┐ ┌─────▼──────┐ +│ elp │ │ ql │ │ ui │ │ arbor │ +│ NLG / │ │engram-el. │ |spreading- │ |arbor │ +│ 31 langs│ │studio+tests│ |activation UI│ |diagram lang│ +└─────────┘ └────────────┘ └─────────────┘ └────────────┘ +``` + +`lang` is the foundation — the compiler and C runtime everything else builds on. `engram` is the graph-native memory/state engine that gives El its identity (property 3 above). Everything else is either a tool for working with El (`epm`, `ide`) or a system built on top of Engram's graph model (`elp`, `ql`, `ui`, `arbor`). + +--- + +## Repository layout + +### [lang/](lang/) — the El language + +The compiler and runtime. Self-hosting: `elc-cli.el` → `compiler.el` → `lexer.el` / `parser.el` / `codegen.el` / `codegen-js.el`, textually inlined and compiled in one pass. Compiles to C11 and links against `el-compiler/runtime/el_seed.c`, a hand-maintained OS-boundary layer (libcurl HTTP, pthreads, filesystem, arena allocation) — everything else in the runtime is native El (`runtime/*.el`). + +Two layers to know: **El programs** (`.el` files — where nearly all work belongs) and **the C seed** (`el_seed.c` — edit only for genuine OS-level access; never re-implement what El can already express). + +Current status (single source of truth: [lang/spec/language.md](lang/spec/language.md)): lexer/parser/codegen and the C runtime's core (I/O, strings, math, lists, maps, filesystem, args) are implemented. In flight: `%` operator, match-statement codegen, `?` nil-propagation, `cgi` block parsing + DHARMA identity resolution, VBD role enforcement (`@manager`/`@engine`/`@accessor`), the real `engram_*` and `dharma_*` runtimes (currently stubs), and libcurl-backed `http_get`/`http_post`/`http_serve`. Bitwise operators, `??`, and `as` casts are explicitly **not** in this language. + +Key docs: [AGENTS.md](lang/AGENTS.md) (agent-facing orientation), [BOOTSTRAP.md](lang/BOOTSTRAP.md) (compiler recovery from scratch), [spec/language.md](lang/spec/language.md), [spec/codegen-js.md](lang/spec/codegen-js.md). + +### [engram/](engram/) — graph intelligence substrate + +**A local-first memory substrate for accumulating intelligence**, and the reason El's runtime doesn't need a database driver. Rust core (`engram-core`, `engram-ffi`) exposed to El and other languages (Kotlin, TypeScript/WASM, Go bindings). + +The model: retrieval is **spreading activation**, not query. You name seed nodes and a query embedding; activation propagates outward through weighted edges, attenuating multiplicatively per hop (`strength = parent_strength × edge_weight × target_salience × cosine_sim`), gets pruned below a threshold, and the top-N nodes by activation strength come back. Storage and retrieval are the same structure — the way long-term potentiation works in biological memory, not the way a relational or vector database works. + +Nodes live in four tiers (Working / Episodic / Semantic / Procedural, mirroring prefrontal / hippocampal / neocortical / cerebellar memory) and migrate between them based on **salience decay** — `importance × recency-decay × log(activation_count)`. Forgetting is adaptive pruning, not a bug: unreinforced memories stop competing for attention without being deleted. + +Backed by `sled` (embedded, local-first, no daemon) with flat cosine scan for vector search — deliberately simple until scale demands an HNSW layer. Full API and design rationale in [engram/README.md](engram/README.md). + +### [elp/](elp/) — Engram Language Protocol + +Bidirectional engine mapping between Engram semantic forms and natural-language surface text, across **31 languages** — from Spanish and Japanese through historical/liturgical languages (Old Norse, Sanskrit, Sumerian, Coptic, Akkadian, Ge'ez). Compilation order runs `language-profile` + `vocabulary` → per-language `morphology-*` → `grammar` → `realizer` → `semantics` → `elp`. This is what lets an Engram graph node round-trip to and from readable text in any of those languages. + +### [epm/](epm/) — El Package Manager + +Manages **vessels** (El's package unit): publish, install, resolve dependencies. Vessels are stored in Engram as graph nodes, not files in a registry index — `epm` reads the local `manifest.el`, talks to Engram over HTTP, and writes resolved vessels to `.epm/vessels/`. Source: `registry.el`, `install.el`, `update.el`, `manifest.el`. + +### [ide/](ide/) — El IDE + +Three vessels: **el-ide-server** (HTTP backend — file ops, build/run, LSP bridge, plugin host, settings), **el-lsp** (the language server — completion, hover, diagnostics, outline, format, type graph), and **el-plugin-host** (first-party plugin lifecycle: install/remove/enable/disable). `ide/projects/` and `ide/examples/` hold sample projects, including the canonical `hello-friends` first-program walkthrough. + +### [ql/](ql/) — engram-el + +The El-native integration layer for a *live* Engram server — not a library (no importable modules, no build artifact), a set of standalone `.el` programs run directly via `el run-file`. Three components: **Studio** (`studio/studio.el`, a full terminal graph explorer), a **Hebbian field-model** proof of concept, and El builtin / LLM-builtin smoke test suites. This is the reference for correct patterns when an El program uses Engram as its substrate. Spec: [ql/spec/elql.md](ql/spec/elql.md). + +### [ui/](ui/) — el-ui + +A frontend framework where **component state is an Engram graph and reactivity is spreading activation** — not virtual-DOM diffing (React), Proxy-based dependency tracking (Vue), or compile-time analysis (Svelte). Re-renders are activated and propagated the same way associative memory retrieval works in `engram/`. + +~15 vessels covering the full frontend surface: `el-platform` (env/fs/network/clock abstraction), `el-config`, `el-html` (SSR emit primitives), `el-layout`, `el-style` (design tokens/themes), `el-i18n`, `el-auth` / `el-identity` (JWT, sessions, OAuth PKCE — Engram-native), `el-services` (REST/gRPC/WebSocket bindings), `el-aop` (`@authenticate`/`@authorize`/`@cache`/`@rate_limit` decorators), `el-secrets`, `el-graph` (graph rendering/editor), `el-publish` (App Store / Play Store automation), and `el-ui-compiler` (El→JS component compiler; currently a stub pending a JS backend in `elc`). Spec: [ui/spec/framework.md](ui/spec/framework.md). + +### [arbor/](arbor/) — diagram language + +A `.arbor` diagram language and toolchain: `arbor-core` (NodeId/shape/edge-kind types), `arbor-parse` (recursive-descent parser), `arbor-diagram` (IR + Mermaid serializer + architecture-diagram builders), `arbor-layout` (hierarchical layout — rank assignment, positioning, group bounds), `arbor-render` (SVG renderer), `arbor-cli`. (The architecture map above is the kind of diagram this is for.) + +--- + +## Getting started + +Install the El SDK from the latest release: + +```bash +bash lang/install.sh +# EL_VERSION=v1.0.0 bash lang/install.sh # pin a specific release tag +# EL_PREFIX=/opt/el bash lang/install.sh # custom install prefix +``` + +Or build the compiler from source and verify the self-hosting chain: + +```bash +cd lang +./dist/platform/elc elc-cli.el > elc-new.c +cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \ + -o dist/platform/elc-new \ + elc-new.c el-compiler/runtime/el_seed.c + +# Confirm the new binary reproduces itself exactly +./dist/platform/elc-new elc-cli.el > elc-verify.c +diff elc-new.c elc-verify.c # should be identical + +mv dist/platform/elc-new dist/platform/elc +``` + +Run your first program: + +```bash +./lang/dist/platform/elc lang/examples/hello.el > hello.c +cc -std=c11 -I lang/el-compiler/runtime -lcurl -lpthread \ + -o hello hello.c lang/el-compiler/runtime/el_seed.c +./hello +``` + +More examples in [lang/examples/](lang/examples/), including a full starter project at `lang/examples/hello-project/`. + +If the compiler binary is ever lost or corrupted, [lang/BOOTSTRAP.md](lang/BOOTSTRAP.md) is the authoritative recovery path. + +--- + +## Development workflow + +Branching follows `dev → stage → main`: work lands on `dev`, promotes to `stage` for integration testing, and is promoted to `main` for release (visible directly in the git history of this repo). CI is defined per-subproject under `.gitea/workflows/` — `lang`/`epm`/`ide` share the root pipeline; `engram` and `ql` carry their own (`ci-dev`, `ci-stage`, and a release workflow each). + +- Language/runtime specs live at `*/spec/*.md` (`lang/spec/`, `ql/spec/`, `ui/spec/`) and are the single source of truth for implemented-vs-planned status — code and docs are expected to agree with the spec's status markers, not the other way around. +- Agent-facing orientation guides live at `*/AGENTS.md` (currently `lang/AGENTS.md`); more subprojects may grow their own as they need agent-specific conventions documented. +- Tagged releases live under `lang/releases/`, each with its own `RELEASE.md`. + +--- + +## Status + +This is an actively developed, internal monorepo — not yet published under an open license. Treat everything here as proprietary to Neuron Technologies unless told otherwise. diff --git a/engram/.gitea/workflows/ci-dev.yaml b/engram/.gitea/workflows/ci-dev.yaml index 7463aa8..a432869 100644 --- a/engram/.gitea/workflows/ci-dev.yaml +++ b/engram/.gitea/workflows/ci-dev.yaml @@ -81,7 +81,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/engram/.gitea/workflows/ci-stage.yaml b/engram/.gitea/workflows/ci-stage.yaml index 69ed162..a134117 100644 --- a/engram/.gitea/workflows/ci-stage.yaml +++ b/engram/.gitea/workflows/ci-stage.yaml @@ -88,7 +88,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/engram/.gitea/workflows/engram-release.yaml b/engram/.gitea/workflows/engram-release.yaml index ffe0e50..fcd241f 100644 --- a/engram/.gitea/workflows/engram-release.yaml +++ b/engram/.gitea/workflows/engram-release.yaml @@ -68,7 +68,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/epm/src/install.el b/epm/src/install.el index 882fc90..1561cf8 100644 --- a/epm/src/install.el +++ b/epm/src/install.el @@ -17,6 +17,16 @@ // 4. Append dep to order after all its transitive deps // 5. Deduplicate: skip already-ordered vessels +// ── Cross-module forward declarations ───────────────────────────────────────── +// Defined in sibling epm modules; resolved at link time. The `extern fn` decls +// give elc the C prototypes so generated install.c compiles cleanly under strict +// compilers (gcc>=14 / clang) that reject implicit function declarations. +extern fn manifest_name(src: String) -> String // manifest.el +extern fn manifest_deps(src: String) -> String // manifest.el +extern fn registry_token() -> String // registry.el +extern fn registry_find(name: String, version: String) -> String // registry.el +extern fn registry_latest_version(name: String) -> String // registry.el + // ── Install paths ───────────────────────────────────────────────────────────── // packages_dir returns the root directory for installed vessels. diff --git a/epm/src/registry.el b/epm/src/registry.el index 8e1b2b5..d6f9730 100644 --- a/epm/src/registry.el +++ b/epm/src/registry.el @@ -14,6 +14,15 @@ // EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies) // EPM_TOKEN — Gitea personal access token (required for publish) +// ── Cross-module forward declarations ───────────────────────────────────────── +// These symbols are defined in sibling epm modules or the El runtime and are +// resolved at link time. The `extern fn` decls give elc the C prototype so the +// generated registry.c compiles cleanly under strict compilers (gcc>=14 / clang) +// that reject implicit function declarations. Signature arity must match the +// definition; return/param types are informational (all lower to el_val_t). +extern fn config(key: String) -> String // El runtime builtin +extern fn read_installed() -> String // install.el + // ── Config helpers ──────────────────────────────────────────────────────────── // registry_api_url returns the Gitea API base URL with no trailing slash. diff --git a/epm/src/update.el b/epm/src/update.el index 42f60a5..3804ab2 100644 --- a/epm/src/update.el +++ b/epm/src/update.el @@ -6,6 +6,15 @@ // Depends on: registry.el (registry_latest_version, registry_find), // install.el (read_installed, install_vessel, installed_version) +// ── Cross-module forward declarations ───────────────────────────────────────── +// Defined in sibling epm modules; resolved at link time. The `extern fn` decls +// give elc the C prototypes so generated update.c compiles cleanly under strict +// compilers (gcc>=14 / clang) that reject implicit function declarations. +extern fn read_installed() -> String // install.el +extern fn installed_version(name: String) -> String // install.el +extern fn install_vessel(name: String, version: String) -> Bool // install.el +extern fn registry_latest_version(name: String) -> String // registry.el + // ── Semver helpers ──────────────────────────────────────────────────────────── // semver_part extracts the Nth dot-separated component from a semver string. diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index f6290dd..8b72adf 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2727,6 +2727,7 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "engram_store_checkpoint") { return 0 } if str_eq(name, "engram_store_close") { return 0 } if str_eq(name, "engram_get_node_json") { return 1 } + if str_eq(name, "engram_get_node_by_label") { return 1 } if str_eq(name, "engram_search_json") { return 2 } if str_eq(name, "engram_scan_nodes_json") { return 2 } if str_eq(name, "engram_neighbors_json") { return 3 } diff --git a/lang/runtime/el_platform_win.h b/lang/runtime/el_platform_win.h index 88204a0..cfc2774 100644 --- a/lang/runtime/el_platform_win.h +++ b/lang/runtime/el_platform_win.h @@ -50,6 +50,16 @@ build defines the same helper as close() so the call sites are identical across platforms. */ static inline int el_closesocket(SOCKET s) { return closesocket(s); } +/* ── setsockopt optval type ───────────────────────────────────────────────── */ +/* Winsock's setsockopt takes optval as (const char*); POSIX takes (const void*), so el_runtime.c + passes &int directly. GCC 14+ makes that an error under -Wincompatible-pointer-types. Wrap it so + the runtime's POSIX-style call sites compile unchanged (defined before the macro so the wrapper + itself resolves to the real winsock setsockopt). */ +static inline int el_setsockopt(SOCKET s, int level, int optname, const void* optval, int optlen) { + return setsockopt(s, level, optname, (const char*)optval, optlen); +} +#define setsockopt(s, l, o, v, n) el_setsockopt((s), (l), (o), (v), (int)(n)) + /* ── winsock init (once, at load) ─────────────────────────────────────────── */ static void el__win_net_init(void) { static int inited = 0; @@ -75,6 +85,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) { #include /* _mkdir */ #define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */ #define timegm _mkgmtime /* UTC tm → time_t */ +#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() () is the equiv */ /* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */ static inline int setenv(const char* name, const char* value, int overwrite) { @@ -114,4 +125,63 @@ static inline struct tm* gmtime_r(const time_t* t, struct tm* out) { return gmtime_s(out, t) == 0 ? out : (struct tm*)0; } +/* ── libcurl: degradable stubs for the curl-less Windows build ─────────────── */ +/* The curl-less validation build (WITH_CURL=0) links no libcurl. el_runtime.c uses libcurl + * unconditionally for its HTTP client / LLM layer; these stubs let it compile and link so the + * runtime, HTTP *server*, graph and memory work natively on Windows. Live outbound HTTP/LLM calls + * degrade to a runtime error (curl_easy_perform returns an error) — matching the documented + * curl-less contract. When HAVE_CURL is defined (WITH_CURL=1) the real is used and + * this whole block is compiled out. POSIX never sees this header, so the POSIX build is untouched. */ +#ifndef HAVE_CURL + +typedef void CURL; +typedef int CURLcode; + +#define CURLE_OK 0 +#define CURLE_HTTP_RETURNED_ERROR 22 +#define CURL_ERROR_SIZE 256 + +/* Option ids: values are irrelevant to the no-op setopt below; kept distinct for readability. */ +#define CURLOPT_URL 10002 +#define CURLOPT_WRITEFUNCTION 20011 +#define CURLOPT_WRITEDATA 10001 +#define CURLOPT_POSTFIELDS 10015 +#define CURLOPT_POSTFIELDSIZE 120 +#define CURLOPT_POST 47 +#define CURLOPT_HTTPHEADER 10023 +#define CURLOPT_TIMEOUT_MS 155 +#define CURLOPT_NOSIGNAL 99 +#define CURLOPT_USERAGENT 10018 +#define CURLOPT_FOLLOWLOCATION 52 +#define CURLOPT_ERRORBUFFER 10010 +#define CURLOPT_CUSTOMREQUEST 10036 +#define CURLOPT_FAILONERROR 45 + +struct curl_slist { char* data; struct curl_slist* next; }; + +static inline struct curl_slist* curl_slist_append(struct curl_slist* list, const char* s) { + struct curl_slist* node = (struct curl_slist*)malloc(sizeof(struct curl_slist)); + if (!node) return list; + node->data = s ? strdup(s) : NULL; + node->next = NULL; + if (!list) return node; + struct curl_slist* p = list; + while (p->next) p = p->next; + p->next = node; + return list; +} +static inline void curl_slist_free_all(struct curl_slist* list) { + while (list) { struct curl_slist* n = list->next; free(list->data); free(list); list = n; } +} + +static inline CURL* curl_easy_init(void) { return (CURL*)malloc(1); } +static inline CURLcode curl_easy_setopt(CURL* h, int opt, ...) { (void)h; (void)opt; return CURLE_OK; } +static inline CURLcode curl_easy_perform(CURL* h) { (void)h; return 7 /* CURLE_COULDNT_CONNECT */; } +static inline void curl_easy_cleanup(CURL* h) { free(h); } +static inline const char* curl_easy_strerror(CURLcode c) { + (void)c; return "libcurl not built in (curl-less build)"; +} + +#endif /* !HAVE_CURL */ + #endif /* EL_PLATFORM_WIN_H */ diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index e3a2de5..7012a3d 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -21,6 +21,10 @@ #include "el_runtime.h" +#ifdef _WIN32 +/* Windows OS-boundary shim (winsock/dlsym/popen). Threading stays on (winpthreads). */ +#include "el_platform_win.h" +#else #include #include /* strcasecmp */ #include @@ -42,7 +46,16 @@ #include #include #include +/* On POSIX, sockets close with the same close() as files; el_platform_win.h supplies the Windows + variant. Defined here so the socket call sites are identical across platforms. */ +static inline int el_closesocket(int s) { return close(s); } +#endif +/* libcurl: present on POSIX and on the WITH_CURL Windows build; absent on the curl-less Windows + validation build, where el_platform_win.h supplies degradable stubs. On POSIX (_WIN32 undefined) + this is always taken, so the POSIX build is unchanged. */ +#if !defined(_WIN32) || defined(HAVE_CURL) #include +#endif /* ── Internal allocators ─────────────────────────────────────────────────── */ @@ -71,8 +84,14 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; static _Thread_local int _tl_arena_active = 0; /* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. - * Allows serving PNGs and other binary files without strlen truncation. */ -static _Thread_local size_t _tl_fs_read_len = 0; + * Allows serving PNGs and other binary files without strlen truncation. + * PAIRED with the buffer pointer it describes: the length may only be applied + * to the exact buffer fs_read returned. Without the pairing, any handler that + * fs_read a file and then WRAPPED it into a larger response had that response + * truncated to the file's length (Content-Length lied AND the send stopped + * short) — the safety-contact onboarding trap, 2026-07-17. */ +static _Thread_local size_t _tl_fs_read_len = 0; +static _Thread_local const char* _tl_fs_read_buf = NULL; static void el_arena_track(char* p) { if (!_tl_arena_active || !p) return; @@ -90,6 +109,8 @@ static void el_arena_track(char* p) { void el_request_start(void) { _tl_arena.count = 0; _tl_arena_active = 1; + _tl_fs_read_len = 0; /* never let a previous request's file length */ + _tl_fs_read_buf = NULL; /* leak into this response's byte accounting */ } /* Called by http_worker after the El handler returns and the response is sent. @@ -1430,11 +1451,14 @@ static void http_send_response(int fd, const char* body) { } const char* eff_body = is_envelope ? env_body : body; - /* Use the real byte count from fs_read if available (handles binary files - * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for - * normal text/JSON responses where _tl_fs_read_len is 0. */ - size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + /* Use the real byte count from fs_read ONLY when this body IS the exact + * buffer fs_read returned (binary files with embedded null bytes — PNG, + * WOFF2, etc.). Any other body — wrapped, enveloped, or derived — must be + * measured with strlen, or it is truncated/over-read to the file's size. */ + size_t blen = (_tl_fs_read_len > 0 && eff_body == _tl_fs_read_buf) + ? _tl_fs_read_len : strlen(eff_body); _tl_fs_read_len = 0; /* consume — one-shot per response */ + _tl_fs_read_buf = NULL; int head_only = _tl_http_head_only; JsonBuf hdrs; jb_init(&hdrs); @@ -1484,12 +1508,20 @@ static void http_send_response(int fd, const char* body) { } typedef struct { +#ifdef _WIN32 + SOCKET fd; +#else int fd; +#endif } HttpWorkerArg; static void* http_worker(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; +#ifdef _WIN32 + SOCKET fd = a->fd; +#else int fd = a->fd; +#endif free(a); char *method = NULL, *path = NULL, *body = NULL; if (http_read_request(fd, &method, &path, &body, NULL) == 0) { @@ -1506,11 +1538,22 @@ static void* http_worker(void* arg) { const char* rs = EL_CSTR(r); /* Copy response out BEFORE arena teardown. * For binary files, _tl_fs_read_len holds the real byte count — - * use memcpy instead of strdup so null bytes are preserved. */ - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + * use memcpy instead of strdup so null bytes are preserved. + * The stored length applies ONLY when the response IS the exact + * fs_read buffer; a wrapped/derived response must use strlen or + * it gets truncated (or over-read) to the file's length. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ } else { response = el_strdup_persist("el-runtime: no http handler registered"); } @@ -1521,7 +1564,7 @@ static void* http_worker(void* arg) { free(response); } free(method); free(path); free(body); - close(fd); + el_closesocket(fd); /* release a slot */ pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; @@ -1551,14 +1594,18 @@ void http_serve(el_val_t port, el_val_t handler) { addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { - perror("bind"); close(sock); return; + perror("bind"); el_closesocket(sock); return; } - if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + if (listen(sock, 64) < 0) { perror("listen"); el_closesocket(sock); return; } fprintf(stderr, "[http] listening on [::]:%d (dual-stack)\n", p); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); +#ifdef _WIN32 + SOCKET cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#else int cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#endif if (cfd < 0) { if (errno == EINTR) continue; perror("accept"); break; @@ -1570,11 +1617,11 @@ void http_serve(el_val_t port, el_val_t handler) { _http_conn_active++; pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); - if (!arg) { close(cfd); continue; } + if (!arg) { el_closesocket(cfd); continue; } arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker, arg) != 0) { - close(cfd); free(arg); + el_closesocket(cfd); free(arg); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); @@ -1583,7 +1630,7 @@ void http_serve(el_val_t port, el_val_t handler) { } pthread_detach(tid); } - close(sock); + el_closesocket(sock); } /* ── http_serve_async — non-blocking HTTP server ─────────────────────────── */ @@ -1819,7 +1866,11 @@ static el_val_t http_build_headers_map(const char* hdr_block) { static void* http_worker_v2(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; +#ifdef _WIN32 + SOCKET fd = a->fd; +#else int fd = a->fd; +#endif free(a); char *method = NULL, *path = NULL, *body = NULL, *hdr_block = NULL; if (http_read_request(fd, &method, &path, &body, &hdr_block) == 0) { @@ -1832,10 +1883,20 @@ static void* http_worker_v2(void* arg) { el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); const char* rs = EL_CSTR(r); - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + /* Same pairing rule as the v1 worker: the fs_read length is only + * trustworthy for the exact buffer fs_read returned. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ el_release(hmap); } else { response = el_strdup_persist( @@ -1849,7 +1910,7 @@ static void* http_worker_v2(void* arg) { free(response); } free(method); free(path); free(body); free(hdr_block); - close(fd); + el_closesocket(fd); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); @@ -1879,14 +1940,18 @@ void http_serve_v2(el_val_t port, el_val_t handler) { addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { - perror("bind"); close(sock); return; + perror("bind"); el_closesocket(sock); return; } - if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + if (listen(sock, 64) < 0) { perror("listen"); el_closesocket(sock); return; } fprintf(stderr, "[http v2] listening on [::]:%d (dual-stack)\n", p); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); +#ifdef _WIN32 + SOCKET cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#else int cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#endif if (cfd < 0) { if (errno == EINTR) continue; perror("accept"); break; @@ -1898,11 +1963,11 @@ void http_serve_v2(el_val_t port, el_val_t handler) { _http_conn_active++; pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); - if (!arg) { close(cfd); continue; } + if (!arg) { el_closesocket(cfd); continue; } arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) { - close(cfd); free(arg); + el_closesocket(cfd); free(arg); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); @@ -1911,7 +1976,7 @@ void http_serve_v2(el_val_t port, el_val_t handler) { } pthread_detach(tid); } - close(sock); + el_closesocket(sock); } /* Build the response envelope a 4-arg handler can return. We hand-write @@ -1951,6 +2016,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { el_val_t fs_read(el_val_t pathv) { const char* path = EL_CSTR(pathv); _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!path) return el_wrap_str(el_strdup("")); FILE* f = fopen(path, "rb"); if (!f) return el_wrap_str(el_strdup("")); @@ -1962,6 +2028,7 @@ el_val_t fs_read(el_val_t pathv) { size_t got = fread(buf, 1, (size_t)sz, f); buf[got] = '\0'; _tl_fs_read_len = got; /* store real byte count for binary-safe send */ + _tl_fs_read_buf = buf; /* ...valid ONLY for this exact buffer */ fclose(f); return el_wrap_str(buf); } @@ -2056,6 +2123,23 @@ el_val_t exec(el_val_t cmdv) { el_val_t exec_bg(el_val_t cmdv) { const char* cmd = EL_CSTR(cmdv); if (!cmd || !*cmd) return el_wrap_str(el_strdup("")); +#ifdef _WIN32 + /* Windows: no fork/exec. Launch a detached `cmd /c ` with no console window via + CreateProcess (DETACHED_PROCESS | CREATE_NO_WINDOW). Returns the PID as a string, "" on fail. + Mirrors the POSIX branch: child runs independently, caller is not blocked. */ + char cmdline[8192]; + snprintf(cmdline, sizeof(cmdline), "cmd.exe /c %s", cmd); + STARTUPINFOA si; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); + PROCESS_INFORMATION pi; ZeroMemory(&pi, sizeof(pi)); + BOOL ok = CreateProcessA(NULL, cmdline, NULL, NULL, FALSE, + DETACHED_PROCESS | CREATE_NO_WINDOW, NULL, NULL, &si, &pi); + if (!ok) return el_wrap_str(el_strdup("")); + char pidbuf[32]; + snprintf(pidbuf, sizeof(pidbuf), "%lu", (unsigned long)pi.dwProcessId); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + return el_wrap_str(el_strdup(pidbuf)); +#else pid_t pid = fork(); if (pid < 0) { /* fork failed */ @@ -2078,6 +2162,7 @@ el_val_t exec_bg(el_val_t cmdv) { char pidbuf[32]; snprintf(pidbuf, sizeof(pidbuf), "%d", (int)pid); return el_wrap_str(el_strdup(pidbuf)); +#endif } el_val_t fs_list(el_val_t pathv) { @@ -3417,8 +3502,10 @@ el_val_t json_get_raw(el_val_t json_str, el_val_t key) { const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); /* Clear fs_read binary-length hint — result is a fresh null-terminated - * string, not the raw file bytes, so Content-Length must use strlen. */ + * string, not the raw file bytes, so Content-Length must use strlen. + * (Kept although the pointer pairing now makes this redundant.) */ _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!p) return el_wrap_str(el_strdup("")); const char* end = json_skip_value(p); size_t n = (size_t)(end - p); @@ -4290,7 +4377,12 @@ static int _el_decompose_earth(el_caltime_t* ct, struct tm* tm_out, int* abbr_le localtime_r(&s, &tm); *tm_out = tm; if (abbr_buf && abbr_cap > 0) { + /* mingw's struct tm has no tm_zone (BSD/glibc extension); no abbrev available there. */ +#ifdef _WIN32 + const char* z_str = ""; +#else const char* z_str = tm.tm_zone ? tm.tm_zone : ""; +#endif size_t n = strlen(z_str); if (n >= abbr_cap) n = abbr_cap - 1; memcpy(abbr_buf, z_str, n); diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index d25f93a..aa4305b 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path) { return engram_save el_val_t __engram_load(el_val_t path) { return engram_load(path); } el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); } +el_val_t __engram_get_node_by_label(el_val_t label) { return engram_get_node_by_label(label); } el_val_t __engram_search_json(el_val_t query, el_val_t limit) { return engram_search_json(query, limit); diff --git a/lang/runtime/el_seed.h b/lang/runtime/el_seed.h index c502f10..7597511 100644 --- a/lang/runtime/el_seed.h +++ b/lang/runtime/el_seed.h @@ -226,6 +226,7 @@ el_val_t __engram_activate(el_val_t query, el_val_t depth); el_val_t __engram_save(el_val_t path); el_val_t __engram_load(el_val_t path); el_val_t __engram_get_node_json(el_val_t id); +el_val_t __engram_get_node_by_label(el_val_t label); el_val_t __engram_search_json(el_val_t query, el_val_t limit); el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset); el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);