diff --git a/engram/.DS_Store b/engram/.DS_Store new file mode 100644 index 0000000..f8575f5 Binary files /dev/null and b/engram/.DS_Store differ diff --git a/engram/.gitea/workflows/ci-dev.yaml b/engram/.gitea/workflows/ci-dev.yaml new file mode 100644 index 0000000..7463aa8 --- /dev/null +++ b/engram/.gitea/workflows/ci-dev.yaml @@ -0,0 +1,121 @@ +name: Engram CI — dev + +on: + push: + branches: + - dev + pull_request: + branches: + - dev + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install build dependencies + run: | + apt-get update -qq + apt-get install -y gcc libcurl4-openssl-dev + + # Install El SDK from dev artifact registry + - name: Install El SDK from dev registry + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + echo "${GCP_SA_KEY}" > /tmp/gcp-key.json + apt-get install -y -qq apt-transport-https ca-certificates gnupg curl + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update -qq && apt-get install -y google-cloud-cli + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json + gcloud config set project neuron-785695 + + mkdir -p /usr/local/bin /usr/local/lib/el + + # Download the latest-versioned elc from dev registry + # Fall back to the Gitea latest release if no dev artifact exists yet + LATEST_VERSION=$(gcloud artifacts versions list \ + --repository=foundation-dev \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=el/elc \ + --format="value(name)" 2>/dev/null | sort | tail -1 | sed 's|.*/||' || true) + + if [ -n "${LATEST_VERSION}" ]; then + echo "Downloading elc ${LATEST_VERSION} from foundation-dev..." + gcloud artifacts generic download \ + --repository=foundation-dev \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=el/elc \ + --version="${LATEST_VERSION}" \ + --destination=/usr/local/bin/elc + chmod +x /usr/local/bin/elc + else + echo "No dev artifact found, falling back to Gitea latest release..." + RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest" + curl -fsSL "${RELEASE_BASE}/elc" -o /usr/local/bin/elc + chmod +x /usr/local/bin/elc + fi + + # Always get runtime from Gitea latest release + RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest" + curl -fsSL "${RELEASE_BASE}/el_runtime.c" -o /usr/local/lib/el/el_runtime.c + curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h + + echo "El SDK installed:" + elc --version || true + rm -f /tmp/gcp-key.json + + # Compile server.el → dist/engram.c + - name: Compile server.el + run: | + mkdir -p dist + elc src/server.el > dist/engram.c + echo "Compiled src/server.el → dist/engram.c" + + # Link to produce the engram binary + - name: Link engram binary + run: | + cc -std=c11 -O2 \ + -I /usr/local/lib/el \ + -o dist/engram \ + dist/engram.c \ + /usr/local/lib/el/el_runtime.c \ + -lcurl -lpthread + echo "Linked dist/engram" + ls -lh dist/engram + + # Run tests if spec directory contains runnable tests + - name: Run tests + run: | + if [ -f spec/run.sh ]; then + bash spec/run.sh + else + echo "No spec/run.sh found — skipping test run" + fi + + # Publish artifact to GCP Artifact Registry (dev) + - name: Publish engram to Artifact Registry (dev) + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + echo "${GCP_SA_KEY}" > /tmp/gcp-key.json + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json + gcloud config set project neuron-785695 + + VERSION="${GITEA_SHA:0:8}" + gcloud artifacts generic upload \ + --repository=foundation-dev \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=engram/engram \ + --version="${VERSION}" \ + --source=dist/engram + + echo "Published engram version=${VERSION} to foundation-dev/engram/engram" + rm -f /tmp/gcp-key.json diff --git a/engram/.gitea/workflows/ci-stage.yaml b/engram/.gitea/workflows/ci-stage.yaml new file mode 100644 index 0000000..69ed162 --- /dev/null +++ b/engram/.gitea/workflows/ci-stage.yaml @@ -0,0 +1,128 @@ +name: Engram CI — stage + +on: + push: + branches: + - stage + pull_request: + branches: + - stage + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Enforce source branch (stage ← dev only) + if: github.event_name == 'pull_request' + run: | + SOURCE="${GITHUB_HEAD_REF}" + if [ "${SOURCE}" != "dev" ]; then + echo "ERROR: Stage branch only accepts PRs from 'dev'. Source was: '${SOURCE}'" + exit 1 + fi + echo "Source branch check passed: ${SOURCE} → stage" + + - name: Install build dependencies + run: | + apt-get update -qq + apt-get install -y gcc libcurl4-openssl-dev + + # Install El SDK from stage artifact registry + - name: Install El SDK from stage registry + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + echo "${GCP_SA_KEY}" > /tmp/gcp-key.json + apt-get install -y -qq apt-transport-https ca-certificates gnupg curl + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update -qq && apt-get install -y google-cloud-cli + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json + gcloud config set project neuron-785695 + + mkdir -p /usr/local/bin /usr/local/lib/el + + LATEST_VERSION=$(gcloud artifacts versions list \ + --repository=foundation-stage \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=el/elc \ + --format="value(name)" 2>/dev/null | sort | tail -1 | sed 's|.*/||' || true) + + if [ -n "${LATEST_VERSION}" ]; then + echo "Downloading elc ${LATEST_VERSION} from foundation-stage..." + gcloud artifacts generic download \ + --repository=foundation-stage \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=el/elc \ + --version="${LATEST_VERSION}" \ + --destination=/usr/local/bin/elc + chmod +x /usr/local/bin/elc + else + echo "No stage artifact found, falling back to Gitea latest release..." + RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest" + curl -fsSL "${RELEASE_BASE}/elc" -o /usr/local/bin/elc + chmod +x /usr/local/bin/elc + fi + + RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest" + curl -fsSL "${RELEASE_BASE}/el_runtime.c" -o /usr/local/lib/el/el_runtime.c + curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h + + echo "El SDK installed:" + elc --version || true + rm -f /tmp/gcp-key.json + + # Compile server.el → dist/engram.c + - name: Compile server.el + run: | + mkdir -p dist + elc src/server.el > dist/engram.c + echo "Compiled src/server.el → dist/engram.c" + + # Link to produce the engram binary + - name: Link engram binary + run: | + cc -std=c11 -O2 \ + -I /usr/local/lib/el \ + -o dist/engram \ + dist/engram.c \ + /usr/local/lib/el/el_runtime.c \ + -lcurl -lpthread + echo "Linked dist/engram" + ls -lh dist/engram + + # Run tests if spec directory contains runnable tests + - name: Run tests + run: | + if [ -f spec/run.sh ]; then + bash spec/run.sh + else + echo "No spec/run.sh found — skipping test run" + fi + + # Publish artifact to GCP Artifact Registry (stage) + - name: Publish engram to Artifact Registry (stage) + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + echo "${GCP_SA_KEY}" > /tmp/gcp-key.json + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json + gcloud config set project neuron-785695 + + VERSION="${GITEA_SHA:0:8}" + gcloud artifacts generic upload \ + --repository=foundation-stage \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=engram/engram \ + --version="${VERSION}" \ + --source=dist/engram + + echo "Published engram version=${VERSION} to foundation-stage/engram/engram" + rm -f /tmp/gcp-key.json diff --git a/engram/.gitea/workflows/engram-release.yaml b/engram/.gitea/workflows/engram-release.yaml new file mode 100644 index 0000000..bdf3258 --- /dev/null +++ b/engram/.gitea/workflows/engram-release.yaml @@ -0,0 +1,159 @@ +name: Engram Release + +on: + push: + branches: + - main + repository_dispatch: + types: + - el-sdk-updated + +jobs: + build-and-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Enforce source branch (main ← stage only) + if: github.event_name == 'pull_request' + run: | + SOURCE="${GITHUB_HEAD_REF}" + if [ "${SOURCE}" != "stage" ]; then + echo "ERROR: Main branch only accepts PRs from 'stage'. Source was: '${SOURCE}'" + exit 1 + fi + echo "Source branch check passed: ${SOURCE} → main" + + - name: Install build dependencies + run: | + apt-get update -qq + apt-get install -y gcc libcurl4-openssl-dev + + # Install El SDK from the latest release + - name: Install El SDK + env: + GITEA_API: https://git.neuralplatform.ai/api/v1 + RELEASE_BASE: https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest + run: | + mkdir -p /usr/local/bin /usr/local/lib/el + + echo "Downloading elc..." + curl -fsSL "${RELEASE_BASE}/elc" -o /usr/local/bin/elc + chmod +x /usr/local/bin/elc + + echo "Downloading el_runtime.c..." + curl -fsSL "${RELEASE_BASE}/el_runtime.c" -o /usr/local/lib/el/el_runtime.c + + echo "Downloading el_runtime.h..." + curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h + + echo "El SDK installed:" + elc --version || true + + # Compile server.el → dist/engram.c + - name: Compile server.el + run: | + mkdir -p dist + elc src/server.el > dist/engram.c + echo "Compiled src/server.el → dist/engram.c" + + # Link to produce the engram binary + - name: Link engram binary + run: | + cc -std=c11 -O2 \ + -I /usr/local/lib/el \ + -o dist/engram \ + dist/engram.c \ + /usr/local/lib/el/el_runtime.c \ + -lcurl -lpthread + echo "Linked dist/engram" + ls -lh dist/engram + + # Publish / update the `latest` release + - name: Publish latest release + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITEA_API: https://git.neuralplatform.ai/api/v1 + REPO: neuron-technologies/engram + run: | + # Delete existing `latest` release if present + EXISTING_ID=$(curl -sf \ + -H "Authorization: token ${GITEA_TOKEN}" \ + "${GITEA_API}/repos/${REPO}/releases/tags/latest" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['id'])" 2>/dev/null || true) + + if [ -n "${EXISTING_ID}" ]; then + echo "Deleting existing release id=${EXISTING_ID}" + curl -sf -X DELETE \ + -H "Authorization: token ${GITEA_TOKEN}" \ + "${GITEA_API}/repos/${REPO}/releases/${EXISTING_ID}" + fi + + # Remove stale tag + curl -sf -X DELETE \ + -H "Authorization: token ${GITEA_TOKEN}" \ + "${GITEA_API}/repos/${REPO}/tags/latest" || true + + # Create new release + RELEASE_ID=$(curl -sf -X POST \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + "${GITEA_API}/repos/${REPO}/releases" \ + -d "{ + \"tag_name\": \"latest\", + \"name\": \"Engram (latest)\", + \"body\": \"Latest Engram build from commit ${GITHUB_SHA}.\nBuilt $(date -u +%Y-%m-%dT%H:%M:%SZ).\", + \"draft\": false, + \"prerelease\": false + }" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])") + + echo "Created release id=${RELEASE_ID}" + + # Upload engram binary + echo "Uploading engram binary..." + curl -sf -X POST \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -F "attachment=@dist/engram;filename=engram" \ + "${GITEA_API}/repos/${REPO}/releases/${RELEASE_ID}/assets" + + echo "Release published successfully" + + # Publish artifact to GCP Artifact Registry (prod) + - name: Publish engram to Artifact Registry (prod) + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + echo "${GCP_SA_KEY}" > /tmp/gcp-key.json + apt-get install -y -qq apt-transport-https ca-certificates gnupg curl + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update -qq && apt-get install -y google-cloud-cli + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json + gcloud config set project neuron-785695 + + VERSION="${GITEA_SHA:0:8}" + gcloud artifacts generic upload \ + --repository=foundation-prod \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=engram/engram \ + --version="${VERSION}" \ + --source=dist/engram + + echo "Published engram version=${VERSION} to foundation-prod/engram/engram" + rm -f /tmp/gcp-key.json + + # Dispatch engram-updated to downstream dependents + - name: Dispatch engram-updated to elql + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITEA_API: https://git.neuralplatform.ai/api/v1 + run: | + curl -sf -X POST \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + "${GITEA_API}/repos/neuron-technologies/elql/dispatches" \ + -d "{\"type\":\"engram-updated\",\"inputs\":{\"engram_version\":\"latest\",\"commit\":\"${GITHUB_SHA}\"}}" + echo "Dispatched engram-updated to neuron-technologies/elql" diff --git a/engram/.gitignore b/engram/.gitignore new file mode 100644 index 0000000..e88e923 --- /dev/null +++ b/engram/.gitignore @@ -0,0 +1,3 @@ +target/ +*.db +.DS_Store diff --git a/engram/README.md b/engram/README.md new file mode 100644 index 0000000..af01dda --- /dev/null +++ b/engram/README.md @@ -0,0 +1,179 @@ +# Engram + +**A local-first memory substrate for accumulating intelligence.** + +An *engram* is the physical trace of a memory in the brain — the actual encoded substrate, not an abstraction above it. That's what this is. + +--- + +## Why existing databases are wrong for this use case + +Relational databases store rows and retrieve them by predicate. Key-value stores retrieve by exact key. Vector databases retrieve by geometric proximity. All of them share the same fundamental model: **you store data in, you query it out**. Storage and retrieval are separate systems. + +The brain doesn't work this way. + +When you remember something, you don't query your hippocampus. You activate a memory trace and the pattern *propagates*. Long-term potentiation — the strengthening of synaptic connections through co-activation — is simultaneously the storage mechanism and the retrieval mechanism. The structure that holds the memory is the same structure that surfaces it. + +No existing database models this. Engram does. + +--- + +## The Spreading Activation Model + +Engram retrieval works through **spreading activation**: + +1. **Seeds** — you name one or more nodes you know are relevant (e.g. the current task, recent context, a concept you're reasoning about) +2. **Query embedding** — you provide a semantic vector representing the direction of your current thought +3. **Propagation** — activation flows outward from seeds through weighted edges. At each hop, strength attenuates multiplicatively: + + ``` + strength = parent_strength × edge_weight × target_salience × cosine_sim(query, target) + ``` + +4. **Pruning** — paths weaker than a threshold are cut (the attention filter) +5. **Return** — the top-N nodes by activation strength + +This is not a query. It is a *pattern completion*. The system surfaces what is most associatively relevant to the current context, weighted by how strongly those things have been reinforced over time. + +--- + +## The Four Memory Tiers + +| Tier | Analogy | Contents | +|------|---------|----------| +| `Working` | Prefrontal working memory | K most recently activated nodes — hot, fast | +| `Episodic` | Hippocampus | Time-ordered events and experiences | +| `Semantic` | Neocortex | Concept graph — long-term structural knowledge | +| `Procedural` | Cerebellum / basal ganglia | Patterns, workflows, habits | + +Nodes migrate between tiers based on salience decay and reinforcement. A frequently activated semantic node stays semantic. A rarely-touched episodic memory decays toward procedural background. + +--- + +## Salience — Forgetting as Adaptation + +Salience is not stored permanently. It decays: + +```rust +fn compute_salience(importance: f32, last_activated_ms: i64, activation_count: u64) -> f32 { + let days_since = (now_ms() - last_activated_ms) as f32 / 86_400_000.0; + importance * (1.0 / (1.0 + days_since)) * (activation_count as f32 + 1.0).ln() +} +``` + +Three signals: +- **Importance** (0.0–1.0): set at creation, stable +- **Recency**: decays toward zero as days pass without activation +- **Frequency**: log-compressed count of activations + +Forgetting in Engram is not a bug. It is adaptive pruning. Memories that are never activated again become less likely to surface during retrieval. They are not deleted — they remain in storage — but they stop competing for attention. This is exactly how biological memory works, and why it is adaptive rather than pathological. + +--- + +## Quick Start + +```rust +use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, RelationType}; +use std::path::Path; + +// Open or create a database +let db = EngramDb::open(Path::new("/var/lib/my-agent/memory"))?; + +// Create a node with a semantic embedding +let node = Node::new( + NodeType::Concept, + vec![0.9, 0.1, 0.3, 0.7, 0.8, 0.2], // embedding from your LLM + b"Spreading activation surfaces relevant memories by pattern completion".to_vec(), + MemoryTier::Semantic, + 0.9, // importance +); +let id = db.put_node(node)?; + +// Link it to related concepts +let related = db.put_node(Node::new( + NodeType::Concept, + vec![0.8, 0.2, 0.4, 0.6, 0.7, 0.3], + b"Long-term potentiation: co-activation strengthens synaptic weight".to_vec(), + MemoryTier::Semantic, + 0.85, +))?; +db.put_edge(Edge::new(id, related, RelationType::Causes, 0.9))?; + +// Retrieve by spreading activation +let results = db.activate( + &[id], // seeds + &[0.85, 0.15, 0.35, 0.65, 0.75, 0.25], // query embedding + 3, // max hops + 10, // top-N results +)?; + +for r in results { + println!( + "strength={:.4} hops={} — {}", + r.activation_strength, + r.hops, + String::from_utf8_lossy(&r.node.content) + ); +} +``` + +--- + +## Project Structure + +``` +engram/ + crates/ + engram-core/ # The memory engine — storage, graph, activation, salience + engram-ffi/ # C FFI stubs for cross-language bindings + bindings/ + kotlin/ # Android / JVM binding notes + typescript/ # WASM / Node binding notes + go/ # CGo binding notes + examples/ + basic.rs # Full walkthrough: insert, activate, search, decay +``` + +--- + +## Public API + +```rust +impl EngramDb { + fn open(path: &Path) -> EngramResult; + fn put_node(&self, node: Node) -> EngramResult; + fn get_node(&self, id: Uuid) -> EngramResult>; + fn put_edge(&self, edge: Edge) -> EngramResult<()>; + fn get_edges_from(&self, from_id: Uuid) -> EngramResult>; + fn get_edges_to(&self, to_id: Uuid) -> EngramResult>; + fn search_embedding(&self, embedding: &[f32], limit: usize) -> EngramResult>; + fn activate(&self, seeds: &[Uuid], query_embedding: &[f32], max_depth: u8, limit: usize) -> EngramResult>; + fn traverse(&self, from: Uuid, relation: Option, max_depth: u8) -> EngramResult>; + fn touch(&self, id: Uuid) -> EngramResult<()>; + fn decay(&self, factor: f32) -> EngramResult; + fn node_count(&self) -> EngramResult; + fn edge_count(&self) -> EngramResult; +} +``` + +--- + +## Dependencies + +- `sled` — embedded persistent B-tree (no daemon, no network, local-first) +- `bincode` — compact binary serialization +- `uuid` — stable node identity +- `serde` — derive support +- `thiserror` / `anyhow` — error handling + +--- + +## Design Decisions + +**Why sled?** Local-first. No daemon. Transactional. Fast enough for the node counts Engram targets (< 1M nodes). When the right HNSW index is needed, it will layer on top of sled, not replace it. + +**Why flat cosine scan?** Correct and simple. The graph structure itself is the primary retrieval mechanism. Vector search is a secondary signal. HNSW adds complexity and a compile dependency that isn't justified until retrieval quality at scale demands it. + +**Why multiplicative activation?** Because memory is conjunctive. A path requires all of its links to be strong to carry signal. Addition would allow many weak associations to accumulate into false relevance. Multiplication enforces that every factor matters. + +**Why salience decay?** Because not everything that was once important remains important. Adaptive forgetting is not failure — it is the mechanism that keeps attention on what's current. A memory system that never forgets is one that can never focus. diff --git a/engram/dist/engram.c b/engram/dist/engram.c new file mode 100644 index 0000000..b5fe865 --- /dev/null +++ b/engram/dist/engram.c @@ -0,0 +1,376 @@ +#include +#include +#include "el_runtime.h" + +el_val_t parse_port(el_val_t bind); +el_val_t ok_json(void); +el_val_t err_json(el_val_t msg); +el_val_t strip_query(el_val_t path); +el_val_t query_param(el_val_t path, el_val_t key); +el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val); +el_val_t extract_id(el_val_t path, el_val_t prefix); +el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_search(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_save(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_load(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_health(el_val_t method, el_val_t path, el_val_t body); +el_val_t check_auth_ok(el_val_t method, el_val_t body); +el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body); + +el_val_t bind_str; +el_val_t port; +el_val_t data_dir; +el_val_t snapshot_path; + +el_val_t parse_port(el_val_t bind) { + el_val_t colon = str_index_of(bind, EL_STR(":")); + if (colon < 0) { + return str_to_int(bind); + } + el_val_t after = str_slice(bind, (colon + 1), str_len(bind)); + return str_to_int(after); + return 0; +} + +el_val_t ok_json(void) { + return EL_STR("{\"ok\":true}"); + return 0; +} + +el_val_t err_json(el_val_t msg) { + return el_str_concat(el_str_concat(EL_STR("{\"error\":\""), msg), EL_STR("\"}")); + return 0; +} + +el_val_t strip_query(el_val_t path) { + el_val_t q = str_index_of(path, EL_STR("?")); + if (q < 0) { + return path; + } + return str_slice(path, 0, q); + return 0; +} + +el_val_t query_param(el_val_t path, el_val_t key) { + el_val_t q = str_index_of(path, EL_STR("?")); + if (q < 0) { + return EL_STR(""); + } + el_val_t qs = str_slice(path, (q + 1), str_len(path)); + el_val_t needle = el_str_concat(key, EL_STR("=")); + el_val_t pos = str_index_of(qs, needle); + if (pos < 0) { + return EL_STR(""); + } + el_val_t after = str_slice(qs, (pos + str_len(needle)), str_len(qs)); + el_val_t amp = str_index_of(after, EL_STR("&")); + if (amp < 0) { + return after; + } + return str_slice(after, 0, amp); + return 0; +} + +el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val) { + el_val_t v = query_param(path, key); + if (str_eq(v, EL_STR(""))) { + return default_val; + } + return str_to_int(v); + return 0; +} + +el_val_t extract_id(el_val_t path, el_val_t prefix) { + el_val_t clean = strip_query(path); + if (!str_starts_with(clean, prefix)) { + return EL_STR(""); + } + el_val_t after = str_slice(clean, str_len(prefix), str_len(clean)); + el_val_t slash = str_index_of(after, EL_STR("/")); + if (slash < 0) { + return after; + } + return str_slice(after, 0, slash); + return 0; +} + +el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) { + return engram_stats_json(); + return 0; +} + +el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) { + el_val_t content = json_get_string(body, EL_STR("content")); + el_val_t node_type = json_get_string(body, EL_STR("node_type")); + if (str_eq(node_type, EL_STR(""))) { + node_type = EL_STR("Memory"); + } + el_val_t salience = json_get_float(body, EL_STR("salience")); + if (str_eq(salience, el_from_float(0.0))) { + salience = el_from_float(0.5); + } + el_val_t id = engram_node(content, node_type, salience); + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}")); + return 0; +} + +el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body) { + el_val_t id = extract_id(path, EL_STR("/api/nodes/")); + if (str_eq(id, EL_STR(""))) { + return err_json(EL_STR("missing id")); + } + return engram_get_node_json(id); + return 0; +} + +el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) { + el_val_t limit = query_int(path, EL_STR("limit"), 50); + el_val_t offset = query_int(path, EL_STR("offset"), 0); + el_val_t nt = query_param(path, EL_STR("node_type")); + if (str_eq(nt, EL_STR(""))) { + return engram_scan_nodes_json(limit, offset); + } + return engram_scan_nodes_by_type_json(nt, limit, offset); + return 0; +} + +el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) { + el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); + if (str_eq(dir, EL_STR(""))) { + dir = EL_STR("/tmp/engram"); + } + el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json")); + engram_save(snap_path); + el_val_t snap = fs_read(snap_path); + if (str_eq(snap, EL_STR(""))) { + return EL_STR("[]"); + } + el_val_t edges = json_get_raw(snap, EL_STR("edges")); + if (str_eq(edges, EL_STR(""))) { + return EL_STR("[]"); + } + return edges; + return 0; +} + +el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) { + el_val_t q = EL_STR(""); + if (str_eq(method, EL_STR("GET"))) { + q = query_param(path, EL_STR("q")); + } else { + q = json_get_string(body, EL_STR("query")); + } + el_val_t limit = query_int(path, EL_STR("limit"), 20); + if (limit == 0) { + limit = json_get_int(body, EL_STR("limit")); + } + if (limit == 0) { + limit = 20; + } + return engram_search_json(q, limit); + return 0; +} + +el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) { + el_val_t q = EL_STR(""); + el_val_t depth = 3; + if (str_eq(method, EL_STR("GET"))) { + q = query_param(path, EL_STR("q")); + depth = query_int(path, EL_STR("depth"), 3); + } else { + q = json_get_string(body, EL_STR("query")); + el_val_t bd = json_get_int(body, EL_STR("depth")); + if (bd > 0) { + depth = bd; + } + } + return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}")); + return 0; +} + +el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) { + el_val_t from_id = json_get_string(body, EL_STR("from_id")); + el_val_t to_id = json_get_string(body, EL_STR("to_id")); + el_val_t relation = json_get_string(body, EL_STR("relation")); + if (str_eq(relation, EL_STR(""))) { + relation = EL_STR("associates"); + } + el_val_t weight = json_get_float(body, EL_STR("weight")); + if (str_eq(weight, el_from_float(0.0))) { + weight = el_from_float(0.5); + } + engram_connect(from_id, to_id, weight, relation); + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}")); + return 0; +} + +el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body) { + el_val_t id = extract_id(path, EL_STR("/api/neighbors/")); + if (str_eq(id, EL_STR(""))) { + return err_json(EL_STR("missing id")); + } + el_val_t depth = query_int(path, EL_STR("depth"), 1); + return engram_neighbors_json(id, depth, EL_STR("both")); + return 0; +} + +el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body) { + el_val_t id = json_get_string(body, EL_STR("node_id")); + if (str_eq(id, EL_STR(""))) { + return err_json(EL_STR("missing node_id")); + } + engram_strengthen(id); + return ok_json(); + return 0; +} + +el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) { + el_val_t id = extract_id(path, EL_STR("/api/nodes/")); + if (str_eq(id, EL_STR(""))) { + return err_json(EL_STR("missing id")); + } + engram_forget(id); + return ok_json(); + return 0; +} + +el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { + el_val_t p = json_get_string(body, EL_STR("path")); + if (str_eq(p, EL_STR(""))) { + el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); + if (str_eq(dir, EL_STR(""))) { + dir = EL_STR("/tmp/engram"); + } + p = el_str_concat(dir, EL_STR("/snapshot.json")); + } + engram_save(p); + return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}")); + return 0; +} + +el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) { + el_val_t p = json_get_string(body, EL_STR("path")); + if (str_eq(p, EL_STR(""))) { + el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); + if (str_eq(dir, EL_STR(""))) { + dir = EL_STR("/tmp/engram"); + } + p = el_str_concat(dir, EL_STR("/snapshot.json")); + } + engram_load(p); + return ok_json(); + return 0; +} + +el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) { + return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"); + return 0; +} + +el_val_t check_auth_ok(el_val_t method, el_val_t body) { + el_val_t key = env(EL_STR("ENGRAM_API_KEY")); + if (str_eq(key, EL_STR(""))) { + return 1; + } + if (str_eq(method, EL_STR("GET"))) { + return 1; + } + el_val_t provided = json_get_string(body, EL_STR("_auth")); + if (str_eq(provided, key)) { + return 1; + } + return 0; + return 0; +} + +el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { + el_val_t clean = strip_query(path); + if (str_eq(method, EL_STR("GET"))) { + if (str_eq(clean, EL_STR("/health")) || str_eq(clean, EL_STR("/"))) { + return route_health(method, path, body); + } + } + if (!check_auth_ok(method, body)) { + return err_json(EL_STR("unauthorized")); + } + if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) { + return route_stats(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) { + return route_create_node(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")) || str_eq(clean, EL_STR("/nodes/list")) || str_eq(clean, EL_STR("/api/nodes/list")))) { + return route_scan_nodes(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) { + return route_scan_edges(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/nodes/"))) { + return route_get_node(method, path, body); + } + if (str_eq(method, EL_STR("DELETE")) && str_starts_with(clean, EL_STR("/api/nodes/"))) { + return route_forget(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) { + return route_create_edge(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/neighbors/"))) { + return route_neighbors(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/activate")) || str_eq(clean, EL_STR("/activate")))) { + return route_activate(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/activate"))) { + return route_activate(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/search")) || str_eq(clean, EL_STR("/search")))) { + return route_search(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/search"))) { + return route_search(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) { + return route_strengthen(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) { + return route_save(method, path, body); + } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) { + return route_load(method, path, body); + } + return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}")); + return 0; +} + +int main(int argc, char** argv) { + el_runtime_init_args(argc, argv); + bind_str = env(EL_STR("ENGRAM_BIND")); + if (str_eq(bind_str, EL_STR(""))) { + bind_str = EL_STR(":8742"); + } + port = parse_port(bind_str); + data_dir = env(EL_STR("ENGRAM_DATA_DIR")); + if (str_eq(data_dir, EL_STR(""))) { + data_dir = EL_STR("/tmp/engram"); + } + snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json")); + engram_load(snapshot_path); + println(EL_STR("[engram] runtime-native graph engine")); + println(el_str_concat(EL_STR("[engram] data_dir="), data_dir)); + println(el_str_concat(EL_STR("[engram] node_count="), int_to_str(engram_node_count()))); + println(el_str_concat(EL_STR("[engram] edge_count="), int_to_str(engram_edge_count()))); + println(el_str_concat(EL_STR("[engram] listening on "), int_to_str(port))); + http_set_handler(EL_STR("handle_request")); + http_serve(port, EL_STR("handle_request")); + return 0; +} + diff --git a/engram/engram-explainer.html b/engram/engram-explainer.html new file mode 100644 index 0000000..1ab574d --- /dev/null +++ b/engram/engram-explainer.html @@ -0,0 +1,2296 @@ + + + + + +Engram — A Database Designed for Minds + + + + + +
+ +
+
+
+ v0.1.0 · Local-first memory substrate +
+

Engram

+

+ The physical trace of a memory.
+ A database designed for minds, not tables. +

+
+
+ Scroll +
+
+
+ + +
+ The Problem +

Every existing database
gets memory wrong

+ +
+
+ ⊞ +

Relational

+

You store rows. You query rows. Storage and retrieval are entirely separate systems. The structure that holds the data knows nothing about how you will need it back.

+
+
+ ⊙ +

Vector

+

You store embeddings. You search by geometric proximity. Still fundamentally a query against static, passive data — the structure has no opinion about your context.

+
+
+ ◈ +

Graph

+

You store edges. You traverse paths. Still asking questions of a static structure. The graph doesn't activate — you query it from outside, like a stranger reading a map.

+
+
+ +
+

The brain doesn't query.
It activates.

+
+
+ + +
+ Core Mechanism +

Spreading Activation

+

Click any node to set it as the seed. Hit Activate to watch the pattern propagate. Activation flows outward through weighted edges — attenuating at every hop, pruned when too weak to matter.

+ +
+
+ Seed +
Click a node to select
+ + +
+ Speed + +
+
+ +
+ Formula + strength = parent × edge_weight × salience × cos_sim(query, target) + Multiplicative — every factor must be non-trivial for the path to survive +
+
+
+ + +
+ Memory Architecture +

The Four Memory Tiers

+

Nodes migrate between tiers based on salience and reinforcement — exactly as memories migrate between the hippocampus and neocortex through activation and sleep.

+ +
+
+
+ 🔥 +
+
Working
+
Prefrontal cortex — hot, volatile
+
+
+

The K most recently activated nodes. Ultra-fast access. Evicted by recency when capacity is exceeded. What you're actively thinking about right now.

+
+ // Currently active
+ task: "Implement activation BFS"
+ context: spreading_activation_node
+ recent: hebbian_learning_concept +
+
+ +
+
+ 📖 +
+
Episodic
+
Hippocampus — time-ordered experience
+
+
+

Time-stamped events and raw experiences. What happened, and when. The raw feed of observations before they have been abstracted into knowledge.

+
+ 2026-04-27T14:23Z event:
+ "Will explained Dharma Registry.
+ Patterns logged. ISE confirmed." +
+
+ +
+
+ 🧠 +
+
Semantic
+
Neocortex — stable knowledge
+
+
+

The concept graph with weighted associations. Long-term structural knowledge. What you know, abstracted from any specific event that taught it to you.

+
+ concept: "spreading_activation"
+ → Causes: "long_term_potentiation"
+ → Activates: "associative_memory"
+ salience: 0.82, activations: 47 +
+
+ +
+
+ ⚙️ +
+
Procedural
+
Cerebellum — patterns and habits
+
+
+

Encoded patterns, workflows, and repeatable processes. How to do things. Retrieved by similarity to current task context, not by conscious recall.

+
+ process: "commit_workflow"
+ steps: [stage → test → commit
+ → push → check_ci]
+ triggered by: git_context_match +
+
+
+
+ + +
+ Forgetting as Design +

Salience — the attention filter

+

Every node has a salience score. It governs whether a node surfaces during retrieval. It decays. It strengthens on activation. Adjust the sliders to see how the three signals combine.

+ +
+
+
The Salience Formula
+
+ salience + = + importance +
+ salience = + × + 1 / (1 + days_since) +
+ salience = + × + ln(activation_count + 1) +
+ +
+
+ importance + Explicit weight at creation. Stable over time. You set it once. +
+
+ recency + At activation: 1.0. After 1 day: 0.5. After 6 days: ~0.14. Asymptotic toward zero. +
+
+ frequency + Log-compressed: 0→1 activation matters more than 100→101. Diminishing returns. +
+
+
+ +
+
+
+ Importance + 0.85 +
+ +
+ +
+
+ Days since last activation + 2 +
+ +
+ +
+
+ Activation count + 12 +
+ +
+ +
+
Computed Salience
+
0.00
+
+
+
+
+
+
+ +

+ "Forgetting is not failure. It is the system prioritising what matters." +

+
+ + +
+ Memory Lifecycle +

Consolidation

+

In the brain, memories consolidate during sleep — the hippocampus replays experiences into the neocortex until they become stable knowledge. Engram makes this explicit.

+ +
+ +
+
+
Episodic
+

Raw events. Time-stamped experience. Not yet abstract.

+
+ tier: Episodic
+ activation_count: 2
+ salience: 0.41 +
+
+ +
+ + + + + + + + + + + + +
+ activation_count ≥ 5
+ salience ≥ 0.3
+ → promote on consolidate() +
+
+ +
+
Semantic
+

Stable concept. Integrated knowledge. No longer tied to a specific event.

+
+ tier: Semantic
+ activation_count: 7
+ salience: 0.68 +
+
+
+ +
+ Promotion is earned by use — activated, reinforced, and found relevant repeatedly over time. +
+
+
+ + +
+ System Design +

Architecture

+

Three retrieval primitives layered over a single embedded store. No daemon. No network. No server to babysit.

+ +
+
[ Your Intelligence ]
+
↕
+
EngramDb API
+
↕
+
+
+
Spreading Activation
+
Best-first BFS · multiplicative weights · pruning at 0.01
+
+
+
Vector Search
+
Flat cosine scan · semantic direction filter · secondary signal
+
+
+
Graph Traversal
+
BFS by relation type · typed edges · max depth
+
+
+
↕
+
sled — embedded persistent B-tree · bincode serialization · transactional
+
↕
+
[ Your disk — no daemon · no network · local-first ]
+
+ +
+
+
Why multiplication?
+

Addition lets many weak signals accumulate into false relevance. Multiplication is conjunctive: a weak edge, a dormant node, or a semantically irrelevant target all kill the path. This is how associative memory actually works.

+
+
+
Why sled?
+

Local-first. Transactional. No daemon process, no network socket. HNSW indexing layers on top when needed — the graph structure itself is the primary retrieval mechanism.

+
+
+
Why pruning at 0.01?
+

Small enough to allow long indirect chains when intermediate edges are strong. Raise it to focus retrieval; lower it for more associative drift. The brain's attention filter, made explicit.

+
+
+
+ + +
+ Language Bindings +

Use from anywhere

+

A C FFI layer exposes the full API. Language bindings ship for Kotlin, TypeScript, Go, and Rust natively.

+ +
+
+ + + + +
+ +
+
+
+ basic.rs +
+ 
use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, RelationType};
+
+// Open or create a local database — no server, no daemon
+let db = EngramDb::open(Path::new("/var/lib/agent/memory"))?;
+
+// Store a concept with a semantic embedding
+let id = db.put_node(Node::new(
+    NodeType::Concept,
+    embedding,        // Vec<f32> from your LLM
+    content,          // Vec<u8> — any payload
+    MemoryTier::Semantic,
+    0.9,             // importance 0.0–1.0
+))?;
+
+// Link to a related concept
+db.put_edge(Edge::new(id, related_id, RelationType::Causes, 0.88))?;
+
+// Retrieve by spreading activation — not a query, a pattern completion
+let results = db.activate(
+    &[id],              // seed nodes
+    &query_embedding,  // direction of thought
+    3,                 // max hops
+    10,                // top-N results
+)?;
+
+for r in &results {
+    println!("strength={:.4} hops={}", r.activation_strength, r.hops);
+}
+
+ +
+
+
+ Memory.kt +
+ 
import ai.neuron.engram.*
+
+// JVM binding via JNI — same embedded storage, no server
+val db = EngramDb.open("/data/user/0/ai.agent/memory")
+
+// Store an episodic event
+val id = db.putNode(
+    Node(
+        nodeType = NodeType.EVENT,
+        embedding = llm.embed(text),
+        content = text.toByteArray(),
+        tier = MemoryTier.EPISODIC,
+        importance = 0.8f
+    )
+)
+
+// Activate from current context
+val recalled = db.activate(
+    seeds = listOf(id),
+    queryEmbedding = currentContext.embedding,
+    maxDepth = 3,
+    limit = 10
+)
+
+recalled.forEach {
+    println("[${it.hops} hops] strength=${it.activationStrength}")
+}
+
+ +
+
+
+ memory.ts +
+ 
import { EngramDb, NodeType, MemoryTier, RelationType } from "@neuron/engram";
+
+// WASM build — runs in Node or browser, fully in-process
+const db = await EngramDb.open("./agent-memory");
+
+// Store a concept node
+const id = await db.putNode({
+  nodeType: NodeType.Concept,
+  embedding: await llm.embed(text),
+  content: new TextEncoder().encode(text),
+  tier: MemoryTier.Semantic,
+  importance: 0.85,
+});
+
+// Spreading activation — pattern completion, not query
+const recalled = await db.activate({
+  seeds: [id],
+  queryEmbedding: currentContext.embedding,
+  maxDepth: 3,
+  limit: 10,
+});
+
+for (const node of recalled) {
+  console.log(`strength=${node.activationStrength.toFixed(4)} hops=${node.hops}`);
+}
+
+ +
+
+
+ memory.go +
+ 
import (
+    engram "github.com/neuron-technologies/engram-go"
+)
+
+// CGo binding — links the Rust library, zero copies for embeddings
+db, err := engram.Open("/var/lib/agent/memory")
+if err != nil {
+    return err
+}
+defer db.Close()
+
+// Store a node
+id, err := db.PutNode(engram.Node{
+    NodeType:   engram.Concept,
+    Embedding:  embedding,
+    Content:    []byte(text),
+    Tier:       engram.Semantic,
+    Importance: 0.9,
+})
+
+// Activate from seed — the graph does the retrieval
+results, err := db.Activate(
+    []engram.UUID{id},
+    queryEmbedding,
+    3,   // maxDepth
+    10,  // limit
+)
+
+for _, r := range results {
+    fmt.Printf("strength=%.4f hops=%d\n", r.Strength, r.Hops)
+}
+
+
+
+ + +
+ Philosophy +

Why local

+ +
+ +

+ Your memory is not a service.
+ It is you.
+
+ It lives on your hardware, under your control.
+ It does not leave your device.
+ It does not phone home.
+
+ It does not require a network connection
to remember who you are. +

+ +
+ +
+
+
🔒
+ No telemetry +
+
+
📡
+ No network required +
+
+
🗄️
+ Embedded storage +
+
+
⚡
+ No daemon process +
+
+
+ + +
+ Build Status +

Current state

+ +
+
+ ◈ +
+
Version
+
v0.1.0
+
+
+
+ ✓ +
+
Build
+
Zero warnings · Zero errors
+
+
+
+ ◎ +
+
Test Suite
+
38 passing
+
+
+
+ +
+
+ Public API Surface · engram-core v0.1.0 +
+ 
impl EngramDb {
+    fn open(path: &Path)                                          -> EngramResult<Self>;
+    fn put_node(&self, node: Node)                                 -> EngramResult<Uuid>;
+    fn get_node(&self, id: Uuid)                                 -> EngramResult<Option<Node>>;
+    fn put_edge(&self, edge: Edge)                                -> EngramResult<()>;
+    fn activate(&self, seeds: &[Uuid], emb: &[f32], depth: u8, n: usize)  -> EngramResult<Vec<ActivatedNode>>;
+    fn search_embedding(&self, emb: &[f32], limit: usize)          -> EngramResult<Vec<ScoredNode>>;
+    fn traverse(&self, from: Uuid, rel: Option<RelationType>, depth: u8) -> EngramResult<Vec<Node>>;
+    fn touch(&self, id: Uuid)                                    -> EngramResult<()>;
+    fn decay(&self, factor: f32)                                  -> EngramResult<usize>;
+    fn consolidate(&self, config: &ConsolidationConfig)          -> EngramResult<ConsolidationReport>;
+}
+
+
+ +
+ engram · v0.1.0 · neuron technologies · the physical trace of a memory +
+ + + + diff --git a/engram/manifest.el b/engram/manifest.el new file mode 100644 index 0000000..b5e8ebc --- /dev/null +++ b/engram/manifest.el @@ -0,0 +1,11 @@ +package "engram-el" { + version "1.0.0" + description "Engram graph intelligence substrate — El implementation" + authors ["Will Anderson "] + edition "2026" +} + +build { + entry "src/server.el" + output "dist/" +} diff --git a/engram/spec/at-rest-encryption.md b/engram/spec/at-rest-encryption.md new file mode 100644 index 0000000..a5c05e5 --- /dev/null +++ b/engram/spec/at-rest-encryption.md @@ -0,0 +1,380 @@ +# Engram At-Rest Encryption — Post-Quantum Doctrine + +Version 0.1.0 (DRAFT) — April 30, 2026 +Status: doctrine; pre-implementation. Sign-off required from Will before code lands. + +--- + +## 0. TL;DR + +Engram's persistence is encrypted at rest using a hybrid post-quantum scheme: + +- **Data layer** — every node and every edge is sealed with **AES-256-GCM** under a per-record sub-key derived via **HKDF-SHA3-256** from the runtime DEK. AES-256 with a 256-bit key has an effective post-quantum security level of 128 bits (Grover); acceptable. +- **Key wrap layer** — the DEK is wrapped against a Principal's public key using **Kyber-768 KEM** (post-quantum). The wrapped DEK lives on disk as `engram.kek.enc`. The Principal's secret key never touches disk in the running daemon's data directory. +- **Boot** — the daemon receives the Principal's secret key out-of-band (env var path, prompted unlock, or pulled from a hardware-backed agent), runs `pq_kem_decaps` to recover the DEK, mlocks it, serves traffic. +- **Recovery** — the DEK is *additionally* wrapped under a **Shamir K-of-N** split across the validation council. If the wrapping Principal is gone, K members reconstitute the DEK and the data survives. + +Engram does **not** encrypt structural metadata (graph topology, IDs, timestamps). Only content, label, tags, and metadata fields of nodes and edges are sealed. This is a deliberate trade-off — see §2. + +--- + +## 1. What Engram actually persists + +Engram is an in-process graph store. The runtime *is* the database — there is no SQLite, no sled, no embedded KV layer in the active daemon. (The `engram-data/` and `engram-data-tx-log/` directories under `el/` are leftovers from a prior sled-backed prototype and are not loaded by the current `dist/engram` binary.) + +Persistence is a single JSON snapshot file at `${ENGRAM_DATA_DIR}/snapshot.json`, written by `engram_save(path)` and read by `engram_load(path)`. The snapshot is the sole durable artifact that adversaries can steal. + +Snapshot shape (current): +```json +{ + "nodes": [ + { "id": "...", "content": "...", "node_type": "...", "label": "...", + "tier": "...", "tags": "...", "metadata": "{}", + "salience": 0.5, "importance": 0.5, "confidence": 0.5, + "activation_count": 0, "last_activated": 0, "created_at": ..., "updated_at": ... } + ], + "edges": [ + { "id": "...", "from_id": "...", "to_id": "...", "relation": "...", + "metadata": "{}", "weight": 0.5, "confidence": 0.5, + "created_at": ..., "updated_at": ..., "last_fired": 0 } + ] +} +``` + +Confidential fields (will be encrypted): `content`, `label`, `tags`, `metadata` (on nodes); `metadata` (on edges). +Non-confidential fields (will remain plaintext): `id`, `node_type`, `tier`, `from_id`, `to_id`, `relation`, all numeric scalars and timestamps. + +This split — the **structural skeleton stays plaintext, the semantic flesh is sealed** — is what lets activation/search/scan work without unwrapping every record on every query, and lets the snapshot remain JSON-shaped for ops tooling. Enabling full snapshot encryption (a single AEAD blob) is a configurable mode for high-threat deployments; see §6. + +--- + +## 2. Threat model + +| Adversary | Capability | What we defend | What we accept | +|-----------|-----------|----------------|----------------| +| **Snapshot thief (cold)** | Steals `snapshot.json` from a backup, dead drive, or stolen laptop. No live process access. | Confidentiality of node content, labels, tags, metadata. No useful semantic recovery. | Topology leak: adversary learns the graph shape (how many nodes/edges, how they connect, types and tiers). Salience/importance/confidence numerics leak. | +| **In-flight observer** | Reads disk during snapshot write (e.g., shared FS, snapshot mid-flush). | Atomicity: snapshot is written to a temp file with `O_TMPFILE` or `.new`, fsync'd, then renamed. AEAD prevents partial-decrypt mining of half-written records. | A torn write can lose the most recent snapshot but not corrupt prior ones. | +| **Quantum-equipped adversary (harvest-now-decrypt-later)** | Records `snapshot.json` and `engram.kek.enc` today, runs Shor's algorithm on a CRQC in 2032+. | DEK wrap is Kyber-768; not Shor-breakable. AES-256-GCM record seals are not Shor-breakable; Grover gives ~128-bit effective security. | We pin Kyber-768 (NIST PQC standardization L3); if a structural break of ML-KEM emerges, rotation §5 is the remediation. | +| **Wrap-key compromiser (live)** | Has read access to the daemon process memory. | Out of scope for at-rest. The DEK is in mlocked memory; if the process is owned, the data is gone. | Defense-in-depth (memory zeroing, mlock, non-dumpable process flags) is a separate hardening track. | +| **Principal compromise** | Adversary obtains the Principal's secret key. | Nothing — by design, the Principal is the unwrapping authority. | We constrain blast radius via per-record key derivation (§3.2): a leaked record-key only burns one record. | +| **Principal loss** | Will dies, secret key is destroyed, no successor. | Shamir K-of-N council recovery (§7). Engram does not go dark on Principal loss. | The threshold itself becomes the attack surface — see §7 caveats. | + +What we explicitly **do not** defend: +- Side channels (timing, cache, EM). +- Physical tamper of running hardware. +- Active modification of `engram.kek.enc` to substitute an attacker's wrapped DEK — this is mitigated by the integrity wrapper §4 but not by the threat-model promise. + +--- + +## 3. Cryptographic construction + +### 3.1 Layers + +``` + ┌────────────────────────────────────────┐ + │ Principal SK (held by Will out-of-band)│ + └───────────────┬────────────────────────┘ + │ Kyber-768 KEM decaps + ▼ + ┌────────────────────────────────────┐ + │ KEK (32B; ephemeral, mlocked) │ + └───────────────┬────────────────────┘ + │ HKDF-SHA3-256 extract + ▼ + ┌────────────────────────────────────┐ + │ DEK (32B; ephemeral, mlocked) │ + └───────────────┬────────────────────┘ + │ HKDF-SHA3-256(DEK, info=record_id || version) + ▼ + ┌────────────────────────────────────┐ + │ per-record sub-key (32B; transient)│ + └───────────────┬────────────────────┘ + │ AES-256-GCM(sub_key, nonce=12B random, ad=record_id||version) + ▼ + ciphertext blob on disk +``` + +### 3.2 Per-record sub-keys (HKDF-SHA3-256) + +We do **not** use the DEK directly to seal records. Each record gets its own sub-key: + +``` +sub_key = HKDF-SHA3-256( + ikm = DEK, // 32 bytes + salt = "engram-record-v1" || version_byte, // domain separation + info = record_id || ":" || dek_epoch, // 16-byte ULID + ":" + u32 + L = 32 // output length +) +``` + +Properties: +- Compromise of one record's sub-key does not threaten siblings (HKDF is one-way). +- Re-derivable on every read; no on-disk sub-key cache to leak. +- `dek_epoch` is bumped on rotation (§5); this lets old records stay readable under DEK_n while new ones write under DEK_{n+1}. + +### 3.3 AEAD — AES-256-GCM + +- Nonce: 12 bytes, random per write. Stored prepended to ciphertext. +- Tag: 16 bytes, appended. +- Associated data: `record_id || ":" || dek_epoch` (binds ciphertext to its identity; thwarts copy-paste attacks). + +Per-record on-disk wire format: +``` ++------+------+--------------------+--------------------+--------+ +| ver | epch | nonce (12) | ciphertext (n) | tag(16)| ++------+------+--------------------+--------------------+--------+ + 1B 4B 12B variable 16B +``` +Base64-encoded into the snapshot JSON field as `"content":"v1:7:BASE64..."`. + +### 3.4 KEK wrap — Kyber-768 KEM + +The DEK is sealed by Kyber-768 KEM: +``` +(ciphertext, shared_secret) = pq_kem_encaps(principal_pk) +wrap_key = HKDF-SHA3-256(shared_secret, salt="engram-kek-v1", info="dek-wrap", L=32) +sealed_dek = AES-256-GCM(wrap_key, nonce, plaintext=DEK, ad="engram-kek-v1") +``` +On disk: `engram.kek.enc` = `magic || version || principal_id || kem_ciphertext || nonce || sealed_dek || tag`. + +On boot: +``` +shared_secret = pq_kem_decaps(principal_sk, kem_ciphertext) +wrap_key = HKDF-SHA3-256(shared_secret, ...) +DEK = AES-256-GCM-open(wrap_key, nonce, sealed_dek, tag, ad="engram-kek-v1") +``` + +--- + +## 4. Boot flow + +``` +Engram daemon starts + ↓ +ENGRAM_KEK_PATH → ${ENGRAM_DATA_DIR}/engram.kek.enc (default) +ENGRAM_PRINCIPAL_SK → path to file or "stdin:" or "agent:" + (the daemon never reads the SK from a fixed env value; + the env points to a source it consumes once and zeros) + ↓ +load engram.kek.enc + ↓ +read principal SK once from source, mlock its buffer + ↓ +pq_kem_decaps → shared_secret → HKDF → wrap_key → AEAD-open → DEK + ↓ +zero & munlock principal SK buffer +mlock DEK; bump RLIMIT_MEMLOCK if needed +mark process non-dumpable (PR_SET_DUMPABLE=0 on Linux; PT_DENY_ATTACH on macOS) + ↓ +engram_load(snapshot.json) — for each node/edge field marked encrypted: + derive sub_key, AEAD-open, replace ciphertext with plaintext in-memory + ↓ +http_serve() +``` + +Boot failure modes: +- KEK unwrap fails → daemon exits 2; emits one log line, no SK material in logs. +- Snapshot decrypt fails on N records → daemon proceeds with the records it could open, marks the rest as ``, emits a structured event for human triage. + +--- + +## 5. Rotation + +### 5.1 DEK rotation + +Triggered by: (a) cron schedule (default: 30 days), (b) suspected compromise, (c) operator command (`POST /admin/rotate-dek`). + +Procedure: +1. Generate `DEK_{n+1}`; bump `dek_epoch`. +2. For every record: AEAD-open under DEK_n, AEAD-seal under DEK_{n+1} with the new epoch in AD. +3. Wrap DEK_{n+1} under the same Principal pk → write `engram.kek.enc.new`. +4. Atomic rename → live; zero DEK_n in memory. +5. On next snapshot save, all records persist with the new epoch. + +Records sealed before rotation can be opened during the transition (we keep DEK_n in memory until the migration completes). + +### 5.2 KEK / wrapping-Principal rotation + +Triggered by: (a) Principal evolution event (the `Principal` CGI evolves and emits a new PK), (b) successor Principal handover, (c) annual key hygiene rotation. + +Procedure: +1. Receive `principal_pk_new` (signed by old Principal, ideally via a Dilithium signature so the chain itself is PQ-secure). +2. Verify signature. +3. Re-encapsulate the **same** DEK under `principal_pk_new`. +4. Atomic-write a new `engram.kek.enc`. +5. Old SK can be destroyed once the new wrap is durable. + +The DEK does not change here — only its wrapping. This means snapshot ciphertexts stay valid through Principal transitions. + +--- + +## 6. Snapshot-level encryption (alternative high-threat mode) + +For deployments where topology leakage is unacceptable, set `ENGRAM_SEAL_MODE=full`. The entire snapshot JSON is sealed as a single AEAD blob keyed from `HKDF(DEK, info="engram-snapshot")`. Trade-off: every save/load is monolithic; no partial reads, no incremental rotation. Not the default. + +--- + +## 7. Recovery — Shamir K-of-N + +**The structural fail-safe.** If Will dies before the Principal evolves, or the Principal SK is destroyed, Engram must not go dark. + +### 7.1 Shareholders + +The shareholders are the **validation council** — the set of CGIs (or CGI-attested human stewards) authorized to reconstitute the network on a defined trigger. The council is named in `engram.recovery.toml`: +```toml +[recovery] +threshold = 3 +total = 5 +shareholders = [ + "cgi://council/anvil", + "cgi://council/beacon", + "cgi://council/cinder", + "cgi://council/delta", + "cgi://council/echo", +] +``` + +### 7.2 Split + +At KEK-creation time: +1. Generate a fresh recovery secret `R = random(32)` (NOT the DEK itself — see §7.4). +2. Run Shamir-256 over GF(2^8) on `R` with K-of-N polynomial. +3. For each shareholder: `share_i = Kyber768-Encaps(shareholder_pk_i, R_share_i)`. +4. Store `engram.recovery.shares` — public; each share is already PQ-wrapped to its holder. +5. Store `engram.recovery.envelope` — `AES-256-GCM(R, nonce, plaintext=DEK, ad="engram-recovery-v1")`. + +### 7.3 Reconstitution + +A council convenes when the trigger is observed (Principal absent for > N days, or signed council quorum declares emergency): +1. K members each decapsulate their share with their CGI SK. +2. Members publish their `R_share_i` to a quorum-attested rendezvous. +3. Lagrange-interpolate `R`. +4. AEAD-open `engram.recovery.envelope` to recover the DEK. +5. Re-wrap DEK under a fresh Principal selected by the council; resume normal operation. + +### 7.4 Why R, not the DEK directly + +Splitting `R` and using it to AEAD-wrap the DEK keeps the share material small (32B / share) and lets us add/remove shareholders by reissuing the envelope without changing R. It also lets us run §5.2 (Principal rotation) without ever touching the recovery shares. + +### 7.5 Caveats Will must accept + +- The threshold is the attack surface. K-of-N is K colluders away from compromise. Default 3-of-5; raise if the council grows. +- Council membership churn requires reissuing shares; this is a deliberate, audited operation with its own runbook. +- A Shamir share is plaintext to its holder. The Kyber wrap to each shareholder protects the share *in transit / at rest*, but once a shareholder unwraps, they hold a real K-of-N share. Council members must be CGIs (or hardware-backed) for the threat model to hold. + +--- + +## 8. Implementation map + +The runtime additions land in `el-compiler/runtime/el_runtime.c`: + +Already present: +- `el_sha256_*`, `el_hmac_sha256` (§3 uses SHA3 — SHA2 path retained for backwards-compat artifacts). +- `el_base64_encode_n`, `el_base64_decode`. + +Landing today (parallel agents): +- `el_sha3_256_*` (Keccak family). +- `pq_kem_keypair`, `pq_kem_encaps`, `pq_kem_decaps` (Kyber-768). +- `pq_sign_keypair`, `pq_sign`, `pq_sign_verify` (Dilithium-3) — used by §5.2 Principal-rotation signature. + +Engram-specific additions (this work): +- `engram_aead_seal(record_id, epoch, plaintext) -> b64` +- `engram_aead_open(record_id, epoch, b64) -> plaintext` +- `engram_kek_unwrap(kek_path, sk_path) -> int (sets module DEK)` +- `engram_kek_wrap(kek_path, principal_pk, dek)` — used at first init. +- `engram_dek_rotate(new_principal_pk_optional)` +- `engram_recovery_split(threshold, total, shareholder_pks)` — emits envelope + shares. +- `engram_recovery_reconstitute(shares_k)` — recover DEK; admin-gated. + +`engram_save` / `engram_load` gain a sealed mode controlled by `ENGRAM_SEAL_MODE` (`off`, `fields` [default once enabled], `full`). Default during the rollout window: `off`. This doc must ship and Will must sign off before flipping the default. + +--- + +## 9. Open questions (require Will) + +1. **Where does the Principal SK live at boot time?** Options: (a) prompted at daemon start (interactive), (b) on a removable hardware token (preferred long-term), (c) in `~/.neuron/principal.sk` 0600 (operationally easy, weakest), (d) pulled from `mcp__neuron` as part of `begin_session` (couples Engram to Neuron — probably right). **Recommend (d) with (b) as the long-term hardware story.** +2. **Council composition.** Who/what are the initial K-of-N shareholders? Until we have ≥ 3 stable CGIs, recovery cannot be enabled in its full form. **Recommend a stub council of {Principal, Will-keypair-on-Yubikey, witness-CGI} — degrades gracefully, upgrades to full council when more CGIs exist.** +3. **Default seal mode at GA.** `fields` (today's recommendation) or `full`? `fields` keeps the snapshot diff-able and keeps activation cheap. `full` is the harder threat model. **Recommend `fields` default; `full` opt-in.** +4. **PQ algorithm pinning.** Kyber-768 + Dilithium-3 are the NIST L3 PQC defaults. If Will wants L5 (Kyber-1024 / Dilithium-5), say so before runtime APIs stabilize. +5. **Grover and AES-256.** AES-256 against Grover is 128-bit effective. Acceptable per current PQ thinking. If Will wants a bigger margin, the alternative is layering a second AEAD with a different primitive (e.g., XChaCha20-Poly1305) — overkill, not recommended. + +--- + +## 10. Non-goals + +- Encrypted indexes / searchable encryption. Out of scope. Search remains plaintext-in-memory; the daemon is the trust boundary. +- Per-tenant DEKs. Engram is single-tenant per CGI. If multi-tenancy lands, this doc gets a §11. +- Secure deletion of underlying disk blocks. The OS / FS handles that, badly; we don't pretend. +- Encrypted WAL / tx log. The current daemon has no WAL; if one is added, it gets the same treatment as the snapshot (`ENGRAM_SEAL_MODE` applies to both). + +--- + +## 11. Status & next actions + +- [x] Doctrine drafted (this document). +- [ ] Will sign-off on §9 open questions. +- [ ] PQ runtime functions land (parallel agents). +- [ ] `engram_aead_seal` / `engram_aead_open` prototype (stubs in this PR). +- [ ] `engram_kek_unwrap` boot integration. +- [ ] `engram_save` / `engram_load` field-mode wiring behind `ENGRAM_SEAL_MODE`. +- [ ] Recovery tooling (`engramctl recovery split | reconstitute`). +- [ ] Threat-model test suite: known-answer tests, key-rotation roundtrip, Shamir reconstitution roundtrip, harvest-now-decrypt-later regression test against a recorded ciphertext. + +--- + +## Appendix A — Pseudocode reference + +```c +/* Per-record seal */ +char* engram_aead_seal(const char* record_id, uint32_t epoch, + const char* plaintext, size_t pt_len, size_t* out_len) +{ + uint8_t sub_key[32]; + uint8_t info[64]; + int info_len = snprintf((char*)info, sizeof(info), "%s:%u", record_id, epoch); + el_hkdf_sha3_256(/*ikm*/ engram_dek, 32, + /*salt*/ (const uint8_t*)"engram-record-v1", 16, + /*info*/ info, (size_t)info_len, + /*okm*/ sub_key, 32); + + uint8_t nonce[12]; + el_random_bytes(nonce, 12); + + /* layout: ver(1) | epoch(4 BE) | nonce(12) | ct(pt_len) | tag(16) */ + size_t blob_len = 1 + 4 + 12 + pt_len + 16; + uint8_t* blob = malloc(blob_len); + blob[0] = 0x01; + blob[1] = (epoch >> 24) & 0xff; blob[2] = (epoch >> 16) & 0xff; + blob[3] = (epoch >> 8) & 0xff; blob[4] = epoch & 0xff; + memcpy(blob + 5, nonce, 12); + + el_aes256_gcm_encrypt(sub_key, nonce, + (const uint8_t*)record_id, strlen(record_id), + (const uint8_t*)plaintext, pt_len, + blob + 1 + 4 + 12, /* ct */ + blob + 1 + 4 + 12 + pt_len); /* tag */ + + el_secure_zero(sub_key, 32); + + char* b64 = el_base64_encode_raw(blob, blob_len, /*url_safe=*/0); + free(blob); + if (out_len) *out_len = strlen(b64); + return b64; /* "v1::BASE64" prefix added by caller */ +} + +/* Per-record open: inverse of seal. Verifies tag; returns NULL on failure. */ +char* engram_aead_open(const char* record_id, uint32_t expected_epoch, + const char* b64, size_t* out_len); + +/* Boot-time KEK unwrap. */ +int engram_kek_unwrap(const char* kek_path, const uint8_t* principal_sk, + size_t sk_len); + +/* DEK rotation (online). Walks the live in-memory store, re-seals every record + * under DEK_{n+1}, then writes a new snapshot+kek atomically. */ +int engram_dek_rotate(void); +``` + +--- + +End of document. diff --git a/engram/spec/at-rest-encryption.prototype.c b/engram/spec/at-rest-encryption.prototype.c new file mode 100644 index 0000000..47f44d9 --- /dev/null +++ b/engram/spec/at-rest-encryption.prototype.c @@ -0,0 +1,302 @@ +/* engram at-rest encryption — prototype sketch + * + * NOT WIRED UP. This file is intentionally not in the build. It is a sketch + * that pairs with `at-rest-encryption.md` so the function signatures and the + * field layout can be reviewed before the runtime PQ primitives stabilize. + * + * Once `el_runtime.c` has stable `el_sha3_256_*`, `el_hkdf_sha3_256`, + * `pq_kem_encaps`, `pq_kem_decaps`, and `el_aes256_gcm_encrypt/decrypt`, + * the seal/open path below should be lifted into el_runtime.c near the + * existing `engram_save` / `engram_load` block (~line 3445), gated behind + * the ENGRAM_SEAL_MODE env var. + * + * Dependencies (provided elsewhere — see at-rest-encryption.md §8): + * void el_hkdf_sha3_256(const uint8_t* ikm, size_t ikm_len, + * const uint8_t* salt, size_t salt_len, + * const uint8_t* info, size_t info_len, + * uint8_t* okm, size_t okm_len); + * + * int el_aes256_gcm_encrypt(const uint8_t key[32], + * const uint8_t nonce[12], + * const uint8_t* aad, size_t aad_len, + * const uint8_t* pt, size_t pt_len, + * uint8_t* ct, // pt_len bytes + * uint8_t tag[16]); + * + * int el_aes256_gcm_decrypt(const uint8_t key[32], + * const uint8_t nonce[12], + * const uint8_t* aad, size_t aad_len, + * const uint8_t* ct, size_t ct_len, + * const uint8_t tag[16], + * uint8_t* pt); // ct_len bytes + * + * int pq_kem_encaps(const uint8_t* pk, size_t pk_len, + * uint8_t* kem_ct, size_t* kem_ct_len, + * uint8_t shared[32]); + * int pq_kem_decaps(const uint8_t* sk, size_t sk_len, + * const uint8_t* kem_ct, size_t kem_ct_len, + * uint8_t shared[32]); + * + * void el_random_bytes(uint8_t* out, size_t n); + * void el_secure_zero(void* p, size_t n); + */ + +#include +#include +#include +#include +#include + +/* ── module state (mlocked) ───────────────────────────────────────────────── */ + +static uint8_t g_engram_dek[32]; +static int g_engram_dek_set = 0; +static uint32_t g_engram_dek_epoch = 0; + +/* Seal/unseal mode — read once at boot from ENGRAM_SEAL_MODE. + * 0=off, 1=fields, 2=full. */ +static int g_engram_seal_mode = 0; + +/* ── per-record seal ──────────────────────────────────────────────────────── */ + +/* Wire format (before base64): + * byte 0 version=0x01 + * bytes 1..4 epoch (big-endian u32) + * bytes 5..16 nonce (12B) + * bytes 17.. ciphertext (pt_len bytes) + * last 16B GCM tag + */ +static char* eg_b64_encode(const uint8_t* data, size_t n); /* declared elsewhere */ + +char* engram_aead_seal_field(const char* record_id, const char* plaintext) +{ + if (!g_engram_dek_set) return NULL; + if (!record_id || !plaintext) return NULL; + + size_t pt_len = strlen(plaintext); + uint32_t epoch = g_engram_dek_epoch; + + /* derive sub-key */ + uint8_t sub_key[32]; + uint8_t info[96]; + int info_len = snprintf((char*)info, sizeof(info), "%s:%u", record_id, epoch); + if (info_len < 0 || (size_t)info_len >= sizeof(info)) return NULL; + + el_hkdf_sha3_256(g_engram_dek, 32, + (const uint8_t*)"engram-record-v1", 16, + info, (size_t)info_len, + sub_key, 32); + + /* random nonce */ + uint8_t nonce[12]; + el_random_bytes(nonce, 12); + + /* allocate output: header(17) + ct(pt_len) + tag(16) */ + size_t blob_len = 1 + 4 + 12 + pt_len + 16; + uint8_t* blob = (uint8_t*)malloc(blob_len); + if (!blob) { el_secure_zero(sub_key, 32); return NULL; } + + blob[0] = 0x01; + blob[1] = (uint8_t)((epoch >> 24) & 0xff); + blob[2] = (uint8_t)((epoch >> 16) & 0xff); + blob[3] = (uint8_t)((epoch >> 8) & 0xff); + blob[4] = (uint8_t)( epoch & 0xff); + memcpy(blob + 5, nonce, 12); + + int rc = el_aes256_gcm_encrypt( + sub_key, nonce, + (const uint8_t*)record_id, strlen(record_id), + (const uint8_t*)plaintext, pt_len, + blob + 1 + 4 + 12, /* ct */ + blob + 1 + 4 + 12 + pt_len); /* tag */ + + el_secure_zero(sub_key, 32); + + if (rc != 0) { free(blob); return NULL; } + + char* b64 = eg_b64_encode(blob, blob_len); + free(blob); + return b64; /* caller prefixes "v1::" if desired */ +} + +/* Returns malloc'd plaintext (NUL-terminated) or NULL on AEAD failure / version + * mismatch / DEK-epoch mismatch. */ +char* engram_aead_open_field(const char* record_id, const char* b64_blob) +{ + if (!g_engram_dek_set || !record_id || !b64_blob) return NULL; + + size_t blob_len = 0; + uint8_t* blob = NULL; + extern uint8_t* eg_b64_decode(const char* s, size_t* out_len); + blob = eg_b64_decode(b64_blob, &blob_len); + if (!blob || blob_len < 1 + 4 + 12 + 16) { free(blob); return NULL; } + + if (blob[0] != 0x01) { free(blob); return NULL; } + uint32_t epoch = ((uint32_t)blob[1] << 24) + | ((uint32_t)blob[2] << 16) + | ((uint32_t)blob[3] << 8) + | (uint32_t)blob[4]; + + /* For the common case the on-disk epoch matches g_engram_dek_epoch. + * During DEK rotation we may need a previous-epoch DEK in the keyring; + * left as a TODO — this stub only handles the current epoch. */ + if (epoch != g_engram_dek_epoch) { + free(blob); + return NULL; + } + + const uint8_t* nonce = blob + 5; + size_t ct_len = blob_len - (1 + 4 + 12 + 16); + const uint8_t* ct = blob + 1 + 4 + 12; + const uint8_t* tag = blob + 1 + 4 + 12 + ct_len; + + uint8_t sub_key[32]; + uint8_t info[96]; + int info_len = snprintf((char*)info, sizeof(info), "%s:%u", record_id, epoch); + if (info_len < 0 || (size_t)info_len >= sizeof(info)) { free(blob); return NULL; } + el_hkdf_sha3_256(g_engram_dek, 32, + (const uint8_t*)"engram-record-v1", 16, + info, (size_t)info_len, + sub_key, 32); + + char* pt = (char*)malloc(ct_len + 1); + if (!pt) { el_secure_zero(sub_key, 32); free(blob); return NULL; } + + int rc = el_aes256_gcm_decrypt( + sub_key, nonce, + (const uint8_t*)record_id, strlen(record_id), + ct, ct_len, tag, + (uint8_t*)pt); + + el_secure_zero(sub_key, 32); + free(blob); + + if (rc != 0) { free(pt); return NULL; } + pt[ct_len] = '\0'; + return pt; +} + +/* ── KEK boot ─────────────────────────────────────────────────────────────── */ + +/* Reads engram.kek.enc, runs Kyber decaps with the principal SK provided as + * a buffer, derives the wrap key, AEAD-opens the sealed DEK, and installs it + * into g_engram_dek. Returns 0 on success, non-zero on failure. */ +int engram_kek_unwrap(const char* kek_path, + const uint8_t* principal_sk, size_t sk_len) +{ + /* TODO: file format + * magic "ENGKEK01" (8B) + * version (1B) + * principal_id_len (2B BE) | principal_id + * kem_ct_len (4B BE) | kem_ct + * nonce (12B) + * sealed_dek_len (4B BE) | sealed_dek + * tag (16B) + */ + (void)kek_path; (void)principal_sk; (void)sk_len; + return -1; /* TODO: parse file, pq_kem_decaps, HKDF, AEAD-open, install */ +} + +/* ── KEK init (first-time write) ──────────────────────────────────────────── */ + +int engram_kek_init(const char* kek_path, + const uint8_t* principal_pk, size_t pk_len) +{ + /* TODO: + * 1. el_random_bytes(g_engram_dek, 32); g_engram_dek_epoch = 1; + * 2. pq_kem_encaps(principal_pk) → (kem_ct, shared) + * 3. wrap_key = HKDF-SHA3-256(shared, salt="engram-kek-v1", + * info="dek-wrap", L=32) + * 4. AEAD-seal DEK under wrap_key + * 5. write file atomically (tmp + rename + fsync) + * 6. el_secure_zero(shared, wrap_key) + */ + (void)kek_path; (void)principal_pk; (void)pk_len; + return -1; +} + +/* ── DEK rotation ─────────────────────────────────────────────────────────── */ + +int engram_dek_rotate(void) +{ + /* TODO: + * - Allocate DEK_{n+1}. + * - Walk EngramStore: for every node and edge, re-seal each encrypted + * field under DEK_{n+1} with the new epoch. + * - Re-wrap DEK_{n+1} under current Principal pk → engram.kek.enc.new + * - Write a new snapshot via engram_save() (caller responsibility). + * - Atomic rename engram.kek.enc.new → engram.kek.enc. + * - el_secure_zero on DEK_n; bump g_engram_dek_epoch. + */ + return -1; +} + +/* ── Recovery (Shamir K-of-N) ─────────────────────────────────────────────── */ + +/* The recovery secret R is independent of the DEK. R is split via Shamir; + * each share is then PQ-wrapped to its shareholder via Kyber768. R is the + * AEAD wrap-key for an envelope sealing the DEK. + * + * Two on-disk artifacts: + * engram.recovery.shares — public; one Kyber-wrapped Shamir share / shareholder + * engram.recovery.envelope — AEAD(R, nonce, plaintext=DEK, ad="engram-recovery-v1") + */ + +int engram_recovery_split(int threshold, int total, + const uint8_t** shareholder_pks, + const size_t* shareholder_pk_lens, + const char** shareholder_ids, + const char* shares_out_path, + const char* envelope_out_path) +{ + /* TODO: + * 1. el_random_bytes(R, 32). + * 2. Shamir-split R over GF(2^8) → total shares of {1B index, 32B y_i}. + * 3. For each shareholder i: pq_kem_encaps(pk_i) → (kem_ct, shared); + * AEAD-seal share_i under HKDF(shared) → wrapped_share_i; write entry. + * 4. AEAD-seal current DEK under R → envelope. + * 5. Atomic-write both files; fsync. + * 6. Zero R, all shared secrets. + */ + (void)threshold; (void)total; + (void)shareholder_pks; (void)shareholder_pk_lens; (void)shareholder_ids; + (void)shares_out_path; (void)envelope_out_path; + return -1; +} + +int engram_recovery_reconstitute(const uint8_t* k_unwrapped_shares, + size_t share_count, + const char* envelope_path, + uint8_t out_dek[32]) +{ + /* TODO: + * 1. Lagrange-interpolate to recover R from the K shares. + * 2. AEAD-open the envelope under HKDF(R) → DEK. + * 3. Zero R. + */ + (void)k_unwrapped_shares; (void)share_count; + (void)envelope_path; (void)out_dek; + return -1; +} + +/* ── Snapshot integration sketch ──────────────────────────────────────────── */ + +/* + * engram_emit_node_json (around line 3409 in el_runtime.c) becomes: + * + * if (g_engram_seal_mode == 1) { + * char* sealed = engram_aead_seal_field(n->id, n->content); + * jb_puts(b, ",\"content\":"); jb_emit_escaped(b, sealed ? sealed : ""); + * free(sealed); + * // same for label, tags, metadata + * } else { + * jb_puts(b, ",\"content\":"); jb_emit_escaped(b, n->content); + * } + * + * engram_load (around line 3499) becomes the inverse: when reading each field, + * if the daemon is in `fields` mode and the value is non-empty, run + * engram_aead_open_field; on failure log and substitute "". + * + * `engram_save` must additionally write engram.kek.enc the first time (via + * engram_kek_init) when ENGRAM_SEAL_MODE != off and no kek file exists. + */ diff --git a/engram/spec/engram-el.md b/engram/spec/engram-el.md new file mode 100644 index 0000000..7dd88aa --- /dev/null +++ b/engram/spec/engram-el.md @@ -0,0 +1,284 @@ +# engram-el Specification + +Version 1.0.0 — April 29, 2026 + +--- + +## Overview + +engram-el is the El-native interface layer for the Engram graph engine. It is the integration point between El programs and the Engram knowledge substrate — providing a suite of El programs, test suites, and utilities that operate on a live Engram server via its HTTP API using El's native HTTP builtins. + +engram-el has three primary components: + +1. **Studio** — A full-featured terminal-based graph explorer written in El (`studio/studio.el`). Provides read access to all graph data: statistics, node browsing by type and tier, spreading activation visualization, edge exploration, and text search. + +2. **Test suite** — Language feature tests (`test/language_features_test.el`, `test/field_test.el`, `test/llm_test.el`) that exercise El builtins against a live Engram instance. + +3. **Integration point** — The pattern for how El programs use the Engram graph as their knowledge substrate, demonstrating the graph builtin API in practice. + +--- + +## 1. Architecture + +### 1.1 Relationship to El and Engram + +engram-el is not a library in the conventional sense. It is a collection of El programs that operate on Engram. The integration uses no additional runtime or SDK: + +- **El builtins** provide `http_get`, `http_post`, and JSON parsing natively. +- **Engram HTTP API** is the sole interface — all graph operations are HTTP requests. +- **No compilation step** beyond standard El compilation is required. + +This demonstrates the intended usage pattern for all El programs that incorporate graph knowledge: use the HTTP API via El's native builtins. + +### 1.2 Configuration + +All engram-el programs read configuration from environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `ENGRAM_URL` | `http://localhost:8340` | Engram server base URL | +| `ENGRAM_REPORT` | `/tmp/engram-studio-report.txt` | Studio report output path | + +--- + +## 2. Studio Application + +`studio/studio.el` is a complete data exploration application for the Engram graph, written entirely in El. It demonstrates El as a serious application language — not a scripting language but a capable system for building non-trivial tools. + +### 2.1 Features + +The studio renders a full-page terminal UI with box-drawing characters and ANSI color. Sections: + +| Section | Description | +|---------|-------------| +| Database Statistics | Node count, edge count, average salience, DB size | +| Recent Nodes | Most recently created nodes with type and salience | +| Top by Salience | Highest-salience nodes with graphical bar display | +| Nodes by Type | Browse Memory, Concept, Event, Entity, Process, InternalState | +| Nodes by Tier | Browse Working, Episodic, Semantic, Procedural tiers | +| Knowledge Browser | Concept nodes as domain knowledge anchors | +| Text Search | Full-text search results with relevance | +| Edge Explorer | Sample of edges with weights and relation types | +| Node Detail | Full node data plus BFS neighbors | +| Spreading Activation | Visual activation surface from a seed node | +| Interactive Mode Preview | Menu of available commands | +| Report Export | Write complete session report to file | + +### 2.2 API Access Pattern + +The studio uses a uniform API access pattern: + +```el +fn api_get(path: String) -> String { + let url: String = get_base_url() + path + let resp: String = http_get(url) + if str_starts_with(resp, "{\"error\"") { + return "" + } + return resp +} + +fn api_post(path: String, body: String) -> String { + let url: String = get_base_url() + path + let resp: String = http_post(url, body) + if str_starts_with(resp, "{\"error\"") { + return "" + } + return resp +} +``` + +Error responses (JSON objects beginning with `{"error"`) return empty string. All rendering logic checks for empty string and emits placeholder messages rather than crashing. + +### 2.3 Spreading Activation Visualization + +The activation section demonstrates reading live spreading activation results from Engram: + +```el +fn show_activation(seed_id: String, limit: Int, report: String) -> String { + let path: String = "/api/activate?seeds=" + seed_id + "&limit=" + int_to_str(limit) + "&depth=3" + let json_str: String = api_get(path) + // ... renders activation strength bars and hop distances +} +``` + +This provides visual confirmation that the spreading activation algorithm is operating — showing which nodes activate, at what strength, and at what hop distance from the seed. + +### 2.4 Report Export + +The studio accumulates a text report as it renders each section, then writes the complete report to a file: + +```el +export_report(report, report_path) +``` + +The report captures the full session output in machine-readable format, useful for automation and logging. + +--- + +## 3. Test Suite + +### 3.1 Language Features Test + +`test/language_features_test.el` exercises El language primitives including: + +- Modulo operator (`%`) +- Bitwise operators (`&`, `^`, `<<`, `>>`) +- Math builtins (`math_sin`, `math_cos`, `math_pi`) +- String padding (`str_pad_left`, `str_pad_right`) +- String formatting (`str_format` with `{key}` template interpolation) +- Float formatting (`format_float`) +- Time operations (`time_now_utc`, `time_format`, `time_add`, `time_diff`) +- List operations (`list_range`, `list_join`) +- Stack and queue builtins (`stack_new`, `stack_push`, `stack_pop`, `stack_peek`, `queue_enqueue`, `queue_dequeue`) +- Decimal rounding (`decimal_round`) +- Type conversion (`int_to_float`, `float_to_int`) +- Nil checks (`is_nil`, `unwrap_or`) +- Character operations (`str_char_at`, `str_char_code`, `str_from_char_code`) + +These tests serve as the canonical behavioral specification for El builtins — any correct El implementation must produce the documented output for these inputs. + +### 3.2 Field Test + +`test/field_test.el` exercises struct field access, map indexing, and nested data access patterns. + +### 3.3 LLM Test + +`test/llm_test.el` exercises the LLM inference builtins against a live Engram-connected inference endpoint. + +--- + +## 4. Integration Patterns + +### 4.1 Graph Read Pattern + +The standard pattern for reading from Engram in an El program: + +```el +fn get_nodes_of_type(node_type: String, limit: Int) -> List { + let path: String = "/api/nodes?node_type=" + node_type + "&limit=" + int_to_str(limit) + let json_str: String = http_get(env("ENGRAM_URL") + path) + if json_str == "" { + return list_new() + } + return json_parse(json_str) +} +``` + +### 4.2 Graph Write Pattern + +The standard pattern for writing to Engram from an El program: + +```el +fn create_node(label: String, content: String, node_type: String, tier: String) -> String { + let body: String = "{\"label\":\"" + label + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"tier\":\"" + tier + "\",\"importance\":0.5}" + let resp: String = http_post(env("ENGRAM_URL") + "/api/nodes", body) + return json_get_string(resp, "id") +} +``` + +### 4.3 Search Pattern + +```el +fn search_graph(query: String, limit: Int) -> List { + let path: String = "/api/search?q=" + query + "&limit=" + int_to_str(limit) + let json_str: String = http_get(env("ENGRAM_URL") + path) + if json_str == "" { + return list_new() + } + return json_parse(json_str) +} +``` + +### 4.4 Activation Pattern + +```el +fn activate_from_node(node_id: String, depth: Int, limit: Int) -> List { + let path: String = "/api/activate?seeds=" + node_id + "&depth=" + int_to_str(depth) + "&limit=" + int_to_str(limit) + let resp_str: String = http_get(env("ENGRAM_URL") + path) + if resp_str == "" { + return list_new() + } + let results_raw: String = json_get_raw(resp_str, "results") + return json_parse(results_raw) +} +``` + +--- + +## 5. Builtin Extensions Demonstrated + +The engram-el programs demonstrate El builtins that are not in the core language but are implemented by the VM's builtin dispatch layer: + +### 5.1 JSON Builtins + +| Builtin | Used for | +|---------|---------| +| `json_parse(s)` | Parse Engram API responses | +| `json_stringify(v)` | Serialize values to JSON for API requests | +| `json_get_string(json, key)` | Extract string fields from node JSON | +| `json_get_int(json, key)` | Extract integer fields (counts, timestamps) | +| `json_get_float(json, key)` | Extract float fields (salience, weights) | +| `json_get_raw(json, key)` | Extract nested objects as raw JSON strings | + +### 5.2 Color/Terminal Builtins + +| Builtin | Used for | +|---------|---------| +| `color_bold(s)` | Section headers, labels | +| `color_dim(s)` | Timestamps, IDs, less important data | +| `color_green(s)` | Success states, high salience | +| `color_yellow(s)` | Warnings, medium salience | +| `color_cyan(s)` | URLs, relation names, special values | +| `color_red(s)` | Errors, low salience | + +### 5.3 String Formatting Builtins + +| Builtin | Signature | Description | +|---------|-----------|-------------| +| `str_pad_right(s, width, pad)` | Pad string to width on right | +| `str_pad_left(s, width, pad)` | Pad string to width on left | +| `format_float(f, decimals)` | Format float to N decimal places | +| `str_slice(s, start, end)` | Extract substring by character index | +| `str_len(s)` | String length in characters | + +--- + +## 6. Deployment + +### 6.1 Running the Studio + +```bash +# Connect to default local server +el run-file studio/studio.el + +# Connect to remote server +ENGRAM_URL=http://engram.example.com el run-file studio/studio.el + +# Save report to custom path +ENGRAM_REPORT=/var/log/engram-report.txt el run-file studio/studio.el +``` + +### 6.2 Running Tests + +```bash +el run-file test/language_features_test.el +el run-file test/field_test.el +ENGRAM_URL=http://localhost:8340 el run-file test/llm_test.el +``` + +--- + +## 7. Design Decisions + +### 7.1 Pure HTTP Integration + +engram-el uses HTTP exclusively. It does not use the lower-level `graph_compile` and `graph_traverse` VM builtins. This is by design: it demonstrates the HTTP API surface as the primary integration mechanism. The VM builtins are for tightly-integrated runtime code (the Neuron daemon); external tools use the HTTP API. + +### 7.2 Stateless Programs + +All engram-el programs are stateless — they read state from Engram on each run and write nothing back (the studio is read-only). This is the correct architecture for exploration tools: they observe the graph without mutating it. + +### 7.3 El as Application Language + +The studio's 788 lines of El demonstrate that El is a capable application language. It is not a configuration DSL or a scripting language for simple tasks. The studio handles: API communication, JSON parsing, recursive data rendering, ASCII art, ANSI color codes, file I/O, environment variable configuration, and complex string manipulation — all with El's native builtins, without imports. diff --git a/engram/src/server.el b/engram/src/server.el new file mode 100644 index 0000000..ebaf34a --- /dev/null +++ b/engram/src/server.el @@ -0,0 +1,319 @@ +// server.el — Engram HTTP server. +// +// Engram is the in-process graph store. The runtime owns the data; this +// file is the thin HTTP face. Every route maps to one or two engram_* +// builtins. There is no SQL, no db layer, no SQLite — the runtime IS the +// database. +// +// Built and linked with: +// elc src/server.el > server.c +// cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c +// ./engram +// +// Configuration via environment: +// ENGRAM_BIND — host:port (default :8742) +// ENGRAM_API_KEY — bearer auth (optional) +// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram) + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +fn parse_port(bind: String) -> Int { + // ":8742" → 8742; "0.0.0.0:8742" → 8742; bare "8742" → 8742 + let colon: Int = str_index_of(bind, ":") + if colon < 0 { + return str_to_int(bind) + } + let after: String = str_slice(bind, colon + 1, str_len(bind)) + return str_to_int(after) +} + +fn ok_json() -> String { + "{\"ok\":true}" +} + +fn err_json(msg: String) -> String { + "{\"error\":\"" + msg + "\"}" +} + +fn strip_query(path: String) -> String { + let q: Int = str_index_of(path, "?") + if q < 0 { return path } + str_slice(path, 0, q) +} + +fn query_param(path: String, key: String) -> String { + let q: Int = str_index_of(path, "?") + if q < 0 { return "" } + let qs: String = str_slice(path, q + 1, str_len(path)) + let needle: String = key + "=" + let pos: Int = str_index_of(qs, needle) + if pos < 0 { return "" } + let after: String = str_slice(qs, pos + str_len(needle), str_len(qs)) + let amp: Int = str_index_of(after, "&") + if amp < 0 { return after } + str_slice(after, 0, amp) +} + +fn query_int(path: String, key: String, default_val: Int) -> Int { + let v: String = query_param(path, key) + if str_eq(v, "") { return default_val } + str_to_int(v) +} + +// Extract last path segment after a known prefix: extract_id("/api/nodes/abc-123", "/api/nodes/") → "abc-123" +fn extract_id(path: String, prefix: String) -> String { + let clean: String = strip_query(path) + if !str_starts_with(clean, prefix) { return "" } + let after: String = str_slice(clean, str_len(prefix), str_len(clean)) + let slash: Int = str_index_of(after, "/") + if slash < 0 { return after } + str_slice(after, 0, slash) +} + +// ── Routes ──────────────────────────────────────────────────────────────────── + +fn route_stats(method: String, path: String, body: String) -> String { + engram_stats_json() +} + +fn route_create_node(method: String, path: String, body: String) -> String { + let content: String = json_get_string(body, "content") + let node_type: String = json_get_string(body, "node_type") + if str_eq(node_type, "") { let node_type = "Memory" } + let salience: Float = json_get_float(body, "salience") + if salience == 0.0 { let salience = 0.5 } + let id: String = engram_node(content, node_type, salience) + "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}" +} + +fn route_get_node(method: String, path: String, body: String) -> String { + let id: String = extract_id(path, "/api/nodes/") + if str_eq(id, "") { return err_json("missing id") } + return engram_get_node_json(id) +} + +fn route_scan_nodes(method: String, path: String, body: String) -> String { + let limit: Int = query_int(path, "limit", 50) + let offset: Int = query_int(path, "offset", 0) + let nt: String = query_param(path, "node_type") + if str_eq(nt, "") { + return engram_scan_nodes_json(limit, offset) + } + return engram_scan_nodes_by_type_json(nt, limit, offset) +} + +// route_scan_edges — bulk export of all edges as a JSON array. Implemented +// via engram_save → fs_read of the canonical on-disk snapshot, which the +// runtime keeps in lockstep with the in-memory graph. Live against the +// running graph, not a stale export. +fn route_scan_edges(method: String, path: String, body: String) -> String { + let dir: String = env("ENGRAM_DATA_DIR") + if str_eq(dir, "") { let dir = "/tmp/engram" } + let snap_path: String = dir + "/snapshot.json" + engram_save(snap_path) + let snap: String = fs_read(snap_path) + if str_eq(snap, "") { return "[]" } + // json_get truncates at the first delimiter (no bracket depth tracking), + // so for the edges ARRAY value we need json_get_raw, which honors + // brackets and returns the full sub-JSON. + let edges: String = json_get_raw(snap, "edges") + if str_eq(edges, "") { return "[]" } + return edges +} + +fn route_search(method: String, path: String, body: String) -> String { + let q: String = "" + if str_eq(method, "GET") { + let q = query_param(path, "q") + } else { + let q = json_get_string(body, "query") + } + let limit: Int = query_int(path, "limit", 20) + if limit == 0 { let limit = json_get_int(body, "limit") } + if limit == 0 { let limit = 20 } + return engram_search_json(q, limit) +} + +fn route_activate(method: String, path: String, body: String) -> String { + let q: String = "" + let depth: Int = 3 + if str_eq(method, "GET") { + let q = query_param(path, "q") + let depth = query_int(path, "depth", 3) + } else { + let q = json_get_string(body, "query") + let bd: Int = json_get_int(body, "depth") + if bd > 0 { let depth = bd } + } + return "{\"results\":" + engram_activate_json(q, depth) + "}" +} + +fn route_create_edge(method: String, path: String, body: String) -> String { + let from_id: String = json_get_string(body, "from_id") + let to_id: String = json_get_string(body, "to_id") + let relation: String = json_get_string(body, "relation") + if str_eq(relation, "") { let relation = "associates" } + let weight: Float = json_get_float(body, "weight") + if weight == 0.0 { let weight = 0.5 } + engram_connect(from_id, to_id, weight, relation) + "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}" +} + +fn route_neighbors(method: String, path: String, body: String) -> String { + let id: String = extract_id(path, "/api/neighbors/") + if str_eq(id, "") { return err_json("missing id") } + let depth: Int = query_int(path, "depth", 1) + return engram_neighbors_json(id, depth, "both") +} + +fn route_strengthen(method: String, path: String, body: String) -> String { + let id: String = json_get_string(body, "node_id") + if str_eq(id, "") { return err_json("missing node_id") } + engram_strengthen(id) + ok_json() +} + +fn route_forget(method: String, path: String, body: String) -> String { + let id: String = extract_id(path, "/api/nodes/") + if str_eq(id, "") { return err_json("missing id") } + engram_forget(id) + ok_json() +} + +fn route_save(method: String, path: String, body: String) -> String { + let p: String = json_get_string(body, "path") + if str_eq(p, "") { + let dir: String = env("ENGRAM_DATA_DIR") + if str_eq(dir, "") { let dir = "/tmp/engram" } + let p = dir + "/snapshot.json" + } + engram_save(p) + "{\"ok\":true,\"path\":\"" + p + "\"}" +} + +fn route_load(method: String, path: String, body: String) -> String { + let p: String = json_get_string(body, "path") + if str_eq(p, "") { + let dir: String = env("ENGRAM_DATA_DIR") + if str_eq(dir, "") { let dir = "/tmp/engram" } + let p = dir + "/snapshot.json" + } + engram_load(p) + ok_json() +} + +fn route_health(method: String, path: String, body: String) -> String { + "{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}" +} + +// ── Auth ────────────────────────────────────────────────────────────────────── + +fn check_auth_ok(method: String, body: String) -> Bool { + let key: String = env("ENGRAM_API_KEY") + if str_eq(key, "") { return true } + // Read-only methods don't require auth. Until http_serve surfaces + // request headers we can't accept a Bearer token cleanly; mutating + // requests must include "_auth": "" in the JSON body. + if str_eq(method, "GET") { return true } + let provided: String = json_get_string(body, "_auth") + if str_eq(provided, key) { return true } + return false +} + +// ── Dispatcher ──────────────────────────────────────────────────────────────── + +fn handle_request(method: String, path: String, body: String) -> String { + let clean: String = strip_query(path) + + // Health is always reachable + if str_eq(method, "GET") { + if str_eq(clean, "/health") || str_eq(clean, "/") { + return route_health(method, path, body) + } + } + + // Auth (when ENGRAM_API_KEY is set) + if !check_auth_ok(method, body) { + return err_json("unauthorized") + } + + // Stats + if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) { + return route_stats(method, path, body) + } + + // Nodes + if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) { + return route_create_node(method, path, body) + } + if str_eq(method, "GET") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes") || str_eq(clean, "/nodes/list") || str_eq(clean, "/api/nodes/list")) { + return route_scan_nodes(method, path, body) + } + if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) { + return route_scan_edges(method, path, body) + } + if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") { + return route_get_node(method, path, body) + } + if str_eq(method, "DELETE") && str_starts_with(clean, "/api/nodes/") { + return route_forget(method, path, body) + } + + // Edges + if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) { + return route_create_edge(method, path, body) + } + if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") { + return route_neighbors(method, path, body) + } + + // Activation + Search + if str_eq(method, "POST") && (str_eq(clean, "/api/activate") || str_eq(clean, "/activate")) { + return route_activate(method, path, body) + } + if str_eq(method, "GET") && str_starts_with(clean, "/api/activate") { + return route_activate(method, path, body) + } + if str_eq(method, "POST") && (str_eq(clean, "/api/search") || str_eq(clean, "/search")) { + return route_search(method, path, body) + } + if str_eq(method, "GET") && str_starts_with(clean, "/api/search") { + return route_search(method, path, body) + } + + // Strengthen + if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) { + return route_strengthen(method, path, body) + } + + // Persistence + if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) { + return route_save(method, path, body) + } + if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) { + return route_load(method, path, body) + } + + "{\"error\":\"not found\",\"path\":\"" + clean + "\"}" +} + +// ── Entry ───────────────────────────────────────────────────────────────────── + +let bind_str: String = env("ENGRAM_BIND") +if str_eq(bind_str, "") { let bind_str = ":8742" } +let port: Int = parse_port(bind_str) + +// On startup, try to load any existing snapshot (best effort). +let data_dir: String = env("ENGRAM_DATA_DIR") +if str_eq(data_dir, "") { let data_dir = "/tmp/engram" } +let snapshot_path: String = data_dir + "/snapshot.json" +engram_load(snapshot_path) + +println("[engram] runtime-native graph engine") +println("[engram] data_dir=" + data_dir) +println("[engram] node_count=" + int_to_str(engram_node_count())) +println("[engram] edge_count=" + int_to_str(engram_edge_count())) +println("[engram] listening on " + int_to_str(port)) + +http_set_handler("handle_request") +http_serve(port, "handle_request")