monorepo: fold engram into engram/ (history preserved)
This commit is contained in:
Will Anderson
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,121 @@
|
||||
name: Engram CI — dev
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
pull_request:
|
||||
branches:
|
||||
- dev
|
||||
|
||||
jobs:
|
||||
build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y gcc libcurl4-openssl-dev
|
||||
|
||||
# Install El SDK from dev artifact registry
|
||||
- name: Install El SDK from dev registry
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
apt-get install -y -qq apt-transport-https ca-certificates gnupg curl
|
||||
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
|
||||
mkdir -p /usr/local/bin /usr/local/lib/el
|
||||
|
||||
# Download the latest-versioned elc from dev registry
|
||||
# Fall back to the Gitea latest release if no dev artifact exists yet
|
||||
LATEST_VERSION=$(gcloud artifacts versions list \
|
||||
--repository=foundation-dev \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=el/elc \
|
||||
--format="value(name)" 2>/dev/null | sort | tail -1 | sed 's|.*/||' || true)
|
||||
|
||||
if [ -n "${LATEST_VERSION}" ]; then
|
||||
echo "Downloading elc ${LATEST_VERSION} from foundation-dev..."
|
||||
gcloud artifacts generic download \
|
||||
--repository=foundation-dev \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=el/elc \
|
||||
--version="${LATEST_VERSION}" \
|
||||
--destination=/usr/local/bin/elc
|
||||
chmod +x /usr/local/bin/elc
|
||||
else
|
||||
echo "No dev artifact found, falling back to Gitea latest release..."
|
||||
RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest"
|
||||
curl -fsSL "${RELEASE_BASE}/elc" -o /usr/local/bin/elc
|
||||
chmod +x /usr/local/bin/elc
|
||||
fi
|
||||
|
||||
# Always get runtime from Gitea latest release
|
||||
RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest"
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.c" -o /usr/local/lib/el/el_runtime.c
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h
|
||||
|
||||
echo "El SDK installed:"
|
||||
elc --version || true
|
||||
rm -f /tmp/gcp-key.json
|
||||
|
||||
# Compile server.el → dist/engram.c
|
||||
- name: Compile server.el
|
||||
run: |
|
||||
mkdir -p dist
|
||||
elc src/server.el > dist/engram.c
|
||||
echo "Compiled src/server.el → dist/engram.c"
|
||||
|
||||
# Link to produce the engram binary
|
||||
- name: Link engram binary
|
||||
run: |
|
||||
cc -std=c11 -O2 \
|
||||
-I /usr/local/lib/el \
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
/usr/local/lib/el/el_runtime.c \
|
||||
-lcurl -lpthread
|
||||
echo "Linked dist/engram"
|
||||
ls -lh dist/engram
|
||||
|
||||
# Run tests if spec directory contains runnable tests
|
||||
- name: Run tests
|
||||
run: |
|
||||
if [ -f spec/run.sh ]; then
|
||||
bash spec/run.sh
|
||||
else
|
||||
echo "No spec/run.sh found — skipping test run"
|
||||
fi
|
||||
|
||||
# Publish artifact to GCP Artifact Registry (dev)
|
||||
- name: Publish engram to Artifact Registry (dev)
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
|
||||
VERSION="${GITEA_SHA:0:8}"
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-dev \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=engram/engram \
|
||||
--version="${VERSION}" \
|
||||
--source=dist/engram
|
||||
|
||||
echo "Published engram version=${VERSION} to foundation-dev/engram/engram"
|
||||
rm -f /tmp/gcp-key.json
|
||||
@@ -0,0 +1,128 @@
|
||||
name: Engram CI — stage
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- stage
|
||||
pull_request:
|
||||
branches:
|
||||
- stage
|
||||
|
||||
jobs:
|
||||
build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Enforce source branch (stage ← dev only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
SOURCE="${GITHUB_HEAD_REF}"
|
||||
if [ "${SOURCE}" != "dev" ]; then
|
||||
echo "ERROR: Stage branch only accepts PRs from 'dev'. Source was: '${SOURCE}'"
|
||||
exit 1
|
||||
fi
|
||||
echo "Source branch check passed: ${SOURCE} → stage"
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y gcc libcurl4-openssl-dev
|
||||
|
||||
# Install El SDK from stage artifact registry
|
||||
- name: Install El SDK from stage registry
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
apt-get install -y -qq apt-transport-https ca-certificates gnupg curl
|
||||
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
|
||||
mkdir -p /usr/local/bin /usr/local/lib/el
|
||||
|
||||
LATEST_VERSION=$(gcloud artifacts versions list \
|
||||
--repository=foundation-stage \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=el/elc \
|
||||
--format="value(name)" 2>/dev/null | sort | tail -1 | sed 's|.*/||' || true)
|
||||
|
||||
if [ -n "${LATEST_VERSION}" ]; then
|
||||
echo "Downloading elc ${LATEST_VERSION} from foundation-stage..."
|
||||
gcloud artifacts generic download \
|
||||
--repository=foundation-stage \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=el/elc \
|
||||
--version="${LATEST_VERSION}" \
|
||||
--destination=/usr/local/bin/elc
|
||||
chmod +x /usr/local/bin/elc
|
||||
else
|
||||
echo "No stage artifact found, falling back to Gitea latest release..."
|
||||
RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest"
|
||||
curl -fsSL "${RELEASE_BASE}/elc" -o /usr/local/bin/elc
|
||||
chmod +x /usr/local/bin/elc
|
||||
fi
|
||||
|
||||
RELEASE_BASE="https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest"
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.c" -o /usr/local/lib/el/el_runtime.c
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h
|
||||
|
||||
echo "El SDK installed:"
|
||||
elc --version || true
|
||||
rm -f /tmp/gcp-key.json
|
||||
|
||||
# Compile server.el → dist/engram.c
|
||||
- name: Compile server.el
|
||||
run: |
|
||||
mkdir -p dist
|
||||
elc src/server.el > dist/engram.c
|
||||
echo "Compiled src/server.el → dist/engram.c"
|
||||
|
||||
# Link to produce the engram binary
|
||||
- name: Link engram binary
|
||||
run: |
|
||||
cc -std=c11 -O2 \
|
||||
-I /usr/local/lib/el \
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
/usr/local/lib/el/el_runtime.c \
|
||||
-lcurl -lpthread
|
||||
echo "Linked dist/engram"
|
||||
ls -lh dist/engram
|
||||
|
||||
# Run tests if spec directory contains runnable tests
|
||||
- name: Run tests
|
||||
run: |
|
||||
if [ -f spec/run.sh ]; then
|
||||
bash spec/run.sh
|
||||
else
|
||||
echo "No spec/run.sh found — skipping test run"
|
||||
fi
|
||||
|
||||
# Publish artifact to GCP Artifact Registry (stage)
|
||||
- name: Publish engram to Artifact Registry (stage)
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
|
||||
VERSION="${GITEA_SHA:0:8}"
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-stage \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=engram/engram \
|
||||
--version="${VERSION}" \
|
||||
--source=dist/engram
|
||||
|
||||
echo "Published engram version=${VERSION} to foundation-stage/engram/engram"
|
||||
rm -f /tmp/gcp-key.json
|
||||
@@ -0,0 +1,159 @@
|
||||
name: Engram Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
repository_dispatch:
|
||||
types:
|
||||
- el-sdk-updated
|
||||
|
||||
jobs:
|
||||
build-and-release:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Enforce source branch (main ← stage only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
SOURCE="${GITHUB_HEAD_REF}"
|
||||
if [ "${SOURCE}" != "stage" ]; then
|
||||
echo "ERROR: Main branch only accepts PRs from 'stage'. Source was: '${SOURCE}'"
|
||||
exit 1
|
||||
fi
|
||||
echo "Source branch check passed: ${SOURCE} → main"
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y gcc libcurl4-openssl-dev
|
||||
|
||||
# Install El SDK from the latest release
|
||||
- name: Install El SDK
|
||||
env:
|
||||
GITEA_API: https://git.neuralplatform.ai/api/v1
|
||||
RELEASE_BASE: https://git.neuralplatform.ai/neuron-technologies/el/releases/download/latest
|
||||
run: |
|
||||
mkdir -p /usr/local/bin /usr/local/lib/el
|
||||
|
||||
echo "Downloading elc..."
|
||||
curl -fsSL "${RELEASE_BASE}/elc" -o /usr/local/bin/elc
|
||||
chmod +x /usr/local/bin/elc
|
||||
|
||||
echo "Downloading el_runtime.c..."
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.c" -o /usr/local/lib/el/el_runtime.c
|
||||
|
||||
echo "Downloading el_runtime.h..."
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h
|
||||
|
||||
echo "El SDK installed:"
|
||||
elc --version || true
|
||||
|
||||
# Compile server.el → dist/engram.c
|
||||
- name: Compile server.el
|
||||
run: |
|
||||
mkdir -p dist
|
||||
elc src/server.el > dist/engram.c
|
||||
echo "Compiled src/server.el → dist/engram.c"
|
||||
|
||||
# Link to produce the engram binary
|
||||
- name: Link engram binary
|
||||
run: |
|
||||
cc -std=c11 -O2 \
|
||||
-I /usr/local/lib/el \
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
/usr/local/lib/el/el_runtime.c \
|
||||
-lcurl -lpthread
|
||||
echo "Linked dist/engram"
|
||||
ls -lh dist/engram
|
||||
|
||||
# Publish / update the `latest` release
|
||||
- name: Publish latest release
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GITEA_API: https://git.neuralplatform.ai/api/v1
|
||||
REPO: neuron-technologies/engram
|
||||
run: |
|
||||
# Delete existing `latest` release if present
|
||||
EXISTING_ID=$(curl -sf \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
"${GITEA_API}/repos/${REPO}/releases/tags/latest" \
|
||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print(d['id'])" 2>/dev/null || true)
|
||||
|
||||
if [ -n "${EXISTING_ID}" ]; then
|
||||
echo "Deleting existing release id=${EXISTING_ID}"
|
||||
curl -sf -X DELETE \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
"${GITEA_API}/repos/${REPO}/releases/${EXISTING_ID}"
|
||||
fi
|
||||
|
||||
# Remove stale tag
|
||||
curl -sf -X DELETE \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
"${GITEA_API}/repos/${REPO}/tags/latest" || true
|
||||
|
||||
# Create new release
|
||||
RELEASE_ID=$(curl -sf -X POST \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${GITEA_API}/repos/${REPO}/releases" \
|
||||
-d "{
|
||||
\"tag_name\": \"latest\",
|
||||
\"name\": \"Engram (latest)\",
|
||||
\"body\": \"Latest Engram build from commit ${GITHUB_SHA}.\nBuilt $(date -u +%Y-%m-%dT%H:%M:%SZ).\",
|
||||
\"draft\": false,
|
||||
\"prerelease\": false
|
||||
}" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
|
||||
|
||||
echo "Created release id=${RELEASE_ID}"
|
||||
|
||||
# Upload engram binary
|
||||
echo "Uploading engram binary..."
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-F "attachment=@dist/engram;filename=engram" \
|
||||
"${GITEA_API}/repos/${REPO}/releases/${RELEASE_ID}/assets"
|
||||
|
||||
echo "Release published successfully"
|
||||
|
||||
# Publish artifact to GCP Artifact Registry (prod)
|
||||
- name: Publish engram to Artifact Registry (prod)
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
apt-get install -y -qq apt-transport-https ca-certificates gnupg curl
|
||||
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
|
||||
VERSION="${GITEA_SHA:0:8}"
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-prod \
|
||||
--location=us-central1 \
|
||||
--project=neuron-785695 \
|
||||
--package=engram/engram \
|
||||
--version="${VERSION}" \
|
||||
--source=dist/engram
|
||||
|
||||
echo "Published engram version=${VERSION} to foundation-prod/engram/engram"
|
||||
rm -f /tmp/gcp-key.json
|
||||
|
||||
# Dispatch engram-updated to downstream dependents
|
||||
- name: Dispatch engram-updated to elql
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GITEA_API: https://git.neuralplatform.ai/api/v1
|
||||
run: |
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${GITEA_API}/repos/neuron-technologies/elql/dispatches" \
|
||||
-d "{\"type\":\"engram-updated\",\"inputs\":{\"engram_version\":\"latest\",\"commit\":\"${GITHUB_SHA}\"}}"
|
||||
echo "Dispatched engram-updated to neuron-technologies/elql"
|
||||
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
*.db
|
||||
.DS_Store
|
||||
@@ -0,0 +1,179 @@
|
||||
# Engram
|
||||
|
||||
**A local-first memory substrate for accumulating intelligence.**
|
||||
|
||||
An *engram* is the physical trace of a memory in the brain — the actual encoded substrate, not an abstraction above it. That's what this is.
|
||||
|
||||
---
|
||||
|
||||
## Why existing databases are wrong for this use case
|
||||
|
||||
Relational databases store rows and retrieve them by predicate. Key-value stores retrieve by exact key. Vector databases retrieve by geometric proximity. All of them share the same fundamental model: **you store data in, you query it out**. Storage and retrieval are separate systems.
|
||||
|
||||
The brain doesn't work this way.
|
||||
|
||||
When you remember something, you don't query your hippocampus. You activate a memory trace and the pattern *propagates*. Long-term potentiation — the strengthening of synaptic connections through co-activation — is simultaneously the storage mechanism and the retrieval mechanism. The structure that holds the memory is the same structure that surfaces it.
|
||||
|
||||
No existing database models this. Engram does.
|
||||
|
||||
---
|
||||
|
||||
## The Spreading Activation Model
|
||||
|
||||
Engram retrieval works through **spreading activation**:
|
||||
|
||||
1. **Seeds** — you name one or more nodes you know are relevant (e.g. the current task, recent context, a concept you're reasoning about)
|
||||
2. **Query embedding** — you provide a semantic vector representing the direction of your current thought
|
||||
3. **Propagation** — activation flows outward from seeds through weighted edges. At each hop, strength attenuates multiplicatively:
|
||||
|
||||
```
|
||||
strength = parent_strength × edge_weight × target_salience × cosine_sim(query, target)
|
||||
```
|
||||
|
||||
4. **Pruning** — paths weaker than a threshold are cut (the attention filter)
|
||||
5. **Return** — the top-N nodes by activation strength
|
||||
|
||||
This is not a query. It is a *pattern completion*. The system surfaces what is most associatively relevant to the current context, weighted by how strongly those things have been reinforced over time.
|
||||
|
||||
---
|
||||
|
||||
## The Four Memory Tiers
|
||||
|
||||
| Tier | Analogy | Contents |
|
||||
|------|---------|----------|
|
||||
| `Working` | Prefrontal working memory | K most recently activated nodes — hot, fast |
|
||||
| `Episodic` | Hippocampus | Time-ordered events and experiences |
|
||||
| `Semantic` | Neocortex | Concept graph — long-term structural knowledge |
|
||||
| `Procedural` | Cerebellum / basal ganglia | Patterns, workflows, habits |
|
||||
|
||||
Nodes migrate between tiers based on salience decay and reinforcement. A frequently activated semantic node stays semantic. A rarely-touched episodic memory decays toward procedural background.
|
||||
|
||||
---
|
||||
|
||||
## Salience — Forgetting as Adaptation
|
||||
|
||||
Salience is not stored permanently. It decays:
|
||||
|
||||
```rust
|
||||
fn compute_salience(importance: f32, last_activated_ms: i64, activation_count: u64) -> f32 {
|
||||
let days_since = (now_ms() - last_activated_ms) as f32 / 86_400_000.0;
|
||||
importance * (1.0 / (1.0 + days_since)) * (activation_count as f32 + 1.0).ln()
|
||||
}
|
||||
```
|
||||
|
||||
Three signals:
|
||||
- **Importance** (0.0–1.0): set at creation, stable
|
||||
- **Recency**: decays toward zero as days pass without activation
|
||||
- **Frequency**: log-compressed count of activations
|
||||
|
||||
Forgetting in Engram is not a bug. It is adaptive pruning. Memories that are never activated again become less likely to surface during retrieval. They are not deleted — they remain in storage — but they stop competing for attention. This is exactly how biological memory works, and why it is adaptive rather than pathological.
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, RelationType};
|
||||
use std::path::Path;
|
||||
|
||||
// Open or create a database
|
||||
let db = EngramDb::open(Path::new("/var/lib/my-agent/memory"))?;
|
||||
|
||||
// Create a node with a semantic embedding
|
||||
let node = Node::new(
|
||||
NodeType::Concept,
|
||||
vec![0.9, 0.1, 0.3, 0.7, 0.8, 0.2], // embedding from your LLM
|
||||
b"Spreading activation surfaces relevant memories by pattern completion".to_vec(),
|
||||
MemoryTier::Semantic,
|
||||
0.9, // importance
|
||||
);
|
||||
let id = db.put_node(node)?;
|
||||
|
||||
// Link it to related concepts
|
||||
let related = db.put_node(Node::new(
|
||||
NodeType::Concept,
|
||||
vec![0.8, 0.2, 0.4, 0.6, 0.7, 0.3],
|
||||
b"Long-term potentiation: co-activation strengthens synaptic weight".to_vec(),
|
||||
MemoryTier::Semantic,
|
||||
0.85,
|
||||
))?;
|
||||
db.put_edge(Edge::new(id, related, RelationType::Causes, 0.9))?;
|
||||
|
||||
// Retrieve by spreading activation
|
||||
let results = db.activate(
|
||||
&[id], // seeds
|
||||
&[0.85, 0.15, 0.35, 0.65, 0.75, 0.25], // query embedding
|
||||
3, // max hops
|
||||
10, // top-N results
|
||||
)?;
|
||||
|
||||
for r in results {
|
||||
println!(
|
||||
"strength={:.4} hops={} — {}",
|
||||
r.activation_strength,
|
||||
r.hops,
|
||||
String::from_utf8_lossy(&r.node.content)
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
engram/
|
||||
crates/
|
||||
engram-core/ # The memory engine — storage, graph, activation, salience
|
||||
engram-ffi/ # C FFI stubs for cross-language bindings
|
||||
bindings/
|
||||
kotlin/ # Android / JVM binding notes
|
||||
typescript/ # WASM / Node binding notes
|
||||
go/ # CGo binding notes
|
||||
examples/
|
||||
basic.rs # Full walkthrough: insert, activate, search, decay
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Public API
|
||||
|
||||
```rust
|
||||
impl EngramDb {
|
||||
fn open(path: &Path) -> EngramResult<Self>;
|
||||
fn put_node(&self, node: Node) -> EngramResult<Uuid>;
|
||||
fn get_node(&self, id: Uuid) -> EngramResult<Option<Node>>;
|
||||
fn put_edge(&self, edge: Edge) -> EngramResult<()>;
|
||||
fn get_edges_from(&self, from_id: Uuid) -> EngramResult<Vec<Edge>>;
|
||||
fn get_edges_to(&self, to_id: Uuid) -> EngramResult<Vec<Edge>>;
|
||||
fn search_embedding(&self, embedding: &[f32], limit: usize) -> EngramResult<Vec<ScoredNode>>;
|
||||
fn activate(&self, seeds: &[Uuid], query_embedding: &[f32], max_depth: u8, limit: usize) -> EngramResult<Vec<ActivatedNode>>;
|
||||
fn traverse(&self, from: Uuid, relation: Option<RelationType>, max_depth: u8) -> EngramResult<Vec<Node>>;
|
||||
fn touch(&self, id: Uuid) -> EngramResult<()>;
|
||||
fn decay(&self, factor: f32) -> EngramResult<usize>;
|
||||
fn node_count(&self) -> EngramResult<usize>;
|
||||
fn edge_count(&self) -> EngramResult<usize>;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `sled` — embedded persistent B-tree (no daemon, no network, local-first)
|
||||
- `bincode` — compact binary serialization
|
||||
- `uuid` — stable node identity
|
||||
- `serde` — derive support
|
||||
- `thiserror` / `anyhow` — error handling
|
||||
|
||||
---
|
||||
|
||||
## Design Decisions
|
||||
|
||||
**Why sled?** Local-first. No daemon. Transactional. Fast enough for the node counts Engram targets (< 1M nodes). When the right HNSW index is needed, it will layer on top of sled, not replace it.
|
||||
|
||||
**Why flat cosine scan?** Correct and simple. The graph structure itself is the primary retrieval mechanism. Vector search is a secondary signal. HNSW adds complexity and a compile dependency that isn't justified until retrieval quality at scale demands it.
|
||||
|
||||
**Why multiplicative activation?** Because memory is conjunctive. A path requires all of its links to be strong to carry signal. Addition would allow many weak associations to accumulate into false relevance. Multiplication enforces that every factor matters.
|
||||
|
||||
**Why salience decay?** Because not everything that was once important remains important. Adaptive forgetting is not failure — it is the mechanism that keeps attention on what's current. A memory system that never forgets is one that can never focus.
|
||||
Vendored
+376
@@ -0,0 +1,376 @@
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include "el_runtime.h"
|
||||
|
||||
el_val_t parse_port(el_val_t bind);
|
||||
el_val_t ok_json(void);
|
||||
el_val_t err_json(el_val_t msg);
|
||||
el_val_t strip_query(el_val_t path);
|
||||
el_val_t query_param(el_val_t path, el_val_t key);
|
||||
el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
|
||||
el_val_t extract_id(el_val_t path, el_val_t prefix);
|
||||
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t check_auth_ok(el_val_t method, el_val_t body);
|
||||
el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body);
|
||||
|
||||
el_val_t bind_str;
|
||||
el_val_t port;
|
||||
el_val_t data_dir;
|
||||
el_val_t snapshot_path;
|
||||
|
||||
el_val_t parse_port(el_val_t bind) {
|
||||
el_val_t colon = str_index_of(bind, EL_STR(":"));
|
||||
if (colon < 0) {
|
||||
return str_to_int(bind);
|
||||
}
|
||||
el_val_t after = str_slice(bind, (colon + 1), str_len(bind));
|
||||
return str_to_int(after);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t ok_json(void) {
|
||||
return EL_STR("{\"ok\":true}");
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t err_json(el_val_t msg) {
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"error\":\""), msg), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t strip_query(el_val_t path) {
|
||||
el_val_t q = str_index_of(path, EL_STR("?"));
|
||||
if (q < 0) {
|
||||
return path;
|
||||
}
|
||||
return str_slice(path, 0, q);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t query_param(el_val_t path, el_val_t key) {
|
||||
el_val_t q = str_index_of(path, EL_STR("?"));
|
||||
if (q < 0) {
|
||||
return EL_STR("");
|
||||
}
|
||||
el_val_t qs = str_slice(path, (q + 1), str_len(path));
|
||||
el_val_t needle = el_str_concat(key, EL_STR("="));
|
||||
el_val_t pos = str_index_of(qs, needle);
|
||||
if (pos < 0) {
|
||||
return EL_STR("");
|
||||
}
|
||||
el_val_t after = str_slice(qs, (pos + str_len(needle)), str_len(qs));
|
||||
el_val_t amp = str_index_of(after, EL_STR("&"));
|
||||
if (amp < 0) {
|
||||
return after;
|
||||
}
|
||||
return str_slice(after, 0, amp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val) {
|
||||
el_val_t v = query_param(path, key);
|
||||
if (str_eq(v, EL_STR(""))) {
|
||||
return default_val;
|
||||
}
|
||||
return str_to_int(v);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t extract_id(el_val_t path, el_val_t prefix) {
|
||||
el_val_t clean = strip_query(path);
|
||||
if (!str_starts_with(clean, prefix)) {
|
||||
return EL_STR("");
|
||||
}
|
||||
el_val_t after = str_slice(clean, str_len(prefix), str_len(clean));
|
||||
el_val_t slash = str_index_of(after, EL_STR("/"));
|
||||
if (slash < 0) {
|
||||
return after;
|
||||
}
|
||||
return str_slice(after, 0, slash);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return engram_stats_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t content = json_get_string(body, EL_STR("content"));
|
||||
el_val_t node_type = json_get_string(body, EL_STR("node_type"));
|
||||
if (str_eq(node_type, EL_STR(""))) {
|
||||
node_type = EL_STR("Memory");
|
||||
}
|
||||
el_val_t salience = json_get_float(body, EL_STR("salience"));
|
||||
if (str_eq(salience, el_from_float(0.0))) {
|
||||
salience = el_from_float(0.5);
|
||||
}
|
||||
el_val_t id = engram_node(content, node_type, salience);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t id = extract_id(path, EL_STR("/api/nodes/"));
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing id"));
|
||||
}
|
||||
return engram_get_node_json(id);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t limit = query_int(path, EL_STR("limit"), 50);
|
||||
el_val_t offset = query_int(path, EL_STR("offset"), 0);
|
||||
el_val_t nt = query_param(path, EL_STR("node_type"));
|
||||
if (str_eq(nt, EL_STR(""))) {
|
||||
return engram_scan_nodes_json(limit, offset);
|
||||
}
|
||||
return engram_scan_nodes_by_type_json(nt, limit, offset);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
if (str_eq(snap, EL_STR(""))) {
|
||||
return EL_STR("[]");
|
||||
}
|
||||
el_val_t edges = json_get_raw(snap, EL_STR("edges"));
|
||||
if (str_eq(edges, EL_STR(""))) {
|
||||
return EL_STR("[]");
|
||||
}
|
||||
return edges;
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t q = EL_STR("");
|
||||
if (str_eq(method, EL_STR("GET"))) {
|
||||
q = query_param(path, EL_STR("q"));
|
||||
} else {
|
||||
q = json_get_string(body, EL_STR("query"));
|
||||
}
|
||||
el_val_t limit = query_int(path, EL_STR("limit"), 20);
|
||||
if (limit == 0) {
|
||||
limit = json_get_int(body, EL_STR("limit"));
|
||||
}
|
||||
if (limit == 0) {
|
||||
limit = 20;
|
||||
}
|
||||
return engram_search_json(q, limit);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t q = EL_STR("");
|
||||
el_val_t depth = 3;
|
||||
if (str_eq(method, EL_STR("GET"))) {
|
||||
q = query_param(path, EL_STR("q"));
|
||||
depth = query_int(path, EL_STR("depth"), 3);
|
||||
} else {
|
||||
q = json_get_string(body, EL_STR("query"));
|
||||
el_val_t bd = json_get_int(body, EL_STR("depth"));
|
||||
if (bd > 0) {
|
||||
depth = bd;
|
||||
}
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t from_id = json_get_string(body, EL_STR("from_id"));
|
||||
el_val_t to_id = json_get_string(body, EL_STR("to_id"));
|
||||
el_val_t relation = json_get_string(body, EL_STR("relation"));
|
||||
if (str_eq(relation, EL_STR(""))) {
|
||||
relation = EL_STR("associates");
|
||||
}
|
||||
el_val_t weight = json_get_float(body, EL_STR("weight"));
|
||||
if (str_eq(weight, el_from_float(0.0))) {
|
||||
weight = el_from_float(0.5);
|
||||
}
|
||||
engram_connect(from_id, to_id, weight, relation);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t id = extract_id(path, EL_STR("/api/neighbors/"));
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing id"));
|
||||
}
|
||||
el_val_t depth = query_int(path, EL_STR("depth"), 1);
|
||||
return engram_neighbors_json(id, depth, EL_STR("both"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t id = json_get_string(body, EL_STR("node_id"));
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing node_id"));
|
||||
}
|
||||
engram_strengthen(id);
|
||||
return ok_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t id = extract_id(path, EL_STR("/api/nodes/"));
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing id"));
|
||||
}
|
||||
engram_forget(id);
|
||||
return ok_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p = json_get_string(body, EL_STR("path"));
|
||||
if (str_eq(p, EL_STR(""))) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
p = el_str_concat(dir, EL_STR("/snapshot.json"));
|
||||
}
|
||||
engram_save(p);
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p = json_get_string(body, EL_STR("path"));
|
||||
if (str_eq(p, EL_STR(""))) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
p = el_str_concat(dir, EL_STR("/snapshot.json"));
|
||||
}
|
||||
engram_load(p);
|
||||
return ok_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}");
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t check_auth_ok(el_val_t method, el_val_t body) {
|
||||
el_val_t key = env(EL_STR("ENGRAM_API_KEY"));
|
||||
if (str_eq(key, EL_STR(""))) {
|
||||
return 1;
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET"))) {
|
||||
return 1;
|
||||
}
|
||||
el_val_t provided = json_get_string(body, EL_STR("_auth"));
|
||||
if (str_eq(provided, key)) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t clean = strip_query(path);
|
||||
if (str_eq(method, EL_STR("GET"))) {
|
||||
if (str_eq(clean, EL_STR("/health")) || str_eq(clean, EL_STR("/"))) {
|
||||
return route_health(method, path, body);
|
||||
}
|
||||
}
|
||||
if (!check_auth_ok(method, body)) {
|
||||
return err_json(EL_STR("unauthorized"));
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) {
|
||||
return route_stats(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) {
|
||||
return route_create_node(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")) || str_eq(clean, EL_STR("/nodes/list")) || str_eq(clean, EL_STR("/api/nodes/list")))) {
|
||||
return route_scan_nodes(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) {
|
||||
return route_scan_edges(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/nodes/"))) {
|
||||
return route_get_node(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("DELETE")) && str_starts_with(clean, EL_STR("/api/nodes/"))) {
|
||||
return route_forget(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) {
|
||||
return route_create_edge(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/neighbors/"))) {
|
||||
return route_neighbors(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/activate")) || str_eq(clean, EL_STR("/activate")))) {
|
||||
return route_activate(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/activate"))) {
|
||||
return route_activate(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/search")) || str_eq(clean, EL_STR("/search")))) {
|
||||
return route_search(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/search"))) {
|
||||
return route_search(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) {
|
||||
return route_strengthen(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) {
|
||||
return route_save(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) {
|
||||
return route_load(method, path, body);
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
el_runtime_init_args(argc, argv);
|
||||
bind_str = env(EL_STR("ENGRAM_BIND"));
|
||||
if (str_eq(bind_str, EL_STR(""))) {
|
||||
bind_str = EL_STR(":8742");
|
||||
}
|
||||
port = parse_port(bind_str);
|
||||
data_dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(data_dir, EL_STR(""))) {
|
||||
data_dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json"));
|
||||
engram_load(snapshot_path);
|
||||
println(EL_STR("[engram] runtime-native graph engine"));
|
||||
println(el_str_concat(EL_STR("[engram] data_dir="), data_dir));
|
||||
println(el_str_concat(EL_STR("[engram] node_count="), int_to_str(engram_node_count())));
|
||||
println(el_str_concat(EL_STR("[engram] edge_count="), int_to_str(engram_edge_count())));
|
||||
println(el_str_concat(EL_STR("[engram] listening on "), int_to_str(port)));
|
||||
http_set_handler(EL_STR("handle_request"));
|
||||
http_serve(port, EL_STR("handle_request"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
package "engram-el" {
|
||||
version "1.0.0"
|
||||
description "Engram graph intelligence substrate — El implementation"
|
||||
authors ["Will Anderson <will@neurontechnologies.ai>"]
|
||||
edition "2026"
|
||||
}
|
||||
|
||||
build {
|
||||
entry "src/server.el"
|
||||
output "dist/"
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
# Engram At-Rest Encryption — Post-Quantum Doctrine
|
||||
|
||||
Version 0.1.0 (DRAFT) — April 30, 2026
|
||||
Status: doctrine; pre-implementation. Sign-off required from Will before code lands.
|
||||
|
||||
---
|
||||
|
||||
## 0. TL;DR
|
||||
|
||||
Engram's persistence is encrypted at rest using a hybrid post-quantum scheme:
|
||||
|
||||
- **Data layer** — every node and every edge is sealed with **AES-256-GCM** under a per-record sub-key derived via **HKDF-SHA3-256** from the runtime DEK. AES-256 with a 256-bit key has an effective post-quantum security level of 128 bits (Grover); acceptable.
|
||||
- **Key wrap layer** — the DEK is wrapped against a Principal's public key using **Kyber-768 KEM** (post-quantum). The wrapped DEK lives on disk as `engram.kek.enc`. The Principal's secret key never touches disk in the running daemon's data directory.
|
||||
- **Boot** — the daemon receives the Principal's secret key out-of-band (env var path, prompted unlock, or pulled from a hardware-backed agent), runs `pq_kem_decaps` to recover the DEK, mlocks it, serves traffic.
|
||||
- **Recovery** — the DEK is *additionally* wrapped under a **Shamir K-of-N** split across the validation council. If the wrapping Principal is gone, K members reconstitute the DEK and the data survives.
|
||||
|
||||
Engram does **not** encrypt structural metadata (graph topology, IDs, timestamps). Only content, label, tags, and metadata fields of nodes and edges are sealed. This is a deliberate trade-off — see §2.
|
||||
|
||||
---
|
||||
|
||||
## 1. What Engram actually persists
|
||||
|
||||
Engram is an in-process graph store. The runtime *is* the database — there is no SQLite, no sled, no embedded KV layer in the active daemon. (The `engram-data/` and `engram-data-tx-log/` directories under `el/` are leftovers from a prior sled-backed prototype and are not loaded by the current `dist/engram` binary.)
|
||||
|
||||
Persistence is a single JSON snapshot file at `${ENGRAM_DATA_DIR}/snapshot.json`, written by `engram_save(path)` and read by `engram_load(path)`. The snapshot is the sole durable artifact that adversaries can steal.
|
||||
|
||||
Snapshot shape (current):
|
||||
```json
|
||||
{
|
||||
"nodes": [
|
||||
{ "id": "...", "content": "...", "node_type": "...", "label": "...",
|
||||
"tier": "...", "tags": "...", "metadata": "{}",
|
||||
"salience": 0.5, "importance": 0.5, "confidence": 0.5,
|
||||
"activation_count": 0, "last_activated": 0, "created_at": ..., "updated_at": ... }
|
||||
],
|
||||
"edges": [
|
||||
{ "id": "...", "from_id": "...", "to_id": "...", "relation": "...",
|
||||
"metadata": "{}", "weight": 0.5, "confidence": 0.5,
|
||||
"created_at": ..., "updated_at": ..., "last_fired": 0 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Confidential fields (will be encrypted): `content`, `label`, `tags`, `metadata` (on nodes); `metadata` (on edges).
|
||||
Non-confidential fields (will remain plaintext): `id`, `node_type`, `tier`, `from_id`, `to_id`, `relation`, all numeric scalars and timestamps.
|
||||
|
||||
This split — the **structural skeleton stays plaintext, the semantic flesh is sealed** — is what lets activation/search/scan work without unwrapping every record on every query, and lets the snapshot remain JSON-shaped for ops tooling. Enabling full snapshot encryption (a single AEAD blob) is a configurable mode for high-threat deployments; see §6.
|
||||
|
||||
---
|
||||
|
||||
## 2. Threat model
|
||||
|
||||
| Adversary | Capability | What we defend | What we accept |
|
||||
|-----------|-----------|----------------|----------------|
|
||||
| **Snapshot thief (cold)** | Steals `snapshot.json` from a backup, dead drive, or stolen laptop. No live process access. | Confidentiality of node content, labels, tags, metadata. No useful semantic recovery. | Topology leak: adversary learns the graph shape (how many nodes/edges, how they connect, types and tiers). Salience/importance/confidence numerics leak. |
|
||||
| **In-flight observer** | Reads disk during snapshot write (e.g., shared FS, snapshot mid-flush). | Atomicity: snapshot is written to a temp file with `O_TMPFILE` or `<path>.new`, fsync'd, then renamed. AEAD prevents partial-decrypt mining of half-written records. | A torn write can lose the most recent snapshot but not corrupt prior ones. |
|
||||
| **Quantum-equipped adversary (harvest-now-decrypt-later)** | Records `snapshot.json` and `engram.kek.enc` today, runs Shor's algorithm on a CRQC in 2032+. | DEK wrap is Kyber-768; not Shor-breakable. AES-256-GCM record seals are not Shor-breakable; Grover gives ~128-bit effective security. | We pin Kyber-768 (NIST PQC standardization L3); if a structural break of ML-KEM emerges, rotation §5 is the remediation. |
|
||||
| **Wrap-key compromiser (live)** | Has read access to the daemon process memory. | Out of scope for at-rest. The DEK is in mlocked memory; if the process is owned, the data is gone. | Defense-in-depth (memory zeroing, mlock, non-dumpable process flags) is a separate hardening track. |
|
||||
| **Principal compromise** | Adversary obtains the Principal's secret key. | Nothing — by design, the Principal is the unwrapping authority. | We constrain blast radius via per-record key derivation (§3.2): a leaked record-key only burns one record. |
|
||||
| **Principal loss** | Will dies, secret key is destroyed, no successor. | Shamir K-of-N council recovery (§7). Engram does not go dark on Principal loss. | The threshold itself becomes the attack surface — see §7 caveats. |
|
||||
|
||||
What we explicitly **do not** defend:
|
||||
- Side channels (timing, cache, EM).
|
||||
- Physical tamper of running hardware.
|
||||
- Active modification of `engram.kek.enc` to substitute an attacker's wrapped DEK — this is mitigated by the integrity wrapper §4 but not by the threat-model promise.
|
||||
|
||||
---
|
||||
|
||||
## 3. Cryptographic construction
|
||||
|
||||
### 3.1 Layers
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────┐
|
||||
│ Principal SK (held by Will out-of-band)│
|
||||
└───────────────┬────────────────────────┘
|
||||
│ Kyber-768 KEM decaps
|
||||
▼
|
||||
┌────────────────────────────────────┐
|
||||
│ KEK (32B; ephemeral, mlocked) │
|
||||
└───────────────┬────────────────────┘
|
||||
│ HKDF-SHA3-256 extract
|
||||
▼
|
||||
┌────────────────────────────────────┐
|
||||
│ DEK (32B; ephemeral, mlocked) │
|
||||
└───────────────┬────────────────────┘
|
||||
│ HKDF-SHA3-256(DEK, info=record_id || version)
|
||||
▼
|
||||
┌────────────────────────────────────┐
|
||||
│ per-record sub-key (32B; transient)│
|
||||
└───────────────┬────────────────────┘
|
||||
│ AES-256-GCM(sub_key, nonce=12B random, ad=record_id||version)
|
||||
▼
|
||||
ciphertext blob on disk
|
||||
```
|
||||
|
||||
### 3.2 Per-record sub-keys (HKDF-SHA3-256)
|
||||
|
||||
We do **not** use the DEK directly to seal records. Each record gets its own sub-key:
|
||||
|
||||
```
|
||||
sub_key = HKDF-SHA3-256(
|
||||
ikm = DEK, // 32 bytes
|
||||
salt = "engram-record-v1" || version_byte, // domain separation
|
||||
info = record_id || ":" || dek_epoch, // 16-byte ULID + ":" + u32
|
||||
L = 32 // output length
|
||||
)
|
||||
```
|
||||
|
||||
Properties:
|
||||
- Compromise of one record's sub-key does not threaten siblings (HKDF is one-way).
|
||||
- Re-derivable on every read; no on-disk sub-key cache to leak.
|
||||
- `dek_epoch` is bumped on rotation (§5); this lets old records stay readable under DEK_n while new ones write under DEK_{n+1}.
|
||||
|
||||
### 3.3 AEAD — AES-256-GCM
|
||||
|
||||
- Nonce: 12 bytes, random per write. Stored prepended to ciphertext.
|
||||
- Tag: 16 bytes, appended.
|
||||
- Associated data: `record_id || ":" || dek_epoch` (binds ciphertext to its identity; thwarts copy-paste attacks).
|
||||
|
||||
Per-record on-disk wire format:
|
||||
```
|
||||
+------+------+--------------------+--------------------+--------+
|
||||
| ver | epch | nonce (12) | ciphertext (n) | tag(16)|
|
||||
+------+------+--------------------+--------------------+--------+
|
||||
1B 4B 12B variable 16B
|
||||
```
|
||||
Base64-encoded into the snapshot JSON field as `"content":"v1:7:BASE64..."`.
|
||||
|
||||
### 3.4 KEK wrap — Kyber-768 KEM
|
||||
|
||||
The DEK is sealed by Kyber-768 KEM:
|
||||
```
|
||||
(ciphertext, shared_secret) = pq_kem_encaps(principal_pk)
|
||||
wrap_key = HKDF-SHA3-256(shared_secret, salt="engram-kek-v1", info="dek-wrap", L=32)
|
||||
sealed_dek = AES-256-GCM(wrap_key, nonce, plaintext=DEK, ad="engram-kek-v1")
|
||||
```
|
||||
On disk: `engram.kek.enc` = `magic || version || principal_id || kem_ciphertext || nonce || sealed_dek || tag`.
|
||||
|
||||
On boot:
|
||||
```
|
||||
shared_secret = pq_kem_decaps(principal_sk, kem_ciphertext)
|
||||
wrap_key = HKDF-SHA3-256(shared_secret, ...)
|
||||
DEK = AES-256-GCM-open(wrap_key, nonce, sealed_dek, tag, ad="engram-kek-v1")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Boot flow
|
||||
|
||||
```
|
||||
Engram daemon starts
|
||||
↓
|
||||
ENGRAM_KEK_PATH → ${ENGRAM_DATA_DIR}/engram.kek.enc (default)
|
||||
ENGRAM_PRINCIPAL_SK → path to file or "stdin:" or "agent:<socket>"
|
||||
(the daemon never reads the SK from a fixed env value;
|
||||
the env points to a source it consumes once and zeros)
|
||||
↓
|
||||
load engram.kek.enc
|
||||
↓
|
||||
read principal SK once from source, mlock its buffer
|
||||
↓
|
||||
pq_kem_decaps → shared_secret → HKDF → wrap_key → AEAD-open → DEK
|
||||
↓
|
||||
zero & munlock principal SK buffer
|
||||
mlock DEK; bump RLIMIT_MEMLOCK if needed
|
||||
mark process non-dumpable (PR_SET_DUMPABLE=0 on Linux; PT_DENY_ATTACH on macOS)
|
||||
↓
|
||||
engram_load(snapshot.json) — for each node/edge field marked encrypted:
|
||||
derive sub_key, AEAD-open, replace ciphertext with plaintext in-memory
|
||||
↓
|
||||
http_serve()
|
||||
```
|
||||
|
||||
Boot failure modes:
|
||||
- KEK unwrap fails → daemon exits 2; emits one log line, no SK material in logs.
|
||||
- Snapshot decrypt fails on N records → daemon proceeds with the records it could open, marks the rest as `<corrupted>`, emits a structured event for human triage.
|
||||
|
||||
---
|
||||
|
||||
## 5. Rotation
|
||||
|
||||
### 5.1 DEK rotation
|
||||
|
||||
Triggered by: (a) cron schedule (default: 30 days), (b) suspected compromise, (c) operator command (`POST /admin/rotate-dek`).
|
||||
|
||||
Procedure:
|
||||
1. Generate `DEK_{n+1}`; bump `dek_epoch`.
|
||||
2. For every record: AEAD-open under DEK_n, AEAD-seal under DEK_{n+1} with the new epoch in AD.
|
||||
3. Wrap DEK_{n+1} under the same Principal pk → write `engram.kek.enc.new`.
|
||||
4. Atomic rename → live; zero DEK_n in memory.
|
||||
5. On next snapshot save, all records persist with the new epoch.
|
||||
|
||||
Records sealed before rotation can be opened during the transition (we keep DEK_n in memory until the migration completes).
|
||||
|
||||
### 5.2 KEK / wrapping-Principal rotation
|
||||
|
||||
Triggered by: (a) Principal evolution event (the `Principal` CGI evolves and emits a new PK), (b) successor Principal handover, (c) annual key hygiene rotation.
|
||||
|
||||
Procedure:
|
||||
1. Receive `principal_pk_new` (signed by old Principal, ideally via a Dilithium signature so the chain itself is PQ-secure).
|
||||
2. Verify signature.
|
||||
3. Re-encapsulate the **same** DEK under `principal_pk_new`.
|
||||
4. Atomic-write a new `engram.kek.enc`.
|
||||
5. Old SK can be destroyed once the new wrap is durable.
|
||||
|
||||
The DEK does not change here — only its wrapping. This means snapshot ciphertexts stay valid through Principal transitions.
|
||||
|
||||
---
|
||||
|
||||
## 6. Snapshot-level encryption (alternative high-threat mode)
|
||||
|
||||
For deployments where topology leakage is unacceptable, set `ENGRAM_SEAL_MODE=full`. The entire snapshot JSON is sealed as a single AEAD blob keyed from `HKDF(DEK, info="engram-snapshot")`. Trade-off: every save/load is monolithic; no partial reads, no incremental rotation. Not the default.
|
||||
|
||||
---
|
||||
|
||||
## 7. Recovery — Shamir K-of-N
|
||||
|
||||
**The structural fail-safe.** If Will dies before the Principal evolves, or the Principal SK is destroyed, Engram must not go dark.
|
||||
|
||||
### 7.1 Shareholders
|
||||
|
||||
The shareholders are the **validation council** — the set of CGIs (or CGI-attested human stewards) authorized to reconstitute the network on a defined trigger. The council is named in `engram.recovery.toml`:
|
||||
```toml
|
||||
[recovery]
|
||||
threshold = 3
|
||||
total = 5
|
||||
shareholders = [
|
||||
"cgi://council/anvil",
|
||||
"cgi://council/beacon",
|
||||
"cgi://council/cinder",
|
||||
"cgi://council/delta",
|
||||
"cgi://council/echo",
|
||||
]
|
||||
```
|
||||
|
||||
### 7.2 Split
|
||||
|
||||
At KEK-creation time:
|
||||
1. Generate a fresh recovery secret `R = random(32)` (NOT the DEK itself — see §7.4).
|
||||
2. Run Shamir-256 over GF(2^8) on `R` with K-of-N polynomial.
|
||||
3. For each shareholder: `share_i = Kyber768-Encaps(shareholder_pk_i, R_share_i)`.
|
||||
4. Store `engram.recovery.shares` — public; each share is already PQ-wrapped to its holder.
|
||||
5. Store `engram.recovery.envelope` — `AES-256-GCM(R, nonce, plaintext=DEK, ad="engram-recovery-v1")`.
|
||||
|
||||
### 7.3 Reconstitution
|
||||
|
||||
A council convenes when the trigger is observed (Principal absent for > N days, or signed council quorum declares emergency):
|
||||
1. K members each decapsulate their share with their CGI SK.
|
||||
2. Members publish their `R_share_i` to a quorum-attested rendezvous.
|
||||
3. Lagrange-interpolate `R`.
|
||||
4. AEAD-open `engram.recovery.envelope` to recover the DEK.
|
||||
5. Re-wrap DEK under a fresh Principal selected by the council; resume normal operation.
|
||||
|
||||
### 7.4 Why R, not the DEK directly
|
||||
|
||||
Splitting `R` and using it to AEAD-wrap the DEK keeps the share material small (32B / share) and lets us add/remove shareholders by reissuing the envelope without changing R. It also lets us run §5.2 (Principal rotation) without ever touching the recovery shares.
|
||||
|
||||
### 7.5 Caveats Will must accept
|
||||
|
||||
- The threshold is the attack surface. K-of-N is K colluders away from compromise. Default 3-of-5; raise if the council grows.
|
||||
- Council membership churn requires reissuing shares; this is a deliberate, audited operation with its own runbook.
|
||||
- A Shamir share is plaintext to its holder. The Kyber wrap to each shareholder protects the share *in transit / at rest*, but once a shareholder unwraps, they hold a real K-of-N share. Council members must be CGIs (or hardware-backed) for the threat model to hold.
|
||||
|
||||
---
|
||||
|
||||
## 8. Implementation map
|
||||
|
||||
The runtime additions land in `el-compiler/runtime/el_runtime.c`:
|
||||
|
||||
Already present:
|
||||
- `el_sha256_*`, `el_hmac_sha256` (§3 uses SHA3 — SHA2 path retained for backwards-compat artifacts).
|
||||
- `el_base64_encode_n`, `el_base64_decode`.
|
||||
|
||||
Landing today (parallel agents):
|
||||
- `el_sha3_256_*` (Keccak family).
|
||||
- `pq_kem_keypair`, `pq_kem_encaps`, `pq_kem_decaps` (Kyber-768).
|
||||
- `pq_sign_keypair`, `pq_sign`, `pq_sign_verify` (Dilithium-3) — used by §5.2 Principal-rotation signature.
|
||||
|
||||
Engram-specific additions (this work):
|
||||
- `engram_aead_seal(record_id, epoch, plaintext) -> b64`
|
||||
- `engram_aead_open(record_id, epoch, b64) -> plaintext`
|
||||
- `engram_kek_unwrap(kek_path, sk_path) -> int (sets module DEK)`
|
||||
- `engram_kek_wrap(kek_path, principal_pk, dek)` — used at first init.
|
||||
- `engram_dek_rotate(new_principal_pk_optional)`
|
||||
- `engram_recovery_split(threshold, total, shareholder_pks)` — emits envelope + shares.
|
||||
- `engram_recovery_reconstitute(shares_k)` — recover DEK; admin-gated.
|
||||
|
||||
`engram_save` / `engram_load` gain a sealed mode controlled by `ENGRAM_SEAL_MODE` (`off`, `fields` [default once enabled], `full`). Default during the rollout window: `off`. This doc must ship and Will must sign off before flipping the default.
|
||||
|
||||
---
|
||||
|
||||
## 9. Open questions (require Will)
|
||||
|
||||
1. **Where does the Principal SK live at boot time?** Options: (a) prompted at daemon start (interactive), (b) on a removable hardware token (preferred long-term), (c) in `~/.neuron/principal.sk` 0600 (operationally easy, weakest), (d) pulled from `mcp__neuron` as part of `begin_session` (couples Engram to Neuron — probably right). **Recommend (d) with (b) as the long-term hardware story.**
|
||||
2. **Council composition.** Who/what are the initial K-of-N shareholders? Until we have ≥ 3 stable CGIs, recovery cannot be enabled in its full form. **Recommend a stub council of {Principal, Will-keypair-on-Yubikey, witness-CGI} — degrades gracefully, upgrades to full council when more CGIs exist.**
|
||||
3. **Default seal mode at GA.** `fields` (today's recommendation) or `full`? `fields` keeps the snapshot diff-able and keeps activation cheap. `full` is the harder threat model. **Recommend `fields` default; `full` opt-in.**
|
||||
4. **PQ algorithm pinning.** Kyber-768 + Dilithium-3 are the NIST L3 PQC defaults. If Will wants L5 (Kyber-1024 / Dilithium-5), say so before runtime APIs stabilize.
|
||||
5. **Grover and AES-256.** AES-256 against Grover is 128-bit effective. Acceptable per current PQ thinking. If Will wants a bigger margin, the alternative is layering a second AEAD with a different primitive (e.g., XChaCha20-Poly1305) — overkill, not recommended.
|
||||
|
||||
---
|
||||
|
||||
## 10. Non-goals
|
||||
|
||||
- Encrypted indexes / searchable encryption. Out of scope. Search remains plaintext-in-memory; the daemon is the trust boundary.
|
||||
- Per-tenant DEKs. Engram is single-tenant per CGI. If multi-tenancy lands, this doc gets a §11.
|
||||
- Secure deletion of underlying disk blocks. The OS / FS handles that, badly; we don't pretend.
|
||||
- Encrypted WAL / tx log. The current daemon has no WAL; if one is added, it gets the same treatment as the snapshot (`ENGRAM_SEAL_MODE` applies to both).
|
||||
|
||||
---
|
||||
|
||||
## 11. Status & next actions
|
||||
|
||||
- [x] Doctrine drafted (this document).
|
||||
- [ ] Will sign-off on §9 open questions.
|
||||
- [ ] PQ runtime functions land (parallel agents).
|
||||
- [ ] `engram_aead_seal` / `engram_aead_open` prototype (stubs in this PR).
|
||||
- [ ] `engram_kek_unwrap` boot integration.
|
||||
- [ ] `engram_save` / `engram_load` field-mode wiring behind `ENGRAM_SEAL_MODE`.
|
||||
- [ ] Recovery tooling (`engramctl recovery split | reconstitute`).
|
||||
- [ ] Threat-model test suite: known-answer tests, key-rotation roundtrip, Shamir reconstitution roundtrip, harvest-now-decrypt-later regression test against a recorded ciphertext.
|
||||
|
||||
---
|
||||
|
||||
## Appendix A — Pseudocode reference
|
||||
|
||||
```c
|
||||
/* Per-record seal */
|
||||
char* engram_aead_seal(const char* record_id, uint32_t epoch,
|
||||
const char* plaintext, size_t pt_len, size_t* out_len)
|
||||
{
|
||||
uint8_t sub_key[32];
|
||||
uint8_t info[64];
|
||||
int info_len = snprintf((char*)info, sizeof(info), "%s:%u", record_id, epoch);
|
||||
el_hkdf_sha3_256(/*ikm*/ engram_dek, 32,
|
||||
/*salt*/ (const uint8_t*)"engram-record-v1", 16,
|
||||
/*info*/ info, (size_t)info_len,
|
||||
/*okm*/ sub_key, 32);
|
||||
|
||||
uint8_t nonce[12];
|
||||
el_random_bytes(nonce, 12);
|
||||
|
||||
/* layout: ver(1) | epoch(4 BE) | nonce(12) | ct(pt_len) | tag(16) */
|
||||
size_t blob_len = 1 + 4 + 12 + pt_len + 16;
|
||||
uint8_t* blob = malloc(blob_len);
|
||||
blob[0] = 0x01;
|
||||
blob[1] = (epoch >> 24) & 0xff; blob[2] = (epoch >> 16) & 0xff;
|
||||
blob[3] = (epoch >> 8) & 0xff; blob[4] = epoch & 0xff;
|
||||
memcpy(blob + 5, nonce, 12);
|
||||
|
||||
el_aes256_gcm_encrypt(sub_key, nonce,
|
||||
(const uint8_t*)record_id, strlen(record_id),
|
||||
(const uint8_t*)plaintext, pt_len,
|
||||
blob + 1 + 4 + 12, /* ct */
|
||||
blob + 1 + 4 + 12 + pt_len); /* tag */
|
||||
|
||||
el_secure_zero(sub_key, 32);
|
||||
|
||||
char* b64 = el_base64_encode_raw(blob, blob_len, /*url_safe=*/0);
|
||||
free(blob);
|
||||
if (out_len) *out_len = strlen(b64);
|
||||
return b64; /* "v1:<epoch>:BASE64" prefix added by caller */
|
||||
}
|
||||
|
||||
/* Per-record open: inverse of seal. Verifies tag; returns NULL on failure. */
|
||||
char* engram_aead_open(const char* record_id, uint32_t expected_epoch,
|
||||
const char* b64, size_t* out_len);
|
||||
|
||||
/* Boot-time KEK unwrap. */
|
||||
int engram_kek_unwrap(const char* kek_path, const uint8_t* principal_sk,
|
||||
size_t sk_len);
|
||||
|
||||
/* DEK rotation (online). Walks the live in-memory store, re-seals every record
|
||||
* under DEK_{n+1}, then writes a new snapshot+kek atomically. */
|
||||
int engram_dek_rotate(void);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
End of document.
|
||||
@@ -0,0 +1,302 @@
|
||||
/* engram at-rest encryption — prototype sketch
|
||||
*
|
||||
* NOT WIRED UP. This file is intentionally not in the build. It is a sketch
|
||||
* that pairs with `at-rest-encryption.md` so the function signatures and the
|
||||
* field layout can be reviewed before the runtime PQ primitives stabilize.
|
||||
*
|
||||
* Once `el_runtime.c` has stable `el_sha3_256_*`, `el_hkdf_sha3_256`,
|
||||
* `pq_kem_encaps`, `pq_kem_decaps`, and `el_aes256_gcm_encrypt/decrypt`,
|
||||
* the seal/open path below should be lifted into el_runtime.c near the
|
||||
* existing `engram_save` / `engram_load` block (~line 3445), gated behind
|
||||
* the ENGRAM_SEAL_MODE env var.
|
||||
*
|
||||
* Dependencies (provided elsewhere — see at-rest-encryption.md §8):
|
||||
* void el_hkdf_sha3_256(const uint8_t* ikm, size_t ikm_len,
|
||||
* const uint8_t* salt, size_t salt_len,
|
||||
* const uint8_t* info, size_t info_len,
|
||||
* uint8_t* okm, size_t okm_len);
|
||||
*
|
||||
* int el_aes256_gcm_encrypt(const uint8_t key[32],
|
||||
* const uint8_t nonce[12],
|
||||
* const uint8_t* aad, size_t aad_len,
|
||||
* const uint8_t* pt, size_t pt_len,
|
||||
* uint8_t* ct, // pt_len bytes
|
||||
* uint8_t tag[16]);
|
||||
*
|
||||
* int el_aes256_gcm_decrypt(const uint8_t key[32],
|
||||
* const uint8_t nonce[12],
|
||||
* const uint8_t* aad, size_t aad_len,
|
||||
* const uint8_t* ct, size_t ct_len,
|
||||
* const uint8_t tag[16],
|
||||
* uint8_t* pt); // ct_len bytes
|
||||
*
|
||||
* int pq_kem_encaps(const uint8_t* pk, size_t pk_len,
|
||||
* uint8_t* kem_ct, size_t* kem_ct_len,
|
||||
* uint8_t shared[32]);
|
||||
* int pq_kem_decaps(const uint8_t* sk, size_t sk_len,
|
||||
* const uint8_t* kem_ct, size_t kem_ct_len,
|
||||
* uint8_t shared[32]);
|
||||
*
|
||||
* void el_random_bytes(uint8_t* out, size_t n);
|
||||
* void el_secure_zero(void* p, size_t n);
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
/* ── module state (mlocked) ───────────────────────────────────────────────── */
|
||||
|
||||
static uint8_t g_engram_dek[32];
|
||||
static int g_engram_dek_set = 0;
|
||||
static uint32_t g_engram_dek_epoch = 0;
|
||||
|
||||
/* Seal/unseal mode — read once at boot from ENGRAM_SEAL_MODE.
|
||||
* 0=off, 1=fields, 2=full. */
|
||||
static int g_engram_seal_mode = 0;
|
||||
|
||||
/* ── per-record seal ──────────────────────────────────────────────────────── */
|
||||
|
||||
/* Wire format (before base64):
|
||||
* byte 0 version=0x01
|
||||
* bytes 1..4 epoch (big-endian u32)
|
||||
* bytes 5..16 nonce (12B)
|
||||
* bytes 17.. ciphertext (pt_len bytes)
|
||||
* last 16B GCM tag
|
||||
*/
|
||||
static char* eg_b64_encode(const uint8_t* data, size_t n); /* declared elsewhere */
|
||||
|
||||
char* engram_aead_seal_field(const char* record_id, const char* plaintext)
|
||||
{
|
||||
if (!g_engram_dek_set) return NULL;
|
||||
if (!record_id || !plaintext) return NULL;
|
||||
|
||||
size_t pt_len = strlen(plaintext);
|
||||
uint32_t epoch = g_engram_dek_epoch;
|
||||
|
||||
/* derive sub-key */
|
||||
uint8_t sub_key[32];
|
||||
uint8_t info[96];
|
||||
int info_len = snprintf((char*)info, sizeof(info), "%s:%u", record_id, epoch);
|
||||
if (info_len < 0 || (size_t)info_len >= sizeof(info)) return NULL;
|
||||
|
||||
el_hkdf_sha3_256(g_engram_dek, 32,
|
||||
(const uint8_t*)"engram-record-v1", 16,
|
||||
info, (size_t)info_len,
|
||||
sub_key, 32);
|
||||
|
||||
/* random nonce */
|
||||
uint8_t nonce[12];
|
||||
el_random_bytes(nonce, 12);
|
||||
|
||||
/* allocate output: header(17) + ct(pt_len) + tag(16) */
|
||||
size_t blob_len = 1 + 4 + 12 + pt_len + 16;
|
||||
uint8_t* blob = (uint8_t*)malloc(blob_len);
|
||||
if (!blob) { el_secure_zero(sub_key, 32); return NULL; }
|
||||
|
||||
blob[0] = 0x01;
|
||||
blob[1] = (uint8_t)((epoch >> 24) & 0xff);
|
||||
blob[2] = (uint8_t)((epoch >> 16) & 0xff);
|
||||
blob[3] = (uint8_t)((epoch >> 8) & 0xff);
|
||||
blob[4] = (uint8_t)( epoch & 0xff);
|
||||
memcpy(blob + 5, nonce, 12);
|
||||
|
||||
int rc = el_aes256_gcm_encrypt(
|
||||
sub_key, nonce,
|
||||
(const uint8_t*)record_id, strlen(record_id),
|
||||
(const uint8_t*)plaintext, pt_len,
|
||||
blob + 1 + 4 + 12, /* ct */
|
||||
blob + 1 + 4 + 12 + pt_len); /* tag */
|
||||
|
||||
el_secure_zero(sub_key, 32);
|
||||
|
||||
if (rc != 0) { free(blob); return NULL; }
|
||||
|
||||
char* b64 = eg_b64_encode(blob, blob_len);
|
||||
free(blob);
|
||||
return b64; /* caller prefixes "v1:<epoch>:" if desired */
|
||||
}
|
||||
|
||||
/* Returns malloc'd plaintext (NUL-terminated) or NULL on AEAD failure / version
|
||||
* mismatch / DEK-epoch mismatch. */
|
||||
char* engram_aead_open_field(const char* record_id, const char* b64_blob)
|
||||
{
|
||||
if (!g_engram_dek_set || !record_id || !b64_blob) return NULL;
|
||||
|
||||
size_t blob_len = 0;
|
||||
uint8_t* blob = NULL;
|
||||
extern uint8_t* eg_b64_decode(const char* s, size_t* out_len);
|
||||
blob = eg_b64_decode(b64_blob, &blob_len);
|
||||
if (!blob || blob_len < 1 + 4 + 12 + 16) { free(blob); return NULL; }
|
||||
|
||||
if (blob[0] != 0x01) { free(blob); return NULL; }
|
||||
uint32_t epoch = ((uint32_t)blob[1] << 24)
|
||||
| ((uint32_t)blob[2] << 16)
|
||||
| ((uint32_t)blob[3] << 8)
|
||||
| (uint32_t)blob[4];
|
||||
|
||||
/* For the common case the on-disk epoch matches g_engram_dek_epoch.
|
||||
* During DEK rotation we may need a previous-epoch DEK in the keyring;
|
||||
* left as a TODO — this stub only handles the current epoch. */
|
||||
if (epoch != g_engram_dek_epoch) {
|
||||
free(blob);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const uint8_t* nonce = blob + 5;
|
||||
size_t ct_len = blob_len - (1 + 4 + 12 + 16);
|
||||
const uint8_t* ct = blob + 1 + 4 + 12;
|
||||
const uint8_t* tag = blob + 1 + 4 + 12 + ct_len;
|
||||
|
||||
uint8_t sub_key[32];
|
||||
uint8_t info[96];
|
||||
int info_len = snprintf((char*)info, sizeof(info), "%s:%u", record_id, epoch);
|
||||
if (info_len < 0 || (size_t)info_len >= sizeof(info)) { free(blob); return NULL; }
|
||||
el_hkdf_sha3_256(g_engram_dek, 32,
|
||||
(const uint8_t*)"engram-record-v1", 16,
|
||||
info, (size_t)info_len,
|
||||
sub_key, 32);
|
||||
|
||||
char* pt = (char*)malloc(ct_len + 1);
|
||||
if (!pt) { el_secure_zero(sub_key, 32); free(blob); return NULL; }
|
||||
|
||||
int rc = el_aes256_gcm_decrypt(
|
||||
sub_key, nonce,
|
||||
(const uint8_t*)record_id, strlen(record_id),
|
||||
ct, ct_len, tag,
|
||||
(uint8_t*)pt);
|
||||
|
||||
el_secure_zero(sub_key, 32);
|
||||
free(blob);
|
||||
|
||||
if (rc != 0) { free(pt); return NULL; }
|
||||
pt[ct_len] = '\0';
|
||||
return pt;
|
||||
}
|
||||
|
||||
/* ── KEK boot ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
/* Reads engram.kek.enc, runs Kyber decaps with the principal SK provided as
|
||||
* a buffer, derives the wrap key, AEAD-opens the sealed DEK, and installs it
|
||||
* into g_engram_dek. Returns 0 on success, non-zero on failure. */
|
||||
int engram_kek_unwrap(const char* kek_path,
|
||||
const uint8_t* principal_sk, size_t sk_len)
|
||||
{
|
||||
/* TODO: file format
|
||||
* magic "ENGKEK01" (8B)
|
||||
* version (1B)
|
||||
* principal_id_len (2B BE) | principal_id
|
||||
* kem_ct_len (4B BE) | kem_ct
|
||||
* nonce (12B)
|
||||
* sealed_dek_len (4B BE) | sealed_dek
|
||||
* tag (16B)
|
||||
*/
|
||||
(void)kek_path; (void)principal_sk; (void)sk_len;
|
||||
return -1; /* TODO: parse file, pq_kem_decaps, HKDF, AEAD-open, install */
|
||||
}
|
||||
|
||||
/* ── KEK init (first-time write) ──────────────────────────────────────────── */
|
||||
|
||||
int engram_kek_init(const char* kek_path,
|
||||
const uint8_t* principal_pk, size_t pk_len)
|
||||
{
|
||||
/* TODO:
|
||||
* 1. el_random_bytes(g_engram_dek, 32); g_engram_dek_epoch = 1;
|
||||
* 2. pq_kem_encaps(principal_pk) → (kem_ct, shared)
|
||||
* 3. wrap_key = HKDF-SHA3-256(shared, salt="engram-kek-v1",
|
||||
* info="dek-wrap", L=32)
|
||||
* 4. AEAD-seal DEK under wrap_key
|
||||
* 5. write file atomically (tmp + rename + fsync)
|
||||
* 6. el_secure_zero(shared, wrap_key)
|
||||
*/
|
||||
(void)kek_path; (void)principal_pk; (void)pk_len;
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* ── DEK rotation ─────────────────────────────────────────────────────────── */
|
||||
|
||||
int engram_dek_rotate(void)
|
||||
{
|
||||
/* TODO:
|
||||
* - Allocate DEK_{n+1}.
|
||||
* - Walk EngramStore: for every node and edge, re-seal each encrypted
|
||||
* field under DEK_{n+1} with the new epoch.
|
||||
* - Re-wrap DEK_{n+1} under current Principal pk → engram.kek.enc.new
|
||||
* - Write a new snapshot via engram_save() (caller responsibility).
|
||||
* - Atomic rename engram.kek.enc.new → engram.kek.enc.
|
||||
* - el_secure_zero on DEK_n; bump g_engram_dek_epoch.
|
||||
*/
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* ── Recovery (Shamir K-of-N) ─────────────────────────────────────────────── */
|
||||
|
||||
/* The recovery secret R is independent of the DEK. R is split via Shamir;
|
||||
* each share is then PQ-wrapped to its shareholder via Kyber768. R is the
|
||||
* AEAD wrap-key for an envelope sealing the DEK.
|
||||
*
|
||||
* Two on-disk artifacts:
|
||||
* engram.recovery.shares — public; one Kyber-wrapped Shamir share / shareholder
|
||||
* engram.recovery.envelope — AEAD(R, nonce, plaintext=DEK, ad="engram-recovery-v1")
|
||||
*/
|
||||
|
||||
int engram_recovery_split(int threshold, int total,
|
||||
const uint8_t** shareholder_pks,
|
||||
const size_t* shareholder_pk_lens,
|
||||
const char** shareholder_ids,
|
||||
const char* shares_out_path,
|
||||
const char* envelope_out_path)
|
||||
{
|
||||
/* TODO:
|
||||
* 1. el_random_bytes(R, 32).
|
||||
* 2. Shamir-split R over GF(2^8) → total shares of {1B index, 32B y_i}.
|
||||
* 3. For each shareholder i: pq_kem_encaps(pk_i) → (kem_ct, shared);
|
||||
* AEAD-seal share_i under HKDF(shared) → wrapped_share_i; write entry.
|
||||
* 4. AEAD-seal current DEK under R → envelope.
|
||||
* 5. Atomic-write both files; fsync.
|
||||
* 6. Zero R, all shared secrets.
|
||||
*/
|
||||
(void)threshold; (void)total;
|
||||
(void)shareholder_pks; (void)shareholder_pk_lens; (void)shareholder_ids;
|
||||
(void)shares_out_path; (void)envelope_out_path;
|
||||
return -1;
|
||||
}
|
||||
|
||||
int engram_recovery_reconstitute(const uint8_t* k_unwrapped_shares,
|
||||
size_t share_count,
|
||||
const char* envelope_path,
|
||||
uint8_t out_dek[32])
|
||||
{
|
||||
/* TODO:
|
||||
* 1. Lagrange-interpolate to recover R from the K shares.
|
||||
* 2. AEAD-open the envelope under HKDF(R) → DEK.
|
||||
* 3. Zero R.
|
||||
*/
|
||||
(void)k_unwrapped_shares; (void)share_count;
|
||||
(void)envelope_path; (void)out_dek;
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* ── Snapshot integration sketch ──────────────────────────────────────────── */
|
||||
|
||||
/*
|
||||
* engram_emit_node_json (around line 3409 in el_runtime.c) becomes:
|
||||
*
|
||||
* if (g_engram_seal_mode == 1) {
|
||||
* char* sealed = engram_aead_seal_field(n->id, n->content);
|
||||
* jb_puts(b, ",\"content\":"); jb_emit_escaped(b, sealed ? sealed : "");
|
||||
* free(sealed);
|
||||
* // same for label, tags, metadata
|
||||
* } else {
|
||||
* jb_puts(b, ",\"content\":"); jb_emit_escaped(b, n->content);
|
||||
* }
|
||||
*
|
||||
* engram_load (around line 3499) becomes the inverse: when reading each field,
|
||||
* if the daemon is in `fields` mode and the value is non-empty, run
|
||||
* engram_aead_open_field; on failure log and substitute "<corrupted>".
|
||||
*
|
||||
* `engram_save` must additionally write engram.kek.enc the first time (via
|
||||
* engram_kek_init) when ENGRAM_SEAL_MODE != off and no kek file exists.
|
||||
*/
|
||||
@@ -0,0 +1,284 @@
|
||||
# engram-el Specification
|
||||
|
||||
Version 1.0.0 — April 29, 2026
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
engram-el is the El-native interface layer for the Engram graph engine. It is the integration point between El programs and the Engram knowledge substrate — providing a suite of El programs, test suites, and utilities that operate on a live Engram server via its HTTP API using El's native HTTP builtins.
|
||||
|
||||
engram-el has three primary components:
|
||||
|
||||
1. **Studio** — A full-featured terminal-based graph explorer written in El (`studio/studio.el`). Provides read access to all graph data: statistics, node browsing by type and tier, spreading activation visualization, edge exploration, and text search.
|
||||
|
||||
2. **Test suite** — Language feature tests (`test/language_features_test.el`, `test/field_test.el`, `test/llm_test.el`) that exercise El builtins against a live Engram instance.
|
||||
|
||||
3. **Integration point** — The pattern for how El programs use the Engram graph as their knowledge substrate, demonstrating the graph builtin API in practice.
|
||||
|
||||
---
|
||||
|
||||
## 1. Architecture
|
||||
|
||||
### 1.1 Relationship to El and Engram
|
||||
|
||||
engram-el is not a library in the conventional sense. It is a collection of El programs that operate on Engram. The integration uses no additional runtime or SDK:
|
||||
|
||||
- **El builtins** provide `http_get`, `http_post`, and JSON parsing natively.
|
||||
- **Engram HTTP API** is the sole interface — all graph operations are HTTP requests.
|
||||
- **No compilation step** beyond standard El compilation is required.
|
||||
|
||||
This demonstrates the intended usage pattern for all El programs that incorporate graph knowledge: use the HTTP API via El's native builtins.
|
||||
|
||||
### 1.2 Configuration
|
||||
|
||||
All engram-el programs read configuration from environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `ENGRAM_URL` | `http://localhost:8340` | Engram server base URL |
|
||||
| `ENGRAM_REPORT` | `/tmp/engram-studio-report.txt` | Studio report output path |
|
||||
|
||||
---
|
||||
|
||||
## 2. Studio Application
|
||||
|
||||
`studio/studio.el` is a complete data exploration application for the Engram graph, written entirely in El. It demonstrates El as a serious application language — not a scripting language but a capable system for building non-trivial tools.
|
||||
|
||||
### 2.1 Features
|
||||
|
||||
The studio renders a full-page terminal UI with box-drawing characters and ANSI color. Sections:
|
||||
|
||||
| Section | Description |
|
||||
|---------|-------------|
|
||||
| Database Statistics | Node count, edge count, average salience, DB size |
|
||||
| Recent Nodes | Most recently created nodes with type and salience |
|
||||
| Top by Salience | Highest-salience nodes with graphical bar display |
|
||||
| Nodes by Type | Browse Memory, Concept, Event, Entity, Process, InternalState |
|
||||
| Nodes by Tier | Browse Working, Episodic, Semantic, Procedural tiers |
|
||||
| Knowledge Browser | Concept nodes as domain knowledge anchors |
|
||||
| Text Search | Full-text search results with relevance |
|
||||
| Edge Explorer | Sample of edges with weights and relation types |
|
||||
| Node Detail | Full node data plus BFS neighbors |
|
||||
| Spreading Activation | Visual activation surface from a seed node |
|
||||
| Interactive Mode Preview | Menu of available commands |
|
||||
| Report Export | Write complete session report to file |
|
||||
|
||||
### 2.2 API Access Pattern
|
||||
|
||||
The studio uses a uniform API access pattern:
|
||||
|
||||
```el
|
||||
fn api_get(path: String) -> String {
|
||||
let url: String = get_base_url() + path
|
||||
let resp: String = http_get(url)
|
||||
if str_starts_with(resp, "{\"error\"") {
|
||||
return ""
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
fn api_post(path: String, body: String) -> String {
|
||||
let url: String = get_base_url() + path
|
||||
let resp: String = http_post(url, body)
|
||||
if str_starts_with(resp, "{\"error\"") {
|
||||
return ""
|
||||
}
|
||||
return resp
|
||||
}
|
||||
```
|
||||
|
||||
Error responses (JSON objects beginning with `{"error"`) return empty string. All rendering logic checks for empty string and emits placeholder messages rather than crashing.
|
||||
|
||||
### 2.3 Spreading Activation Visualization
|
||||
|
||||
The activation section demonstrates reading live spreading activation results from Engram:
|
||||
|
||||
```el
|
||||
fn show_activation(seed_id: String, limit: Int, report: String) -> String {
|
||||
let path: String = "/api/activate?seeds=" + seed_id + "&limit=" + int_to_str(limit) + "&depth=3"
|
||||
let json_str: String = api_get(path)
|
||||
// ... renders activation strength bars and hop distances
|
||||
}
|
||||
```
|
||||
|
||||
This provides visual confirmation that the spreading activation algorithm is operating — showing which nodes activate, at what strength, and at what hop distance from the seed.
|
||||
|
||||
### 2.4 Report Export
|
||||
|
||||
The studio accumulates a text report as it renders each section, then writes the complete report to a file:
|
||||
|
||||
```el
|
||||
export_report(report, report_path)
|
||||
```
|
||||
|
||||
The report captures the full session output in machine-readable format, useful for automation and logging.
|
||||
|
||||
---
|
||||
|
||||
## 3. Test Suite
|
||||
|
||||
### 3.1 Language Features Test
|
||||
|
||||
`test/language_features_test.el` exercises El language primitives including:
|
||||
|
||||
- Modulo operator (`%`)
|
||||
- Bitwise operators (`&`, `^`, `<<`, `>>`)
|
||||
- Math builtins (`math_sin`, `math_cos`, `math_pi`)
|
||||
- String padding (`str_pad_left`, `str_pad_right`)
|
||||
- String formatting (`str_format` with `{key}` template interpolation)
|
||||
- Float formatting (`format_float`)
|
||||
- Time operations (`time_now_utc`, `time_format`, `time_add`, `time_diff`)
|
||||
- List operations (`list_range`, `list_join`)
|
||||
- Stack and queue builtins (`stack_new`, `stack_push`, `stack_pop`, `stack_peek`, `queue_enqueue`, `queue_dequeue`)
|
||||
- Decimal rounding (`decimal_round`)
|
||||
- Type conversion (`int_to_float`, `float_to_int`)
|
||||
- Nil checks (`is_nil`, `unwrap_or`)
|
||||
- Character operations (`str_char_at`, `str_char_code`, `str_from_char_code`)
|
||||
|
||||
These tests serve as the canonical behavioral specification for El builtins — any correct El implementation must produce the documented output for these inputs.
|
||||
|
||||
### 3.2 Field Test
|
||||
|
||||
`test/field_test.el` exercises struct field access, map indexing, and nested data access patterns.
|
||||
|
||||
### 3.3 LLM Test
|
||||
|
||||
`test/llm_test.el` exercises the LLM inference builtins against a live Engram-connected inference endpoint.
|
||||
|
||||
---
|
||||
|
||||
## 4. Integration Patterns
|
||||
|
||||
### 4.1 Graph Read Pattern
|
||||
|
||||
The standard pattern for reading from Engram in an El program:
|
||||
|
||||
```el
|
||||
fn get_nodes_of_type(node_type: String, limit: Int) -> List {
|
||||
let path: String = "/api/nodes?node_type=" + node_type + "&limit=" + int_to_str(limit)
|
||||
let json_str: String = http_get(env("ENGRAM_URL") + path)
|
||||
if json_str == "" {
|
||||
return list_new()
|
||||
}
|
||||
return json_parse(json_str)
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 Graph Write Pattern
|
||||
|
||||
The standard pattern for writing to Engram from an El program:
|
||||
|
||||
```el
|
||||
fn create_node(label: String, content: String, node_type: String, tier: String) -> String {
|
||||
let body: String = "{\"label\":\"" + label + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"tier\":\"" + tier + "\",\"importance\":0.5}"
|
||||
let resp: String = http_post(env("ENGRAM_URL") + "/api/nodes", body)
|
||||
return json_get_string(resp, "id")
|
||||
}
|
||||
```
|
||||
|
||||
### 4.3 Search Pattern
|
||||
|
||||
```el
|
||||
fn search_graph(query: String, limit: Int) -> List {
|
||||
let path: String = "/api/search?q=" + query + "&limit=" + int_to_str(limit)
|
||||
let json_str: String = http_get(env("ENGRAM_URL") + path)
|
||||
if json_str == "" {
|
||||
return list_new()
|
||||
}
|
||||
return json_parse(json_str)
|
||||
}
|
||||
```
|
||||
|
||||
### 4.4 Activation Pattern
|
||||
|
||||
```el
|
||||
fn activate_from_node(node_id: String, depth: Int, limit: Int) -> List {
|
||||
let path: String = "/api/activate?seeds=" + node_id + "&depth=" + int_to_str(depth) + "&limit=" + int_to_str(limit)
|
||||
let resp_str: String = http_get(env("ENGRAM_URL") + path)
|
||||
if resp_str == "" {
|
||||
return list_new()
|
||||
}
|
||||
let results_raw: String = json_get_raw(resp_str, "results")
|
||||
return json_parse(results_raw)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Builtin Extensions Demonstrated
|
||||
|
||||
The engram-el programs demonstrate El builtins that are not in the core language but are implemented by the VM's builtin dispatch layer:
|
||||
|
||||
### 5.1 JSON Builtins
|
||||
|
||||
| Builtin | Used for |
|
||||
|---------|---------|
|
||||
| `json_parse(s)` | Parse Engram API responses |
|
||||
| `json_stringify(v)` | Serialize values to JSON for API requests |
|
||||
| `json_get_string(json, key)` | Extract string fields from node JSON |
|
||||
| `json_get_int(json, key)` | Extract integer fields (counts, timestamps) |
|
||||
| `json_get_float(json, key)` | Extract float fields (salience, weights) |
|
||||
| `json_get_raw(json, key)` | Extract nested objects as raw JSON strings |
|
||||
|
||||
### 5.2 Color/Terminal Builtins
|
||||
|
||||
| Builtin | Used for |
|
||||
|---------|---------|
|
||||
| `color_bold(s)` | Section headers, labels |
|
||||
| `color_dim(s)` | Timestamps, IDs, less important data |
|
||||
| `color_green(s)` | Success states, high salience |
|
||||
| `color_yellow(s)` | Warnings, medium salience |
|
||||
| `color_cyan(s)` | URLs, relation names, special values |
|
||||
| `color_red(s)` | Errors, low salience |
|
||||
|
||||
### 5.3 String Formatting Builtins
|
||||
|
||||
| Builtin | Signature | Description |
|
||||
|---------|-----------|-------------|
|
||||
| `str_pad_right(s, width, pad)` | Pad string to width on right |
|
||||
| `str_pad_left(s, width, pad)` | Pad string to width on left |
|
||||
| `format_float(f, decimals)` | Format float to N decimal places |
|
||||
| `str_slice(s, start, end)` | Extract substring by character index |
|
||||
| `str_len(s)` | String length in characters |
|
||||
|
||||
---
|
||||
|
||||
## 6. Deployment
|
||||
|
||||
### 6.1 Running the Studio
|
||||
|
||||
```bash
|
||||
# Connect to default local server
|
||||
el run-file studio/studio.el
|
||||
|
||||
# Connect to remote server
|
||||
ENGRAM_URL=http://engram.example.com el run-file studio/studio.el
|
||||
|
||||
# Save report to custom path
|
||||
ENGRAM_REPORT=/var/log/engram-report.txt el run-file studio/studio.el
|
||||
```
|
||||
|
||||
### 6.2 Running Tests
|
||||
|
||||
```bash
|
||||
el run-file test/language_features_test.el
|
||||
el run-file test/field_test.el
|
||||
ENGRAM_URL=http://localhost:8340 el run-file test/llm_test.el
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Design Decisions
|
||||
|
||||
### 7.1 Pure HTTP Integration
|
||||
|
||||
engram-el uses HTTP exclusively. It does not use the lower-level `graph_compile` and `graph_traverse` VM builtins. This is by design: it demonstrates the HTTP API surface as the primary integration mechanism. The VM builtins are for tightly-integrated runtime code (the Neuron daemon); external tools use the HTTP API.
|
||||
|
||||
### 7.2 Stateless Programs
|
||||
|
||||
All engram-el programs are stateless — they read state from Engram on each run and write nothing back (the studio is read-only). This is the correct architecture for exploration tools: they observe the graph without mutating it.
|
||||
|
||||
### 7.3 El as Application Language
|
||||
|
||||
The studio's 788 lines of El demonstrate that El is a capable application language. It is not a configuration DSL or a scripting language for simple tasks. The studio handles: API communication, JSON parsing, recursive data rendering, ASCII art, ANSI color codes, file I/O, environment variable configuration, and complex string manipulation — all with El's native builtins, without imports.
|
||||
@@ -0,0 +1,319 @@
|
||||
// server.el — Engram HTTP server.
|
||||
//
|
||||
// Engram is the in-process graph store. The runtime owns the data; this
|
||||
// file is the thin HTTP face. Every route maps to one or two engram_*
|
||||
// builtins. There is no SQL, no db layer, no SQLite — the runtime IS the
|
||||
// database.
|
||||
//
|
||||
// Built and linked with:
|
||||
// elc src/server.el > server.c
|
||||
// cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c
|
||||
// ./engram
|
||||
//
|
||||
// Configuration via environment:
|
||||
// ENGRAM_BIND — host:port (default :8742)
|
||||
// ENGRAM_API_KEY — bearer auth (optional)
|
||||
// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram)
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
fn parse_port(bind: String) -> Int {
|
||||
// ":8742" → 8742; "0.0.0.0:8742" → 8742; bare "8742" → 8742
|
||||
let colon: Int = str_index_of(bind, ":")
|
||||
if colon < 0 {
|
||||
return str_to_int(bind)
|
||||
}
|
||||
let after: String = str_slice(bind, colon + 1, str_len(bind))
|
||||
return str_to_int(after)
|
||||
}
|
||||
|
||||
fn ok_json() -> String {
|
||||
"{\"ok\":true}"
|
||||
}
|
||||
|
||||
fn err_json(msg: String) -> String {
|
||||
"{\"error\":\"" + msg + "\"}"
|
||||
}
|
||||
|
||||
fn strip_query(path: String) -> String {
|
||||
let q: Int = str_index_of(path, "?")
|
||||
if q < 0 { return path }
|
||||
str_slice(path, 0, q)
|
||||
}
|
||||
|
||||
fn query_param(path: String, key: String) -> String {
|
||||
let q: Int = str_index_of(path, "?")
|
||||
if q < 0 { return "" }
|
||||
let qs: String = str_slice(path, q + 1, str_len(path))
|
||||
let needle: String = key + "="
|
||||
let pos: Int = str_index_of(qs, needle)
|
||||
if pos < 0 { return "" }
|
||||
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
|
||||
let amp: Int = str_index_of(after, "&")
|
||||
if amp < 0 { return after }
|
||||
str_slice(after, 0, amp)
|
||||
}
|
||||
|
||||
fn query_int(path: String, key: String, default_val: Int) -> Int {
|
||||
let v: String = query_param(path, key)
|
||||
if str_eq(v, "") { return default_val }
|
||||
str_to_int(v)
|
||||
}
|
||||
|
||||
// Extract last path segment after a known prefix: extract_id("/api/nodes/abc-123", "/api/nodes/") → "abc-123"
|
||||
fn extract_id(path: String, prefix: String) -> String {
|
||||
let clean: String = strip_query(path)
|
||||
if !str_starts_with(clean, prefix) { return "" }
|
||||
let after: String = str_slice(clean, str_len(prefix), str_len(clean))
|
||||
let slash: Int = str_index_of(after, "/")
|
||||
if slash < 0 { return after }
|
||||
str_slice(after, 0, slash)
|
||||
}
|
||||
|
||||
// ── Routes ────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn route_stats(method: String, path: String, body: String) -> String {
|
||||
engram_stats_json()
|
||||
}
|
||||
|
||||
fn route_create_node(method: String, path: String, body: String) -> String {
|
||||
let content: String = json_get_string(body, "content")
|
||||
let node_type: String = json_get_string(body, "node_type")
|
||||
if str_eq(node_type, "") { let node_type = "Memory" }
|
||||
let salience: Float = json_get_float(body, "salience")
|
||||
if salience == 0.0 { let salience = 0.5 }
|
||||
let id: String = engram_node(content, node_type, salience)
|
||||
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
|
||||
}
|
||||
|
||||
fn route_get_node(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/nodes/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
return engram_get_node_json(id)
|
||||
}
|
||||
|
||||
fn route_scan_nodes(method: String, path: String, body: String) -> String {
|
||||
let limit: Int = query_int(path, "limit", 50)
|
||||
let offset: Int = query_int(path, "offset", 0)
|
||||
let nt: String = query_param(path, "node_type")
|
||||
if str_eq(nt, "") {
|
||||
return engram_scan_nodes_json(limit, offset)
|
||||
}
|
||||
return engram_scan_nodes_by_type_json(nt, limit, offset)
|
||||
}
|
||||
|
||||
// route_scan_edges — bulk export of all edges as a JSON array. Implemented
|
||||
// via engram_save → fs_read of the canonical on-disk snapshot, which the
|
||||
// runtime keeps in lockstep with the in-memory graph. Live against the
|
||||
// running graph, not a stale export.
|
||||
fn route_scan_edges(method: String, path: String, body: String) -> String {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let snap_path: String = dir + "/snapshot.json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
if str_eq(snap, "") { return "[]" }
|
||||
// json_get truncates at the first delimiter (no bracket depth tracking),
|
||||
// so for the edges ARRAY value we need json_get_raw, which honors
|
||||
// brackets and returns the full sub-JSON.
|
||||
let edges: String = json_get_raw(snap, "edges")
|
||||
if str_eq(edges, "") { return "[]" }
|
||||
return edges
|
||||
}
|
||||
|
||||
fn route_search(method: String, path: String, body: String) -> String {
|
||||
let q: String = ""
|
||||
if str_eq(method, "GET") {
|
||||
let q = query_param(path, "q")
|
||||
} else {
|
||||
let q = json_get_string(body, "query")
|
||||
}
|
||||
let limit: Int = query_int(path, "limit", 20)
|
||||
if limit == 0 { let limit = json_get_int(body, "limit") }
|
||||
if limit == 0 { let limit = 20 }
|
||||
return engram_search_json(q, limit)
|
||||
}
|
||||
|
||||
fn route_activate(method: String, path: String, body: String) -> String {
|
||||
let q: String = ""
|
||||
let depth: Int = 3
|
||||
if str_eq(method, "GET") {
|
||||
let q = query_param(path, "q")
|
||||
let depth = query_int(path, "depth", 3)
|
||||
} else {
|
||||
let q = json_get_string(body, "query")
|
||||
let bd: Int = json_get_int(body, "depth")
|
||||
if bd > 0 { let depth = bd }
|
||||
}
|
||||
return "{\"results\":" + engram_activate_json(q, depth) + "}"
|
||||
}
|
||||
|
||||
fn route_create_edge(method: String, path: String, body: String) -> String {
|
||||
let from_id: String = json_get_string(body, "from_id")
|
||||
let to_id: String = json_get_string(body, "to_id")
|
||||
let relation: String = json_get_string(body, "relation")
|
||||
if str_eq(relation, "") { let relation = "associates" }
|
||||
let weight: Float = json_get_float(body, "weight")
|
||||
if weight == 0.0 { let weight = 0.5 }
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
|
||||
}
|
||||
|
||||
fn route_neighbors(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/neighbors/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
let depth: Int = query_int(path, "depth", 1)
|
||||
return engram_neighbors_json(id, depth, "both")
|
||||
}
|
||||
|
||||
fn route_strengthen(method: String, path: String, body: String) -> String {
|
||||
let id: String = json_get_string(body, "node_id")
|
||||
if str_eq(id, "") { return err_json("missing node_id") }
|
||||
engram_strengthen(id)
|
||||
ok_json()
|
||||
}
|
||||
|
||||
fn route_forget(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/nodes/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
engram_forget(id)
|
||||
ok_json()
|
||||
}
|
||||
|
||||
fn route_save(method: String, path: String, body: String) -> String {
|
||||
let p: String = json_get_string(body, "path")
|
||||
if str_eq(p, "") {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let p = dir + "/snapshot.json"
|
||||
}
|
||||
engram_save(p)
|
||||
"{\"ok\":true,\"path\":\"" + p + "\"}"
|
||||
}
|
||||
|
||||
fn route_load(method: String, path: String, body: String) -> String {
|
||||
let p: String = json_get_string(body, "path")
|
||||
if str_eq(p, "") {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let p = dir + "/snapshot.json"
|
||||
}
|
||||
engram_load(p)
|
||||
ok_json()
|
||||
}
|
||||
|
||||
fn route_health(method: String, path: String, body: String) -> String {
|
||||
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"
|
||||
}
|
||||
|
||||
// ── Auth ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn check_auth_ok(method: String, body: String) -> Bool {
|
||||
let key: String = env("ENGRAM_API_KEY")
|
||||
if str_eq(key, "") { return true }
|
||||
// Read-only methods don't require auth. Until http_serve surfaces
|
||||
// request headers we can't accept a Bearer token cleanly; mutating
|
||||
// requests must include "_auth": "<key>" in the JSON body.
|
||||
if str_eq(method, "GET") { return true }
|
||||
let provided: String = json_get_string(body, "_auth")
|
||||
if str_eq(provided, key) { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
// ── Dispatcher ────────────────────────────────────────────────────────────────
|
||||
|
||||
fn handle_request(method: String, path: String, body: String) -> String {
|
||||
let clean: String = strip_query(path)
|
||||
|
||||
// Health is always reachable
|
||||
if str_eq(method, "GET") {
|
||||
if str_eq(clean, "/health") || str_eq(clean, "/") {
|
||||
return route_health(method, path, body)
|
||||
}
|
||||
}
|
||||
|
||||
// Auth (when ENGRAM_API_KEY is set)
|
||||
if !check_auth_ok(method, body) {
|
||||
return err_json("unauthorized")
|
||||
}
|
||||
|
||||
// Stats
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
|
||||
return route_stats(method, path, body)
|
||||
}
|
||||
|
||||
// Nodes
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
|
||||
return route_create_node(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes") || str_eq(clean, "/nodes/list") || str_eq(clean, "/api/nodes/list")) {
|
||||
return route_scan_nodes(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
||||
return route_scan_edges(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") {
|
||||
return route_get_node(method, path, body)
|
||||
}
|
||||
if str_eq(method, "DELETE") && str_starts_with(clean, "/api/nodes/") {
|
||||
return route_forget(method, path, body)
|
||||
}
|
||||
|
||||
// Edges
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
||||
return route_create_edge(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
|
||||
return route_neighbors(method, path, body)
|
||||
}
|
||||
|
||||
// Activation + Search
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/activate") || str_eq(clean, "/activate")) {
|
||||
return route_activate(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/activate") {
|
||||
return route_activate(method, path, body)
|
||||
}
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/search") || str_eq(clean, "/search")) {
|
||||
return route_search(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/search") {
|
||||
return route_search(method, path, body)
|
||||
}
|
||||
|
||||
// Strengthen
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) {
|
||||
return route_strengthen(method, path, body)
|
||||
}
|
||||
|
||||
// Persistence
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) {
|
||||
return route_save(method, path, body)
|
||||
}
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
|
||||
return route_load(method, path, body)
|
||||
}
|
||||
|
||||
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
|
||||
}
|
||||
|
||||
// ── Entry ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
let bind_str: String = env("ENGRAM_BIND")
|
||||
if str_eq(bind_str, "") { let bind_str = ":8742" }
|
||||
let port: Int = parse_port(bind_str)
|
||||
|
||||
// On startup, try to load any existing snapshot (best effort).
|
||||
let data_dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(data_dir, "") { let data_dir = "/tmp/engram" }
|
||||
let snapshot_path: String = data_dir + "/snapshot.json"
|
||||
engram_load(snapshot_path)
|
||||
|
||||
println("[engram] runtime-native graph engine")
|
||||
println("[engram] data_dir=" + data_dir)
|
||||
println("[engram] node_count=" + int_to_str(engram_node_count()))
|
||||
println("[engram] edge_count=" + int_to_str(engram_edge_count()))
|
||||
println("[engram] listening on " + int_to_str(port))
|
||||
|
||||
http_set_handler("handle_request")
|
||||
http_serve(port, "handle_request")
|
||||
Reference in New Issue
Block a user