Compare commits
35 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7e4b21c779 | |||
| d71fc4c1c0 | |||
| a118d19393 | |||
| c6aa1e5c53 | |||
| ff577391f2 | |||
| ee0d5f9b97 | |||
| 391bd818ea | |||
| 43636aed99 | |||
| 8f8ccc945e | |||
| 409ec99397 | |||
| dc39a61e2c | |||
| eba9eac8a8 | |||
| ab6b52a0b4 | |||
| 2baa0b9a41 | |||
| 6a8b2461cd | |||
| bcb356fe69 | |||
| dd7827059a | |||
| 208e36c899 | |||
| b97ce74d1f | |||
| 155a449c4e | |||
| 4696fd6833 | |||
| 581a351fb1 | |||
| 8ce8656de2 | |||
| 1e49560f1f | |||
| e8f0b5a9de | |||
| 40287c4cfc | |||
| 0481bea44d | |||
| 9d565ca080 | |||
| 4773dd0aa2 | |||
| 6b9d9e6c4a | |||
| b4967af13e | |||
| e3dabe3e08 | |||
| 0a0a2bcb44 | |||
| 2b2a1246e7 | |||
| 5c41c66a0f |
@@ -214,9 +214,18 @@ jobs:
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||
# and the SDK silently never published. Surface failures now.
|
||||
set -euo pipefail
|
||||
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||
|
||||
VERSION="${GITHUB_SHA:0:8}"
|
||||
|
||||
@@ -268,6 +277,12 @@ jobs:
|
||||
# Patches ci-base:dev in-place: pulls the existing image (which has all
|
||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||
#
|
||||
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||
# runner where DinD/Docker availability is fragile. A failure here must
|
||||
# never block or redden the job — the SDK publish above is the deliverable.
|
||||
continue-on-error: true
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
|
||||
@@ -212,12 +212,21 @@ jobs:
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||
# and the SDK silently never published. Surface failures now.
|
||||
set -euo pipefail
|
||||
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
apt-get install -y -qq apt-transport-https ca-certificates curl
|
||||
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||
|
||||
VERSION="${GITHUB_SHA:0:8}"
|
||||
|
||||
@@ -253,6 +262,12 @@ jobs:
|
||||
# Patches ci-base:stage in-place: pulls the existing image (which has all
|
||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||
#
|
||||
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||
# runner where DinD/Docker availability is fragile. A failure here must
|
||||
# never block or redden the job — the SDK publish above is the deliverable.
|
||||
continue-on-error: true
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
|
||||
@@ -288,12 +288,21 @@ jobs:
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
run: |
|
||||
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||
# and the SDK silently never published. Surface failures now.
|
||||
set -euo pipefail
|
||||
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||
apt-get install -y -qq apt-transport-https ca-certificates curl
|
||||
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||
gcloud config set project neuron-785695
|
||||
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||
|
||||
VERSION="${GITHUB_SHA:0:8}"
|
||||
|
||||
@@ -345,6 +354,12 @@ jobs:
|
||||
# Patches ci-base:latest in-place: pulls the existing image (which has all
|
||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||
#
|
||||
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||
# runner where DinD/Docker availability is fragile. A failure here must
|
||||
# never block or redden the job — the SDK publish above is the deliverable.
|
||||
continue-on-error: true
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
# Link to produce the engram binary
|
||||
- name: Link engram binary
|
||||
run: |
|
||||
cc -std=c11 -O2 \
|
||||
cc -std=c11 -O2 -DHAVE_CURL \
|
||||
-I /usr/local/lib/el \
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
# Link to produce the engram binary
|
||||
- name: Link engram binary
|
||||
run: |
|
||||
cc -std=c11 -O2 \
|
||||
cc -std=c11 -O2 -DHAVE_CURL \
|
||||
-I /usr/local/lib/el \
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
# Link to produce the engram binary
|
||||
- name: Link engram binary
|
||||
run: |
|
||||
cc -std=c11 -O2 \
|
||||
cc -std=c11 -O2 -DHAVE_CURL \
|
||||
-I /usr/local/lib/el \
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
|
||||
Vendored
BIN
Binary file not shown.
Vendored
+142
-95
@@ -10,6 +10,7 @@ el_val_t query_param(el_val_t path, el_val_t key);
|
||||
el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
|
||||
el_val_t extract_id(el_val_t path, el_val_t prefix);
|
||||
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t persist_canonical(void);
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body);
|
||||
@@ -20,18 +21,23 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t check_auth_ok(el_val_t method, el_val_t body);
|
||||
el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body);
|
||||
|
||||
el_val_t bind_raw;
|
||||
el_val_t bind_str;
|
||||
el_val_t port;
|
||||
el_val_t data_dir_raw;
|
||||
el_val_t data_dir;
|
||||
el_val_t snapshot_path;
|
||||
el_val_t boot_snap;
|
||||
|
||||
el_val_t parse_port(el_val_t bind) {
|
||||
el_val_t colon = str_index_of(bind, EL_STR(":"));
|
||||
@@ -110,17 +116,22 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t persist_canonical(void) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; });
|
||||
engram_save(el_str_concat(dir, EL_STR("/snapshot.json")));
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t content = json_get_string(body, EL_STR("content"));
|
||||
el_val_t node_type = json_get_string(body, EL_STR("node_type"));
|
||||
if (str_eq(node_type, EL_STR(""))) {
|
||||
node_type = EL_STR("Memory");
|
||||
}
|
||||
el_val_t salience = json_get_float(body, EL_STR("salience"));
|
||||
if (salience == el_from_float(0.0)) {
|
||||
salience = el_from_float(0.5);
|
||||
}
|
||||
el_val_t nt_raw = json_get_string(body, EL_STR("node_type"));
|
||||
el_val_t node_type = ({ el_val_t _if_result_2 = 0; if (str_eq(nt_raw, EL_STR(""))) { _if_result_2 = (EL_STR("Memory")); } else { _if_result_2 = (nt_raw); } _if_result_2; });
|
||||
el_val_t sal_raw = json_get_float(body, EL_STR("salience"));
|
||||
el_val_t salience = ({ el_val_t _if_result_3 = 0; if ((sal_raw == el_from_float(0.0))) { _if_result_3 = (el_from_float(0.5)); } else { _if_result_3 = (sal_raw); } _if_result_3; });
|
||||
el_val_t id = engram_node(content, node_type, salience);
|
||||
el_val_t saved = persist_canonical();
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -146,11 +157,9 @@ el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) {
|
||||
}
|
||||
|
||||
el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_4 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_4 = (EL_STR("/tmp/engram")); } else { _if_result_4 = (dir_raw); } _if_result_4; });
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/.scan-export.json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
if (str_eq(snap, EL_STR(""))) {
|
||||
@@ -165,36 +174,22 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
|
||||
}
|
||||
|
||||
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t q = EL_STR("");
|
||||
if (str_eq(method, EL_STR("GET"))) {
|
||||
q = query_param(path, EL_STR("q"));
|
||||
} else {
|
||||
q = json_get_string(body, EL_STR("query"));
|
||||
}
|
||||
el_val_t limit = query_int(path, EL_STR("limit"), 20);
|
||||
if (limit == 0) {
|
||||
limit = json_get_int(body, EL_STR("limit"));
|
||||
}
|
||||
if (limit == 0) {
|
||||
limit = 20;
|
||||
}
|
||||
el_val_t q = ({ el_val_t _if_result_5 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_5 = (query_param(path, EL_STR("q"))); } else { _if_result_5 = (json_get_string(body, EL_STR("query"))); } _if_result_5; });
|
||||
el_val_t lim_url = query_int(path, EL_STR("limit"), 0);
|
||||
el_val_t lim_body = json_get_int(body, EL_STR("limit"));
|
||||
el_val_t lim_either = ({ el_val_t _if_result_6 = 0; if ((lim_url > 0)) { _if_result_6 = (lim_url); } else { _if_result_6 = (lim_body); } _if_result_6; });
|
||||
el_val_t limit = ({ el_val_t _if_result_7 = 0; if ((lim_either > 0)) { _if_result_7 = (lim_either); } else { _if_result_7 = (20); } _if_result_7; });
|
||||
return engram_search_json(q, limit);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t q = EL_STR("");
|
||||
el_val_t depth = 3;
|
||||
if (str_eq(method, EL_STR("GET"))) {
|
||||
q = query_param(path, EL_STR("q"));
|
||||
depth = query_int(path, EL_STR("depth"), 3);
|
||||
} else {
|
||||
q = json_get_string(body, EL_STR("query"));
|
||||
el_val_t bd = json_get_int(body, EL_STR("depth"));
|
||||
if (bd > 0) {
|
||||
depth = bd;
|
||||
}
|
||||
el_val_t q = ({ el_val_t _if_result_8 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_8 = (query_param(path, EL_STR("q"))); } else { _if_result_8 = (json_get_string(body, EL_STR("query"))); } _if_result_8; });
|
||||
if (str_eq(q, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing query"));
|
||||
}
|
||||
el_val_t d_raw = ({ el_val_t _if_result_9 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_9 = (query_int(path, EL_STR("depth"), 3)); } else { _if_result_9 = (json_get_int(body, EL_STR("depth"))); } _if_result_9; });
|
||||
el_val_t depth = ({ el_val_t _if_result_10 = 0; if ((d_raw > 0)) { _if_result_10 = (d_raw); } else { _if_result_10 = (3); } _if_result_10; });
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -202,15 +197,12 @@ el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t from_id = json_get_string(body, EL_STR("from_id"));
|
||||
el_val_t to_id = json_get_string(body, EL_STR("to_id"));
|
||||
el_val_t relation = json_get_string(body, EL_STR("relation"));
|
||||
if (str_eq(relation, EL_STR(""))) {
|
||||
relation = EL_STR("associates");
|
||||
}
|
||||
el_val_t weight = json_get_float(body, EL_STR("weight"));
|
||||
if (weight == el_from_float(0.0)) {
|
||||
weight = el_from_float(0.5);
|
||||
}
|
||||
el_val_t rel_raw = json_get_string(body, EL_STR("relation"));
|
||||
el_val_t relation = ({ el_val_t _if_result_11 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_11 = (EL_STR("associates")); } else { _if_result_11 = (rel_raw); } _if_result_11; });
|
||||
el_val_t w_raw = json_get_float(body, EL_STR("weight"));
|
||||
el_val_t weight = ({ el_val_t _if_result_12 = 0; if ((w_raw == el_from_float(0.0))) { _if_result_12 = (el_from_float(0.5)); } else { _if_result_12 = (w_raw); } _if_result_12; });
|
||||
engram_connect(from_id, to_id, weight, relation);
|
||||
el_val_t saved = persist_canonical();
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -231,6 +223,7 @@ el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return err_json(EL_STR("missing node_id"));
|
||||
}
|
||||
engram_strengthen(id);
|
||||
el_val_t saved = persist_canonical();
|
||||
return ok_json();
|
||||
return 0;
|
||||
}
|
||||
@@ -241,29 +234,40 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return err_json(EL_STR("missing id"));
|
||||
}
|
||||
engram_forget(id);
|
||||
el_val_t saved = persist_canonical();
|
||||
return ok_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t content = json_get_string(body, EL_STR("content"));
|
||||
if (str_eq(content, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing content"));
|
||||
}
|
||||
el_val_t sal = el_from_float(0.3);
|
||||
el_val_t imp = el_from_float(0.3);
|
||||
el_val_t conf = el_from_float(0.8);
|
||||
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p_raw = json_get_string(body, EL_STR("path"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_13 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_13 = (EL_STR("/tmp/engram")); } else { _if_result_13 = (dir_raw); } _if_result_13; });
|
||||
el_val_t p = ({ el_val_t _if_result_14 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_14 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_14 = (p_raw); } _if_result_14; });
|
||||
engram_save(p);
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p_raw = json_get_string(body, EL_STR("path"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_15 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_15 = (EL_STR("/tmp/engram")); } else { _if_result_15 = (dir_raw); } _if_result_15; });
|
||||
el_val_t p = ({ el_val_t _if_result_16 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_16 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_16 = (p_raw); } _if_result_16; });
|
||||
engram_load(p);
|
||||
return ok_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}");
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/sync-export.json"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_17 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_17 = (EL_STR("/tmp/engram")); } else { _if_result_17 = (dir_raw); } _if_result_17; });
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/.sync-export.json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
if (str_eq(snap, EL_STR(""))) {
|
||||
@@ -273,36 +277,68 @@ el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p = json_get_string(body, EL_STR("path"));
|
||||
if (str_eq(p, EL_STR(""))) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
p = el_str_concat(dir, EL_STR("/snapshot.json"));
|
||||
return err_json(EL_STR("path is required"));
|
||||
}
|
||||
engram_save(p);
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}"));
|
||||
if (str_eq(fs_read(p), EL_STR(""))) {
|
||||
return err_json(EL_STR("file missing or empty"));
|
||||
}
|
||||
el_val_t before_n = engram_node_count();
|
||||
el_val_t before_e = engram_edge_count();
|
||||
engram_load_merge(p);
|
||||
el_val_t added_n = (engram_node_count() - before_n);
|
||||
el_val_t added_e = (engram_edge_count() - before_e);
|
||||
el_val_t saved = persist_canonical();
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"nodes_added\":"), int_to_str(added_n)), EL_STR(",\"edges_added\":")), int_to_str(added_e)), EL_STR(",\"node_count\":")), int_to_str(engram_node_count())), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p = json_get_string(body, EL_STR("path"));
|
||||
if (str_eq(p, EL_STR(""))) {
|
||||
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(dir, EL_STR(""))) {
|
||||
dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
p = el_str_concat(dir, EL_STR("/snapshot.json"));
|
||||
el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t content = json_get_string(body, EL_STR("content"));
|
||||
if (str_eq(content, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing content"));
|
||||
}
|
||||
engram_load(p);
|
||||
return ok_json();
|
||||
el_val_t sal = el_from_float(0.3);
|
||||
el_val_t imp = el_from_float(0.3);
|
||||
el_val_t conf = el_from_float(0.8);
|
||||
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
|
||||
el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS"));
|
||||
el_val_t ret_ms = ({ el_val_t _if_result_18 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_18 = (172800000); } else { _if_result_18 = (str_to_int(ret_raw)); } _if_result_18; });
|
||||
el_val_t pruned = engram_prune_telemetry(ret_ms);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}");
|
||||
el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t content = json_get_string(body, EL_STR("content"));
|
||||
if (str_eq(content, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing content"));
|
||||
}
|
||||
el_val_t title = json_get_string(body, EL_STR("title"));
|
||||
el_val_t label = ({ el_val_t _if_result_19 = 0; if (str_eq(title, EL_STR(""))) { _if_result_19 = (str_slice(content, 0, 60)); } else { _if_result_19 = (title); } _if_result_19; });
|
||||
el_val_t category_raw = json_get_string(body, EL_STR("category"));
|
||||
el_val_t category = ({ el_val_t _if_result_20 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_20 = (EL_STR("other")); } else { _if_result_20 = (category_raw); } _if_result_20; });
|
||||
el_val_t ktier_raw = json_get_string(body, EL_STR("tier"));
|
||||
el_val_t ktier = ({ el_val_t _if_result_21 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_21 = (EL_STR("note")); } else { _if_result_21 = (ktier_raw); } _if_result_21; });
|
||||
el_val_t project = json_get_string(body, EL_STR("project"));
|
||||
el_val_t tags_raw = json_get_raw(body, EL_STR("tags"));
|
||||
el_val_t tags_base = ({ el_val_t _if_result_22 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_22 = (EL_STR("[]")); } else { _if_result_22 = (tags_raw); } _if_result_22; });
|
||||
el_val_t base_len = str_len(tags_base);
|
||||
el_val_t head = str_slice(tags_base, 0, (base_len - 1));
|
||||
el_val_t sep = ({ el_val_t _if_result_23 = 0; if (str_eq(head, EL_STR("["))) { _if_result_23 = (EL_STR("")); } else { _if_result_23 = (EL_STR(",")); } _if_result_23; });
|
||||
el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t proj_tag = ({ el_val_t _if_result_24 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_24 = (EL_STR("")); } else { _if_result_24 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_24; });
|
||||
el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]"));
|
||||
el_val_t sal = el_from_float(0.5);
|
||||
el_val_t imp = el_from_float(0.5);
|
||||
el_val_t conf = el_from_float(0.9);
|
||||
el_val_t id = engram_node_full(content, EL_STR("Knowledge"), label, sal, imp, conf, EL_STR("Semantic"), tags);
|
||||
el_val_t saved = persist_canonical();
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -329,12 +365,15 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return route_health(method, path, body);
|
||||
}
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && str_starts_with(clean, EL_STR("/api/neuron/state-events"))) {
|
||||
return route_create_ise(method, path, body);
|
||||
if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/state-events"))) {
|
||||
return route_emit_ise(method, path, body);
|
||||
}
|
||||
if (!check_auth_ok(method, body)) {
|
||||
return err_json(EL_STR("unauthorized"));
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/knowledge/capture"))) {
|
||||
return route_capture_knowledge(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) {
|
||||
return route_stats(method, path, body);
|
||||
}
|
||||
@@ -374,32 +413,40 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) {
|
||||
return route_strengthen(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/sync")) || str_eq(clean, EL_STR("/sync")))) {
|
||||
return route_sync(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) {
|
||||
return route_save(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) {
|
||||
return route_load(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load-merge")) || str_eq(clean, EL_STR("/load-merge")))) {
|
||||
return route_load_merge(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) {
|
||||
return route_sync(method, path, body);
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int _argc, char** _argv) {
|
||||
el_runtime_init_args(_argc, _argv);
|
||||
bind_str = env(EL_STR("ENGRAM_BIND"));
|
||||
if (str_eq(bind_str, EL_STR(""))) {
|
||||
bind_str = EL_STR(":8742");
|
||||
}
|
||||
bind_raw = env(EL_STR("ENGRAM_BIND"));
|
||||
bind_str = ({ el_val_t _if_result_25 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_25 = (EL_STR(":8742")); } else { _if_result_25 = (bind_raw); } _if_result_25; });
|
||||
port = parse_port(bind_str);
|
||||
data_dir = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
if (str_eq(data_dir, EL_STR(""))) {
|
||||
data_dir = EL_STR("/tmp/engram");
|
||||
}
|
||||
data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
data_dir = ({ el_val_t _if_result_26 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_26 = (EL_STR("/tmp/engram")); } else { _if_result_26 = (data_dir_raw); } _if_result_26; });
|
||||
snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json"));
|
||||
engram_load(snapshot_path);
|
||||
boot_snap = fs_read(snapshot_path);
|
||||
if (!str_eq(boot_snap, EL_STR(""))) {
|
||||
if (engram_node_count() == 0) {
|
||||
println(EL_STR("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes \xe2\x80\x94 preserving copy at snapshot.failed-load.json"));
|
||||
fs_write(el_str_concat(data_dir, EL_STR("/snapshot.failed-load.json")), boot_snap);
|
||||
} else {
|
||||
fs_write(el_str_concat(data_dir, EL_STR("/snapshot.boot-backup.json")), boot_snap);
|
||||
}
|
||||
}
|
||||
println(EL_STR("[engram] runtime-native graph engine"));
|
||||
println(el_str_concat(EL_STR("[engram] data_dir="), data_dir));
|
||||
println(el_str_concat(EL_STR("[engram] node_count="), int_to_str(engram_node_count())));
|
||||
|
||||
+190
-54
@@ -76,13 +76,43 @@ fn route_stats(method: String, path: String, body: String) -> String {
|
||||
engram_stats_json()
|
||||
}
|
||||
|
||||
// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
|
||||
// inner scope only — it does NOT mutate the outer binding (documented with
|
||||
// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
|
||||
// that broken pattern, so defaults never applied: nodes were created with
|
||||
// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with
|
||||
// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and
|
||||
// save/load with no "path" hit engram_save(""). Rewritten to the
|
||||
// `let x = if cond { a } else { b }` expression form (the pattern the newer
|
||||
// routes route_emit_ise/route_capture_knowledge already use correctly).
|
||||
// persist_canonical — save the canonical snapshot after a durable write.
|
||||
//
|
||||
// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ
|
||||
// routes from writing the canonical snapshot.json — but nothing was left
|
||||
// that saved it on WRITE. Every mutation (node create, edge create,
|
||||
// knowledge capture, forget, merge) lived only in RAM until someone POSTed
|
||||
// /api/save manually; a process restart silently discarded everything since
|
||||
// the last manual save. Observed live: two engram restarts during the
|
||||
// 2026-07-22 review reverted the store to a ~17h-old snapshot, destroying
|
||||
// same-day writes. Reads must never write the canonical; writes must always
|
||||
// persist it. ISE telemetry is deliberately excluded (48h-pruned, loss-
|
||||
// tolerant, ~2/min — snapshotting the whole store per heartbeat is waste;
|
||||
// any durable write that follows persists the pruning too).
|
||||
fn persist_canonical() -> Int {
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
engram_save(dir + "/snapshot.json")
|
||||
return 1
|
||||
}
|
||||
|
||||
fn route_create_node(method: String, path: String, body: String) -> String {
|
||||
let content: String = json_get_string(body, "content")
|
||||
let node_type: String = json_get_string(body, "node_type")
|
||||
if str_eq(node_type, "") { let node_type = "Memory" }
|
||||
let salience: Float = json_get_float(body, "salience")
|
||||
if salience == 0.0 { let salience = 0.5 }
|
||||
let nt_raw: String = json_get_string(body, "node_type")
|
||||
let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw }
|
||||
let sal_raw: Float = json_get_float(body, "salience")
|
||||
let salience: Float = if sal_raw == 0.0 { 0.5 } else { sal_raw }
|
||||
let id: String = engram_node(content, node_type, salience)
|
||||
let saved: Int = persist_canonical()
|
||||
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
|
||||
}
|
||||
|
||||
@@ -103,13 +133,14 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String {
|
||||
}
|
||||
|
||||
// route_scan_edges — bulk export of all edges as a JSON array. Implemented
|
||||
// via engram_save → fs_read of the canonical on-disk snapshot, which the
|
||||
// runtime keeps in lockstep with the in-memory graph. Live against the
|
||||
// running graph, not a stale export.
|
||||
// via engram_save → fs_read of a SCRATCH export path. (2026-07-21 self-review:
|
||||
// previously this saved over the canonical snapshot.json on every GET — if the
|
||||
// process ever booted with a partial/empty store, the first read request
|
||||
// clobbered the good snapshot. Read routes must never write the canonical path.)
|
||||
fn route_scan_edges(method: String, path: String, body: String) -> String {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let snap_path: String = dir + "/snapshot.json"
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let snap_path: String = dir + "/.scan-export.json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
if str_eq(snap, "") { return "[]" }
|
||||
@@ -122,40 +153,34 @@ fn route_scan_edges(method: String, path: String, body: String) -> String {
|
||||
}
|
||||
|
||||
fn route_search(method: String, path: String, body: String) -> String {
|
||||
let q: String = ""
|
||||
if str_eq(method, "GET") {
|
||||
let q = query_param(path, "q")
|
||||
} else {
|
||||
let q = json_get_string(body, "query")
|
||||
}
|
||||
let limit: Int = query_int(path, "limit", 20)
|
||||
if limit == 0 { let limit = json_get_int(body, "limit") }
|
||||
if limit == 0 { let limit = 20 }
|
||||
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
|
||||
let lim_url: Int = query_int(path, "limit", 0)
|
||||
let lim_body: Int = json_get_int(body, "limit")
|
||||
let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body }
|
||||
let limit: Int = if lim_either > 0 { lim_either } else { 20 }
|
||||
return engram_search_json(q, limit)
|
||||
}
|
||||
|
||||
fn route_activate(method: String, path: String, body: String) -> String {
|
||||
let q: String = ""
|
||||
let depth: Int = 3
|
||||
if str_eq(method, "GET") {
|
||||
let q = query_param(path, "q")
|
||||
let depth = query_int(path, "depth", 3)
|
||||
} else {
|
||||
let q = json_get_string(body, "query")
|
||||
let bd: Int = json_get_int(body, "depth")
|
||||
if bd > 0 { let depth = bd }
|
||||
}
|
||||
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
|
||||
// Guard: engram_activate with an empty query matches zero seeds, which
|
||||
// zeroes ALL carried working-memory weights (documented in awareness.el
|
||||
// perceive()). Never let an empty activation through to wipe WM.
|
||||
if str_eq(q, "") { return err_json("missing query") }
|
||||
let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") }
|
||||
let depth: Int = if d_raw > 0 { d_raw } else { 3 }
|
||||
return "{\"results\":" + engram_activate_json(q, depth) + "}"
|
||||
}
|
||||
|
||||
fn route_create_edge(method: String, path: String, body: String) -> String {
|
||||
let from_id: String = json_get_string(body, "from_id")
|
||||
let to_id: String = json_get_string(body, "to_id")
|
||||
let relation: String = json_get_string(body, "relation")
|
||||
if str_eq(relation, "") { let relation = "associates" }
|
||||
let weight: Float = json_get_float(body, "weight")
|
||||
if weight == 0.0 { let weight = 0.5 }
|
||||
let rel_raw: String = json_get_string(body, "relation")
|
||||
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
||||
let w_raw: Float = json_get_float(body, "weight")
|
||||
let weight: Float = if w_raw == 0.0 { 0.5 } else { w_raw }
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
let saved: Int = persist_canonical()
|
||||
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
|
||||
}
|
||||
|
||||
@@ -170,6 +195,7 @@ fn route_strengthen(method: String, path: String, body: String) -> String {
|
||||
let id: String = json_get_string(body, "node_id")
|
||||
if str_eq(id, "") { return err_json("missing node_id") }
|
||||
engram_strengthen(id)
|
||||
let saved: Int = persist_canonical()
|
||||
ok_json()
|
||||
}
|
||||
|
||||
@@ -177,27 +203,24 @@ fn route_forget(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/nodes/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
engram_forget(id)
|
||||
let saved: Int = persist_canonical()
|
||||
ok_json()
|
||||
}
|
||||
|
||||
fn route_save(method: String, path: String, body: String) -> String {
|
||||
let p: String = json_get_string(body, "path")
|
||||
if str_eq(p, "") {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let p = dir + "/snapshot.json"
|
||||
}
|
||||
let p_raw: String = json_get_string(body, "path")
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
||||
engram_save(p)
|
||||
"{\"ok\":true,\"path\":\"" + p + "\"}"
|
||||
}
|
||||
|
||||
fn route_load(method: String, path: String, body: String) -> String {
|
||||
let p: String = json_get_string(body, "path")
|
||||
if str_eq(p, "") {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let p = dir + "/snapshot.json"
|
||||
}
|
||||
let p_raw: String = json_get_string(body, "path")
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
||||
engram_load(p)
|
||||
ok_json()
|
||||
}
|
||||
@@ -219,15 +242,36 @@ fn route_health(method: String, path: String, body: String) -> String {
|
||||
// (it skips nodes already present by ID). Auth-exempt: same-host internal call.
|
||||
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
|
||||
fn route_sync(method: String, path: String, body: String) -> String {
|
||||
let dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(dir, "") { let dir = "/tmp/engram" }
|
||||
let snap_path: String = dir + "/snapshot.json"
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
// 2026-07-21 self-review: export to a scratch path, never the canonical
|
||||
// snapshot.json — read routes must not be able to clobber the good snapshot.
|
||||
let snap_path: String = dir + "/.sync-export.json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
if str_eq(snap, "") { return "{\"nodes\":[],\"edges\":[]}" }
|
||||
return snap
|
||||
}
|
||||
|
||||
// route_load_merge — POST /api/load-merge {"path": "..."} — merge a snapshot
|
||||
// file into the live store WITHOUT resetting it (engram_load_merge skips nodes
|
||||
// already present by id). Added 2026-07-21 self-review to restore the 244 kn-
|
||||
// identity Knowledge nodes lost from the snapshot lineage between 05-13 and
|
||||
// 07-13. Requires an explicit path: refuses to run without one so it can never
|
||||
// be triggered accidentally against a default.
|
||||
fn route_load_merge(method: String, path: String, body: String) -> String {
|
||||
let p: String = json_get_string(body, "path")
|
||||
if str_eq(p, "") { return err_json("path is required") }
|
||||
if str_eq(fs_read(p), "") { return err_json("file missing or empty") }
|
||||
let before_n: Int = engram_node_count()
|
||||
let before_e: Int = engram_edge_count()
|
||||
engram_load_merge(p)
|
||||
let added_n: Int = engram_node_count() - before_n
|
||||
let added_e: Int = engram_edge_count() - before_e
|
||||
let saved: Int = persist_canonical()
|
||||
"{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}"
|
||||
}
|
||||
|
||||
// route_emit_ise — write an InternalStateEvent node from the soul daemon.
|
||||
//
|
||||
// Endpoint: POST /api/neuron/state-events
|
||||
@@ -241,10 +285,20 @@ fn route_sync(method: String, path: String, body: String) -> String {
|
||||
//
|
||||
// Salience/importance set to match engram_node_full ISE defaults used by the
|
||||
// in-process fallback path in awareness.el (salience=0.3, importance=0.3,
|
||||
// confidence=0.8, tier=Episodic). High temporal_decay_rate (1.617) — ISEs
|
||||
// are inherently transient; they should decay faster than structural knowledge.
|
||||
// confidence=0.8, tier=Episodic).
|
||||
// (2026-06-26 self-review: added this route after discovering ise_post was
|
||||
// silently failing — the soul posts here but the endpoint didn't exist.)
|
||||
//
|
||||
// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs
|
||||
// got temporal_decay_rate=1.617 — that was never implemented (engram_node_full
|
||||
// hardcodes 0.0), and per-node decay only dampens activation anyway; it never
|
||||
// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522
|
||||
// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in
|
||||
// engram_activate, so the fix is retention, not decay: every insert calls
|
||||
// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that
|
||||
// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting
|
||||
// "session-start" labels and self_review events as durable history. At
|
||||
// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever.
|
||||
fn route_emit_ise(method: String, path: String, body: String) -> String {
|
||||
let content: String = json_get_string(body, "content")
|
||||
if str_eq(content, "") { return err_json("missing content") }
|
||||
@@ -256,6 +310,64 @@ fn route_emit_ise(method: String, path: String, body: String) -> String {
|
||||
sal, imp, conf,
|
||||
"Episodic", "[\"internal-state\",\"InternalStateEvent\"]"
|
||||
)
|
||||
let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
|
||||
let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
|
||||
let pruned: Int = engram_prune_telemetry(ret_ms)
|
||||
"{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}"
|
||||
}
|
||||
|
||||
// ── Knowledge capture ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// route_capture_knowledge — direct Knowledge-node capture over HTTP.
|
||||
//
|
||||
// Endpoint: POST /api/neuron/knowledge/capture (auth required: "_auth" in body)
|
||||
// Body: {"content": "...", "title": "...", "category": "...",
|
||||
// "tier": "note|lesson|canonical", "tags": [...], "project": "...",
|
||||
// "_auth": "<key>"}
|
||||
//
|
||||
// WHY (2026-07-15 self-review): the world-ingestor integrator was designed
|
||||
// against this endpoint (its MCP-unavailable fallback), but the route never
|
||||
// existed — every direct push 404'd, and because the auth gate ran before
|
||||
// routing, the failure surfaced as {"error":"unauthorized"} and was
|
||||
// misdiagnosed for two weeks while world knowledge silently dropped.
|
||||
// POST /api/nodes was no substitute: it discards label/tags/tier, which
|
||||
// makes captured knowledge invisible to tag-scoped search and curiosity.
|
||||
//
|
||||
// The incoming knowledge tier (note/lesson/canonical) is preserved as a
|
||||
// "tier:<x>" tag rather than mapped onto Engram's cognitive tiers — Knowledge
|
||||
// nodes land in Semantic (stable reference), and the epistemic tier stays
|
||||
// queryable without inventing a lossy mapping.
|
||||
fn route_capture_knowledge(method: String, path: String, body: String) -> String {
|
||||
let content: String = json_get_string(body, "content")
|
||||
if str_eq(content, "") { return err_json("missing content") }
|
||||
let title: String = json_get_string(body, "title")
|
||||
let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title }
|
||||
let category_raw: String = json_get_string(body, "category")
|
||||
let category: String = if str_eq(category_raw, "") { "other" } else { category_raw }
|
||||
let ktier_raw: String = json_get_string(body, "tier")
|
||||
let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw }
|
||||
let project: String = json_get_string(body, "project")
|
||||
let tags_raw: String = json_get_raw(body, "tags")
|
||||
let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw }
|
||||
// Merge category/tier/project markers into the tag array. Search matches
|
||||
// against the tags string, so these make captures findable by facet.
|
||||
let base_len: Int = str_len(tags_base)
|
||||
let head: String = str_slice(tags_base, 0, base_len - 1)
|
||||
let sep: String = if str_eq(head, "[") { "" } else { "," }
|
||||
let safe_cat: String = str_replace(category, "\"", "'")
|
||||
let safe_tier: String = str_replace(ktier, "\"", "'")
|
||||
let safe_proj: String = str_replace(project, "\"", "'")
|
||||
let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" }
|
||||
let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]"
|
||||
let sal: Float = 0.5
|
||||
let imp: Float = 0.5
|
||||
let conf: Float = 0.9
|
||||
let id: String = engram_node_full(
|
||||
content, "Knowledge", label,
|
||||
sal, imp, conf,
|
||||
"Semantic", tags
|
||||
)
|
||||
let saved: Int = persist_canonical()
|
||||
"{\"ok\":true,\"id\":\"" + id + "\"}"
|
||||
}
|
||||
|
||||
@@ -295,6 +407,12 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
return err_json("unauthorized")
|
||||
}
|
||||
|
||||
// Knowledge capture (auth enforced above; the world-ingestor integrator
|
||||
// and any headless session without MCP push knowledge through this)
|
||||
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") {
|
||||
return route_capture_knowledge(method, path, body)
|
||||
}
|
||||
|
||||
// Stats
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
|
||||
return route_stats(method, path, body)
|
||||
@@ -351,6 +469,9 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
|
||||
return route_load(method, path, body)
|
||||
}
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/load-merge") || str_eq(clean, "/load-merge")) {
|
||||
return route_load_merge(method, path, body)
|
||||
}
|
||||
|
||||
// Sync — soul daemon periodic pull of non-ISE knowledge into in-process graph
|
||||
if str_eq(method, "GET") && str_eq(clean, "/api/sync") {
|
||||
@@ -362,16 +483,31 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
|
||||
// ── Entry ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
let bind_str: String = env("ENGRAM_BIND")
|
||||
if str_eq(bind_str, "") { let bind_str = ":8742" }
|
||||
let bind_raw: String = env("ENGRAM_BIND")
|
||||
let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
|
||||
let port: Int = parse_port(bind_str)
|
||||
|
||||
// On startup, try to load any existing snapshot (best effort).
|
||||
let data_dir: String = env("ENGRAM_DATA_DIR")
|
||||
if str_eq(data_dir, "") { let data_dir = "/tmp/engram" }
|
||||
let data_dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw }
|
||||
let snapshot_path: String = data_dir + "/snapshot.json"
|
||||
engram_load(snapshot_path)
|
||||
|
||||
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
||||
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
||||
// Preserve the evidence and warn loudly — and since read routes no longer write
|
||||
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
||||
let boot_snap: String = fs_read(snapshot_path)
|
||||
if !str_eq(boot_snap, "") {
|
||||
if engram_node_count() == 0 {
|
||||
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
||||
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
||||
} else {
|
||||
// Good load: keep a boot-time backup of the snapshot as loaded.
|
||||
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
||||
}
|
||||
}
|
||||
|
||||
println("[engram] runtime-native graph engine")
|
||||
println("[engram] data_dir=" + data_dir)
|
||||
println("[engram] node_count=" + int_to_str(engram_node_count()))
|
||||
|
||||
@@ -17,6 +17,16 @@
|
||||
// 4. Append dep to order after all its transitive deps
|
||||
// 5. Deduplicate: skip already-ordered vessels
|
||||
|
||||
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
|
||||
// give elc the C prototypes so generated install.c compiles cleanly under strict
|
||||
// compilers (gcc>=14 / clang) that reject implicit function declarations.
|
||||
extern fn manifest_name(src: String) -> String // manifest.el
|
||||
extern fn manifest_deps(src: String) -> String // manifest.el
|
||||
extern fn registry_token() -> String // registry.el
|
||||
extern fn registry_find(name: String, version: String) -> String // registry.el
|
||||
extern fn registry_latest_version(name: String) -> String // registry.el
|
||||
|
||||
// ── Install paths ─────────────────────────────────────────────────────────────
|
||||
|
||||
// packages_dir returns the root directory for installed vessels.
|
||||
|
||||
@@ -14,6 +14,15 @@
|
||||
// EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies)
|
||||
// EPM_TOKEN — Gitea personal access token (required for publish)
|
||||
|
||||
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||
// These symbols are defined in sibling epm modules or the El runtime and are
|
||||
// resolved at link time. The `extern fn` decls give elc the C prototype so the
|
||||
// generated registry.c compiles cleanly under strict compilers (gcc>=14 / clang)
|
||||
// that reject implicit function declarations. Signature arity must match the
|
||||
// definition; return/param types are informational (all lower to el_val_t).
|
||||
extern fn config(key: String) -> String // El runtime builtin
|
||||
extern fn read_installed() -> String // install.el
|
||||
|
||||
// ── Config helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
// registry_api_url returns the Gitea API base URL with no trailing slash.
|
||||
|
||||
@@ -6,6 +6,15 @@
|
||||
// Depends on: registry.el (registry_latest_version, registry_find),
|
||||
// install.el (read_installed, install_vessel, installed_version)
|
||||
|
||||
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
|
||||
// give elc the C prototypes so generated update.c compiles cleanly under strict
|
||||
// compilers (gcc>=14 / clang) that reject implicit function declarations.
|
||||
extern fn read_installed() -> String // install.el
|
||||
extern fn installed_version(name: String) -> String // install.el
|
||||
extern fn install_vessel(name: String, version: String) -> Bool // install.el
|
||||
extern fn registry_latest_version(name: String) -> String // registry.el
|
||||
|
||||
// ── Semver helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
// semver_part extracts the Nth dot-separated component from a semver string.
|
||||
|
||||
@@ -75,6 +75,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) {
|
||||
#include <direct.h> /* _mkdir */
|
||||
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
||||
#define timegm _mkgmtime /* UTC tm → time_t */
|
||||
#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() (<io.h>) is the equiv */
|
||||
|
||||
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
||||
static inline int setenv(const char* name, const char* value, int overwrite) {
|
||||
|
||||
@@ -82,8 +82,14 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0};
|
||||
static _Thread_local int _tl_arena_active = 0;
|
||||
|
||||
/* Binary-safe fs_read length — set by fs_read, consumed by http_send_response.
|
||||
* Allows serving PNGs and other binary files without strlen truncation. */
|
||||
static _Thread_local size_t _tl_fs_read_len = 0;
|
||||
* Allows serving PNGs and other binary files without strlen truncation.
|
||||
* PAIRED with the buffer pointer it describes: the length may only be applied
|
||||
* to the exact buffer fs_read returned. Without the pairing, any handler that
|
||||
* fs_read a file and then WRAPPED it into a larger response had that response
|
||||
* truncated to the file's length (Content-Length lied AND the send stopped
|
||||
* short) — the safety-contact onboarding trap, 2026-07-17. */
|
||||
static _Thread_local size_t _tl_fs_read_len = 0;
|
||||
static _Thread_local const char* _tl_fs_read_buf = NULL;
|
||||
|
||||
static void el_arena_track(char* p) {
|
||||
if (!_tl_arena_active || !p) return;
|
||||
@@ -101,6 +107,8 @@ static void el_arena_track(char* p) {
|
||||
void el_request_start(void) {
|
||||
_tl_arena.count = 0;
|
||||
_tl_arena_active = 1;
|
||||
_tl_fs_read_len = 0; /* never let a previous request's file length */
|
||||
_tl_fs_read_buf = NULL; /* leak into this response's byte accounting */
|
||||
}
|
||||
|
||||
/* Called by http_worker after the El handler returns and the response is sent.
|
||||
@@ -1484,11 +1492,14 @@ static void http_send_response(int fd, const char* body) {
|
||||
}
|
||||
|
||||
const char* eff_body = is_envelope ? env_body : body;
|
||||
/* Use the real byte count from fs_read if available (handles binary files
|
||||
* with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for
|
||||
* normal text/JSON responses where _tl_fs_read_len is 0. */
|
||||
size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body);
|
||||
/* Use the real byte count from fs_read ONLY when this body IS the exact
|
||||
* buffer fs_read returned (binary files with embedded null bytes — PNG,
|
||||
* WOFF2, etc.). Any other body — wrapped, enveloped, or derived — must be
|
||||
* measured with strlen, or it is truncated/over-read to the file's size. */
|
||||
size_t blen = (_tl_fs_read_len > 0 && eff_body == _tl_fs_read_buf)
|
||||
? _tl_fs_read_len : strlen(eff_body);
|
||||
_tl_fs_read_len = 0; /* consume — one-shot per response */
|
||||
_tl_fs_read_buf = NULL;
|
||||
int head_only = _tl_http_head_only;
|
||||
|
||||
JsonBuf hdrs; jb_init(&hdrs);
|
||||
@@ -1568,11 +1579,22 @@ static void* http_worker(void* arg) {
|
||||
const char* rs = EL_CSTR(r);
|
||||
/* Copy response out BEFORE arena teardown.
|
||||
* For binary files, _tl_fs_read_len holds the real byte count —
|
||||
* use memcpy instead of strdup so null bytes are preserved. */
|
||||
size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
|
||||
* use memcpy instead of strdup so null bytes are preserved.
|
||||
* The stored length applies ONLY when the response IS the exact
|
||||
* fs_read buffer; a wrapped/derived response must use strlen or
|
||||
* it gets truncated (or over-read) to the file's length. */
|
||||
size_t rlen;
|
||||
if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) {
|
||||
rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */
|
||||
} else {
|
||||
rlen = rs ? strlen(rs) : 0;
|
||||
_tl_fs_read_len = 0; /* hint doesn't describe this body */
|
||||
_tl_fs_read_buf = NULL;
|
||||
}
|
||||
response = malloc(rlen + 1);
|
||||
if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; }
|
||||
else if (response) { response[0] = '\0'; }
|
||||
if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */
|
||||
} else {
|
||||
response = el_strdup_persist("el-runtime: no http handler registered");
|
||||
}
|
||||
@@ -1822,10 +1844,20 @@ static void* http_worker_v2(void* arg) {
|
||||
el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : "");
|
||||
el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body));
|
||||
const char* rs = EL_CSTR(r);
|
||||
size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
|
||||
/* Same pairing rule as the v1 worker: the fs_read length is only
|
||||
* trustworthy for the exact buffer fs_read returned. */
|
||||
size_t rlen;
|
||||
if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) {
|
||||
rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */
|
||||
} else {
|
||||
rlen = rs ? strlen(rs) : 0;
|
||||
_tl_fs_read_len = 0; /* hint doesn't describe this body */
|
||||
_tl_fs_read_buf = NULL;
|
||||
}
|
||||
response = malloc(rlen + 1);
|
||||
if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; }
|
||||
else if (response) { response[0] = '\0'; }
|
||||
if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */
|
||||
el_release(hmap);
|
||||
} else {
|
||||
response = el_strdup_persist(
|
||||
@@ -1963,8 +1995,9 @@ void http_serve_async(el_val_t port, el_val_t handler) {
|
||||
int sock = socket(AF_INET6, SOCK_STREAM, 0);
|
||||
if (sock < 0) { perror("socket"); return; }
|
||||
int yes = 1; int no = 0;
|
||||
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no));
|
||||
/* Win32/mingw setsockopt takes optval as (const char*); the cast is portable on POSIX too. */
|
||||
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));
|
||||
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no));
|
||||
struct sockaddr_in6 addr;
|
||||
memset(&addr, 0, sizeof(addr));
|
||||
addr.sin6_family = AF_INET6;
|
||||
@@ -2023,6 +2056,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) {
|
||||
el_val_t fs_read(el_val_t pathv) {
|
||||
const char* path = EL_CSTR(pathv);
|
||||
_tl_fs_read_len = 0;
|
||||
_tl_fs_read_buf = NULL;
|
||||
if (!path) return el_wrap_str(el_strdup(""));
|
||||
FILE* f = fopen(path, "rb");
|
||||
if (!f) return el_wrap_str(el_strdup(""));
|
||||
@@ -2034,6 +2068,7 @@ el_val_t fs_read(el_val_t pathv) {
|
||||
size_t got = fread(buf, 1, (size_t)sz, f);
|
||||
buf[got] = '\0';
|
||||
_tl_fs_read_len = got; /* store real byte count for binary-safe send */
|
||||
_tl_fs_read_buf = buf; /* ...valid ONLY for this exact buffer */
|
||||
fclose(f);
|
||||
return el_wrap_str(buf);
|
||||
}
|
||||
@@ -3576,8 +3611,10 @@ el_val_t json_get_raw(el_val_t json_str, el_val_t key) {
|
||||
const char* k = EL_CSTR(key);
|
||||
const char* p = json_find_key(json, k);
|
||||
/* Clear fs_read binary-length hint — result is a fresh null-terminated
|
||||
* string, not the raw file bytes, so Content-Length must use strlen. */
|
||||
* string, not the raw file bytes, so Content-Length must use strlen.
|
||||
* (Kept although the pointer pairing now makes this redundant.) */
|
||||
_tl_fs_read_len = 0;
|
||||
_tl_fs_read_buf = NULL;
|
||||
if (!p) return el_wrap_str(el_strdup(""));
|
||||
const char* end = json_skip_value(p);
|
||||
size_t n = (size_t)(end - p);
|
||||
@@ -6826,6 +6863,312 @@ static int istr_contains(const char* hay, const char* needle) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Tokenized query matching ───────────────────────────────────────────
|
||||
* The engram query surface (search / activate / goal-bias) historically
|
||||
* matched the ENTIRE raw query string as a single case-insensitive
|
||||
* substring via istr_contains(field, q). That is Ctrl-F, not search:
|
||||
* a multi-word query like "windows msi signing" only matched a node whose
|
||||
* text contained that exact contiguous run, so real multi-word queries
|
||||
* returned zero. istr_contains stays as the per-TOKEN primitive; these
|
||||
* helpers split the query on whitespace and match ANY token, then rank by
|
||||
* how many DISTINCT tokens a node covers. Single-token queries are a strict
|
||||
* special case (score is 0 or 1) so single-word callers never regress. */
|
||||
#define ENGRAM_MAX_QTOKENS 32
|
||||
#define ENGRAM_QTOK_LEN 256
|
||||
|
||||
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
|
||||
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
|
||||
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
|
||||
static int engram_tokenize_query(const char* q,
|
||||
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
|
||||
int n = 0;
|
||||
if (!q) return 0;
|
||||
const char* p = q;
|
||||
while (*p && n < maxtok) {
|
||||
while (*p && isspace((unsigned char)*p)) p++;
|
||||
if (!*p) break;
|
||||
char buf[ENGRAM_QTOK_LEN];
|
||||
size_t tl = 0;
|
||||
while (*p && !isspace((unsigned char)*p)) {
|
||||
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
|
||||
p++;
|
||||
}
|
||||
buf[tl] = '\0';
|
||||
if (tl == 0) continue;
|
||||
int dup = 0;
|
||||
for (int s = 0; s < n; s++) {
|
||||
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
|
||||
}
|
||||
if (dup) continue;
|
||||
memcpy(toks[n], buf, tl + 1);
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Count how many of the ntok distinct query tokens appear (case-insensitive)
|
||||
* in the node's content, label, or tags. 0 == no match. */
|
||||
static int engram_node_match_score(const EngramNode* n,
|
||||
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||
int score = 0;
|
||||
for (int t = 0; t < ntok; t++) {
|
||||
if (istr_contains(n->content, toks[t]) ||
|
||||
istr_contains(n->label, toks[t]) ||
|
||||
istr_contains(n->tags, toks[t]))
|
||||
score++;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
/* Rank entry: distinct-token match count (primary, desc) then salience
|
||||
* (tiebreak, desc). */
|
||||
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||
static int engram_rank_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
|
||||
if (ea->salience < eb->salience) return 1;
|
||||
if (ea->salience > eb->salience) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════════════
|
||||
* SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
|
||||
* ──────────────────────────────────────────────────────────────────────────
|
||||
* Augments the lexical (istr_contains) matcher with dense-vector retrieval.
|
||||
* Node content and the query are embedded through a local Ollama server;
|
||||
* nodes are ranked by cosine similarity and UNIONED with lexical hits. This
|
||||
* lets a paraphrase query surface a node whose words never appear in it.
|
||||
*
|
||||
* DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot
|
||||
* runtime probe of the embedding endpoint. If curl is not compiled in, or
|
||||
* Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns
|
||||
* "no semantic signal" and callers fall back to pure lexical behaviour —
|
||||
* byte-for-byte the pre-existing search.
|
||||
*
|
||||
* CACHE. Node embeddings are computed lazily on first use and cached in
|
||||
* process memory keyed by node id, with an FNV-1a content hash for
|
||||
* invalidation (edited content re-embeds). The query is embedded once per
|
||||
* search call. This is what "avoid re-embedding the whole graph every query"
|
||||
* buys us: a warm cache serves cosine from RAM. (A cold process still pays
|
||||
* O(N) embed calls the first time each node is scanned — persisting the cache
|
||||
* to a snapshot sidecar is the documented next step, not done here.)
|
||||
*
|
||||
* nomic task prefixes ("search_query:" / "search_document:") are applied
|
||||
* because nomic-embed-text is trained with them; they materially improve
|
||||
* retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48).
|
||||
*
|
||||
* ENV:
|
||||
* ENGRAM_SEMANTIC "0" disables; unset/other = auto-probe
|
||||
* ENGRAM_EMBED_URL default http://localhost:11434/api/embeddings
|
||||
* ENGRAM_EMBED_MODEL default nomic-embed-text
|
||||
* ENGRAM_SEMANTIC_MIN cosine threshold for a pure-semantic match (def 0.6)
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static double engram_semantic_min(void) {
|
||||
static double v = -1.0;
|
||||
if (v >= 0.0) return v;
|
||||
const char* s = getenv("ENGRAM_SEMANTIC_MIN");
|
||||
double d = 0.6;
|
||||
if (s && *s) { char* e = NULL; double t = strtod(s, &e);
|
||||
if (e != s && t >= 0.0 && t <= 1.0) d = t; }
|
||||
v = d; return v;
|
||||
}
|
||||
|
||||
#ifdef HAVE_CURL
|
||||
|
||||
typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry;
|
||||
static EngramEmbEntry* g_emb_items = NULL;
|
||||
static int64_t g_emb_count = 0, g_emb_cap = 0;
|
||||
static int g_emb_state = 0; /* 0=unprobed, 1=available, -1=disabled */
|
||||
|
||||
static uint64_t engram_fnv1a(const char* s) {
|
||||
uint64_t h = 1469598103934665603ULL;
|
||||
if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||
h ^= *p; h *= 1099511628211ULL;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */
|
||||
static float* engram_parse_embedding(const char* json, int* out_dim) {
|
||||
if (!json) return NULL;
|
||||
const char* p = strstr(json, "\"embedding\"");
|
||||
if (!p) return NULL;
|
||||
p = strchr(p, '[');
|
||||
if (!p) return NULL;
|
||||
p++;
|
||||
int cap = 1024, n = 0;
|
||||
float* v = malloc((size_t)cap * sizeof(float));
|
||||
if (!v) return NULL;
|
||||
while (*p && *p != ']') {
|
||||
while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++;
|
||||
if (*p == ']' || !*p) break;
|
||||
char* e = NULL;
|
||||
double d = strtod(p, &e);
|
||||
if (e == p) break;
|
||||
if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float));
|
||||
if (!nv) { free(v); return NULL; } v = nv; }
|
||||
v[n++] = (float)d;
|
||||
p = e;
|
||||
}
|
||||
if (n == 0) { free(v); return NULL; }
|
||||
*out_dim = n;
|
||||
return v;
|
||||
}
|
||||
|
||||
/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */
|
||||
static char* engram_json_escape(const char* src) {
|
||||
if (!src) src = "";
|
||||
size_t n = strlen(src);
|
||||
char* out = malloc(n * 2 + 1);
|
||||
if (!out) return NULL;
|
||||
size_t j = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
unsigned char c = (unsigned char)src[i];
|
||||
if (c == '"') { out[j++] = '\\'; out[j++] = '"'; }
|
||||
else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; }
|
||||
else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; }
|
||||
else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; }
|
||||
else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; }
|
||||
else if (c < 0x20) { /* drop other control bytes */ }
|
||||
else { out[j++] = (char)c; }
|
||||
}
|
||||
out[j] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */
|
||||
static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) {
|
||||
if (!text) return NULL;
|
||||
const char* url = getenv("ENGRAM_EMBED_URL");
|
||||
if (!url || !*url) url = "http://localhost:11434/api/embeddings";
|
||||
const char* model = getenv("ENGRAM_EMBED_MODEL");
|
||||
if (!model || !*model) model = "nomic-embed-text";
|
||||
/* Bound content length to keep latency/memory sane on huge nodes. */
|
||||
char* trunc = NULL;
|
||||
size_t maxlen = 8192;
|
||||
if (strlen(text) > maxlen) {
|
||||
trunc = malloc(maxlen + 1);
|
||||
if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; }
|
||||
}
|
||||
char* esc_prefix = engram_json_escape(prefix ? prefix : "");
|
||||
char* esc = engram_json_escape(text);
|
||||
free(trunc);
|
||||
if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; }
|
||||
size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64;
|
||||
char* body = malloc(blen);
|
||||
if (!body) { free(esc); free(esc_prefix); return NULL; }
|
||||
snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc);
|
||||
free(esc); free(esc_prefix);
|
||||
|
||||
CURL* c = curl_easy_init();
|
||||
if (!c) { free(body); return NULL; }
|
||||
HttpBuf rb; httpbuf_init(&rb);
|
||||
struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json");
|
||||
char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
|
||||
curl_easy_setopt(c, CURLOPT_URL, url);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
|
||||
curl_easy_setopt(c, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_POSTFIELDS, body);
|
||||
curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body));
|
||||
curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
|
||||
curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
|
||||
curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
|
||||
CURLcode rc = curl_easy_perform(c);
|
||||
curl_slist_free_all(h);
|
||||
curl_easy_cleanup(c);
|
||||
free(body);
|
||||
if (rc != CURLE_OK) { free(rb.data); return NULL; }
|
||||
float* v = engram_parse_embedding(rb.data, out_dim);
|
||||
free(rb.data);
|
||||
return v;
|
||||
}
|
||||
|
||||
/* One-shot probe: is semantic search available? Caches the verdict. */
|
||||
static int engram_semantic_enabled(void) {
|
||||
if (g_emb_state != 0) return g_emb_state == 1;
|
||||
const char* s = getenv("ENGRAM_SEMANTIC");
|
||||
if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; }
|
||||
int dim = 0;
|
||||
float* v = engram_embed_raw("search_query: ", "probe", &dim);
|
||||
if (v && dim > 0) { free(v); g_emb_state = 1; return 1; }
|
||||
free(v);
|
||||
g_emb_state = -1; return 0;
|
||||
}
|
||||
|
||||
/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */
|
||||
static float* engram_embed_query(const char* q, int* dim) {
|
||||
if (!engram_semantic_enabled()) return NULL;
|
||||
if (!q || !*q) return NULL;
|
||||
return engram_embed_raw("search_query: ", q, dim);
|
||||
}
|
||||
|
||||
/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */
|
||||
static const float* engram_node_vec(EngramNode* n, int* out_dim) {
|
||||
if (!n || !n->id) return NULL;
|
||||
uint64_t h = engram_fnv1a(n->content);
|
||||
for (int64_t i = 0; i < g_emb_count; i++) {
|
||||
if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) {
|
||||
if (g_emb_items[i].hash == h && g_emb_items[i].vec) {
|
||||
*out_dim = g_emb_items[i].dim; return g_emb_items[i].vec;
|
||||
}
|
||||
/* content changed → re-embed in place */
|
||||
int dim = 0;
|
||||
float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
|
||||
if (!v) return NULL;
|
||||
free(g_emb_items[i].vec);
|
||||
g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h;
|
||||
*out_dim = dim; return v;
|
||||
}
|
||||
}
|
||||
int dim = 0;
|
||||
float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
|
||||
if (!v) return NULL;
|
||||
if (g_emb_count >= g_emb_cap) {
|
||||
int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256;
|
||||
EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry));
|
||||
if (!ni) { free(v); return NULL; }
|
||||
g_emb_items = ni; g_emb_cap = nc;
|
||||
}
|
||||
g_emb_items[g_emb_count].id = strdup(n->id);
|
||||
g_emb_items[g_emb_count].hash = h;
|
||||
g_emb_items[g_emb_count].vec = v;
|
||||
g_emb_items[g_emb_count].dim = dim;
|
||||
g_emb_count++;
|
||||
*out_dim = dim; return v;
|
||||
}
|
||||
|
||||
static double engram_cosine(const float* a, const float* b, int dim) {
|
||||
double dot = 0, na = 0, nb = 0;
|
||||
for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i];
|
||||
na += (double)a[i] * a[i];
|
||||
nb += (double)b[i] * b[i]; }
|
||||
if (na <= 0 || nb <= 0) return 0.0;
|
||||
return dot / (sqrt(na) * sqrt(nb));
|
||||
}
|
||||
|
||||
/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */
|
||||
static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
|
||||
if (!qvec || qdim <= 0) return 0.0;
|
||||
int ndim = 0;
|
||||
const float* nv = engram_node_vec(n, &ndim);
|
||||
if (!nv || ndim != qdim) return 0.0;
|
||||
return engram_cosine(qvec, nv, qdim);
|
||||
}
|
||||
|
||||
#else /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical.
|
||||
* Only the two boundary functions the always-compiled search/activate
|
||||
* code calls are stubbed; the query embed always yields NULL so every
|
||||
* cosine is 0 and every caller collapses to lexical-only. */
|
||||
static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; }
|
||||
static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
|
||||
(void)n; (void)qvec; (void)qdim; return 0.0;
|
||||
}
|
||||
#endif /* HAVE_CURL */
|
||||
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
@@ -6833,21 +7176,45 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
if (lim <= 0) lim = 100;
|
||||
el_val_t lst = el_list_empty();
|
||||
if (!q || !*q) return lst;
|
||||
int64_t found = 0;
|
||||
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||
if (ntok == 0) return lst;
|
||||
/* Semantic augmentation: embed the query once; a node is a hit if it covers
|
||||
* >=1 query token (tokenized-lexical, #66) OR its cosine clears the
|
||||
* threshold (#67). qvec is NULL (cosine 0) when semantic is unavailable →
|
||||
* pure tokenized-lexical, byte-identical to the lexical-only behaviour. */
|
||||
int qdim = 0;
|
||||
float* qvec = engram_embed_query(q, &qdim);
|
||||
double sem_min = engram_semantic_min();
|
||||
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||
if (!hits) { free(qvec); return lst; }
|
||||
int64_t nhits = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
||||
* shape output but are invisible to introspection ("what do you
|
||||
* know about yourself"). They still surface via engram_activate
|
||||
* + engram_compile_layered_json — that's the legitimate path. */
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
if (istr_contains(n->content, q) ||
|
||||
istr_contains(n->label, q) ||
|
||||
istr_contains(n->tags, q)) {
|
||||
lst = el_list_append(lst, engram_node_to_map(n));
|
||||
found++;
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
||||
if (sc > 0 || sem >= sem_min) {
|
||||
hits[nhits].idx = i;
|
||||
hits[nhits].score = sc;
|
||||
hits[nhits].salience = n->salience;
|
||||
nhits++;
|
||||
}
|
||||
}
|
||||
/* Rank by distinct tokens matched (desc) then salience (desc), then cap.
|
||||
* Pure-semantic hits (token score 0) sort after every lexical hit — a
|
||||
* lexical ∪ semantic union with lexical precedence. */
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
||||
}
|
||||
free(hits);
|
||||
free(qvec);
|
||||
return lst;
|
||||
}
|
||||
|
||||
@@ -7124,10 +7491,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
|
||||
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
||||
if (!query || !*query) return 1.0;
|
||||
double bias = 1.0;
|
||||
/* Direct lexical overlap: node content/label/tags share text with query. */
|
||||
if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
|
||||
istr_contains(n->tags, query)) {
|
||||
bias += 0.5;
|
||||
/* Direct lexical overlap, graded by token coverage: a node covering all
|
||||
* query tokens gets the full +0.5; partial coverage gets a proportional
|
||||
* share. Single-token queries → full +0.5 on match, identical to before. */
|
||||
{
|
||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||
int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
|
||||
}
|
||||
/* Node-type resonance with query intent. */
|
||||
int technical_query = istr_contains(query, "code") ||
|
||||
@@ -7193,14 +7564,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
||||
if (!seeds) {
|
||||
free(best_bg); free(best_hops); free(reached); return out;
|
||||
}
|
||||
/* Tokenized + semantic seeding: a node seeds if it covers >=1 query token
|
||||
* (tokenized-lexical, #66) OR its cosine clears the threshold (#67). A
|
||||
* lexical seed's activation is scaled by token coverage (fraction of
|
||||
* distinct query tokens covered) so a node matching all words seeds more
|
||||
* strongly than one matching a single word; single-word queries → coverage
|
||||
* 1.0. A pure-semantic seed (no token match) is instead down-weighted by
|
||||
* its cosine so paraphrase matches spread without overpowering exact seeds.
|
||||
* q_vec is NULL (cosine 0) when semantic is unavailable → the seed set is
|
||||
* exactly the tokenized-lexical one. q_vec is freed right after this loop
|
||||
* so the many downstream early-returns need no cleanup change. */
|
||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||
int q_dim = 0;
|
||||
float* q_vec = engram_embed_query(q, &q_dim);
|
||||
double q_sem_min = engram_semantic_min();
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
if (istr_contains(n->content, q) ||
|
||||
istr_contains(n->label, q) ||
|
||||
istr_contains(n->tags, q)) {
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
|
||||
if (sc > 0 || sem >= q_sem_min) {
|
||||
double tdecay = engram_temporal_decay(n, now_ms);
|
||||
double dampen = engram_activation_dampen(n);
|
||||
double act = n->salience * tdecay * dampen;
|
||||
if (sc > 0) act *= (ntok > 0 ? (double)sc / (double)ntok : 1.0);
|
||||
else act *= sem; /* pure-semantic seed: down-weight by cosine */
|
||||
seeds[seed_count].idx = i;
|
||||
seeds[seed_count].act = act;
|
||||
seeds[seed_count].created_at = n->created_at;
|
||||
@@ -7210,6 +7598,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
||||
reached[i] = 1;
|
||||
}
|
||||
}
|
||||
free(q_vec);
|
||||
/* Compute mean seed created_at for temporal proximity bonus. */
|
||||
int64_t seed_epoch = 0;
|
||||
if (seed_count > 0) {
|
||||
@@ -7761,6 +8150,35 @@ el_val_t engram_get_node_json(el_val_t id) {
|
||||
return el_wrap_str(jb_finish(&b));
|
||||
}
|
||||
|
||||
/* engram_get_node_by_label — find the first node whose label field exactly
|
||||
* matches the given string. Returns the node as a JSON object string, or "{}"
|
||||
* if no match is found.
|
||||
*
|
||||
* Used by chat.el to retrieve well-known nodes (e.g. "conv:history",
|
||||
* "session:summary") by their stable label rather than by ID, which is immune
|
||||
* to vector index drift across restarts.
|
||||
*
|
||||
* Exact match (strcmp, not istr_contains) because labels like "conv:history"
|
||||
* must not collide with nodes whose content happens to contain that substring.
|
||||
*
|
||||
* Backported verbatim (idiom-adapted to jb_finish) from release runtime
|
||||
* v1.0.0-20260501 to unblock the soul regen link: chat.el references this
|
||||
* native but the current runtime lacked its definition. */
|
||||
el_val_t engram_get_node_by_label(el_val_t label) {
|
||||
const char* lbl = EL_CSTR(label);
|
||||
if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}"));
|
||||
EngramStore* g = engram_get();
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
if (n->label && strcmp(n->label, lbl) == 0) {
|
||||
JsonBuf b; jb_init(&b);
|
||||
engram_emit_node_json(&b, n);
|
||||
return el_wrap_str(jb_finish(&b));
|
||||
}
|
||||
}
|
||||
return el_wrap_str(el_strdup("{}"));
|
||||
}
|
||||
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
@@ -7768,22 +8186,50 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
if (lim <= 0) lim = 100;
|
||||
JsonBuf b; jb_init(&b);
|
||||
jb_putc(&b, '[');
|
||||
int first = 1;
|
||||
int64_t found = 0;
|
||||
if (q && *q) {
|
||||
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
/* Filter transparent layers — same as engram_search. */
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
if (istr_contains(n->content, q) ||
|
||||
istr_contains(n->label, q) ||
|
||||
istr_contains(n->tags, q)) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, n);
|
||||
first = 0;
|
||||
found++;
|
||||
if (q && *q && g->node_count > 0) {
|
||||
/* Collect candidates from the UNION of tokenized-lexical and semantic
|
||||
* matches, score each, rank by score, emit the top `lim`. A node is a
|
||||
* candidate if it covers >=1 query token (tokenized-lexical, #66) OR its
|
||||
* query cosine clears the threshold (#67). Lexical score is the distinct
|
||||
* token count (>=1), so any lexical hit outranks a pure-semantic hit
|
||||
* (cosine < 1); pure-semantic hits are scored by cosine alone. When
|
||||
* semantic is unavailable qvec is NULL, sem is 0, only tokenized-lexical
|
||||
* hits are collected, and the stable insertion sort preserves order. */
|
||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||
int qdim = 0;
|
||||
float* qvec = engram_embed_query(q, &qdim);
|
||||
double sem_min = engram_semantic_min();
|
||||
typedef struct { int64_t idx; double score; } Cand;
|
||||
Cand* cand = malloc((size_t)g->node_count * sizeof(Cand));
|
||||
if (cand) {
|
||||
int64_t nc = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
||||
if (sc > 0 || sem >= sem_min) {
|
||||
cand[nc].idx = i;
|
||||
cand[nc].score = (double)sc + sem;
|
||||
nc++;
|
||||
}
|
||||
}
|
||||
/* Insertion sort by score desc; stable for equal scores. */
|
||||
for (int64_t i = 1; i < nc; i++) {
|
||||
Cand k = cand[i]; int64_t j = i - 1;
|
||||
while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; }
|
||||
cand[j + 1] = k;
|
||||
}
|
||||
int first = 1;
|
||||
for (int64_t i = 0; i < nc && i < lim; i++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[cand[i].idx]);
|
||||
first = 0;
|
||||
}
|
||||
free(cand);
|
||||
}
|
||||
free(qvec);
|
||||
}
|
||||
jb_putc(&b, ']');
|
||||
return el_wrap_str(jb_finish(&b));
|
||||
|
||||
@@ -632,6 +632,7 @@ el_val_t engram_load(el_val_t path);
|
||||
* can pass results straight through without round-tripping ElList/ElMap
|
||||
* through json_stringify. */
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
|
||||
@@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path) { return engram_save
|
||||
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
||||
|
||||
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
||||
el_val_t __engram_get_node_by_label(el_val_t label) { return engram_get_node_by_label(label); }
|
||||
|
||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
||||
return engram_search_json(query, limit);
|
||||
|
||||
@@ -226,6 +226,7 @@ el_val_t __engram_activate(el_val_t query, el_val_t depth);
|
||||
el_val_t __engram_save(el_val_t path);
|
||||
el_val_t __engram_load(el_val_t path);
|
||||
el_val_t __engram_get_node_json(el_val_t id);
|
||||
el_val_t __engram_get_node_by_label(el_val_t label);
|
||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
|
||||
@@ -2670,6 +2670,7 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "engram_save") { return 1 }
|
||||
if str_eq(name, "engram_load") { return 1 }
|
||||
if str_eq(name, "engram_get_node_json") { return 1 }
|
||||
if str_eq(name, "engram_get_node_by_label") { return 1 }
|
||||
if str_eq(name, "engram_search_json") { return 2 }
|
||||
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
||||
if str_eq(name, "engram_neighbors_json") { return 3 }
|
||||
|
||||
@@ -23,10 +23,29 @@ fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
|
||||
}
|
||||
|
||||
fn tok_kind(tokens: [Any], pos: Int) -> String {
|
||||
// Out-of-range reads must report the Eof sentinel so every `== "Eof"`
|
||||
// termination guard in the parser fires. Without this, reading past the
|
||||
// single trailing Eof token returns runtime null (el_list_get OOB -> 0),
|
||||
// which matches no delimiter, letting inner parse loops append AST nodes
|
||||
// forever on malformed input -> unbounded allocation -> OOM.
|
||||
let n: Int = native_list_len(tokens) / 2
|
||||
if pos < 0 {
|
||||
return "Eof"
|
||||
}
|
||||
if pos >= n {
|
||||
return "Eof"
|
||||
}
|
||||
native_list_get(tokens, pos * 2)
|
||||
}
|
||||
|
||||
fn tok_value(tokens: [Any], pos: Int) -> String {
|
||||
let n: Int = native_list_len(tokens) / 2
|
||||
if pos < 0 {
|
||||
return ""
|
||||
}
|
||||
if pos >= n {
|
||||
return ""
|
||||
}
|
||||
native_list_get(tokens, pos * 2 + 1)
|
||||
}
|
||||
|
||||
@@ -35,7 +54,12 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
|
||||
if k == kind {
|
||||
return pos + 1
|
||||
}
|
||||
// On mismatch just advance; error recovery is best-effort
|
||||
// On mismatch, error recovery is best-effort. But never step PAST the Eof
|
||||
// sentinel: once at Eof a mismatch means the input ended early, and
|
||||
// advancing would run the cursor off the token list.
|
||||
if k == "Eof" {
|
||||
return pos
|
||||
}
|
||||
pos + 1
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
#ifndef EL_PLATFORM_WIN_H
|
||||
#define EL_PLATFORM_WIN_H
|
||||
/*
|
||||
* el_platform_win.h — Windows OS-boundary shim for el_runtime.c.
|
||||
*
|
||||
* Branch: feat/windows-el-runtime. Included ONLY when _WIN32 is defined; the POSIX build is
|
||||
* untouched. Goal: let el_runtime.c (a BSD-sockets / dlfcn / fork host) compile and link with
|
||||
* mingw-w64 into a native neuron.exe, with no behavioural change to the Linux/macOS build.
|
||||
*
|
||||
* What it maps:
|
||||
* - sockets : winsock2 (same call names: socket/bind/listen/accept/recv/send/setsockopt).
|
||||
* Sockets close with closesocket() (see el_closesocket), and the stack must be
|
||||
* started once with WSAStartup — done automatically via a load-time constructor.
|
||||
* - dlsym : el_runtime.c uses dlsym(RTLD_DEFAULT, name) to resolve callback/tool symbols
|
||||
* exported by the main module. Windows equivalent: GetProcAddress on the process
|
||||
* module. Link the soul with -Wl,--export-all-symbols so the symbols are findable.
|
||||
* - popen : mapped to _popen/_pclose.
|
||||
* - threads : UNCHANGED. mingw-w64 ships winpthreads, so <pthread.h> + -lpthread just work.
|
||||
*/
|
||||
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <winsock2.h>
|
||||
#include <ws2tcpip.h>
|
||||
#include <windows.h>
|
||||
#include <io.h>
|
||||
#include <process.h>
|
||||
|
||||
/* Portable headers mingw-w64 provides (verified present). */
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <strings.h> /* strcasecmp */
|
||||
#include <ctype.h>
|
||||
#include <math.h>
|
||||
#include <time.h>
|
||||
#include <sys/time.h> /* mingw-w64 provides gettimeofday here */
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
|
||||
/* ── socket close ─────────────────────────────────────────────────────────── */
|
||||
/* Winsock closes sockets with closesocket(), not close() (close() is for file fds). The POSIX
|
||||
build defines the same helper as close() so the call sites are identical across platforms. */
|
||||
static inline int el_closesocket(SOCKET s) { return closesocket(s); }
|
||||
|
||||
/* ── setsockopt optval type ───────────────────────────────────────────────── */
|
||||
/* Winsock's setsockopt takes optval as (const char*); POSIX takes (const void*), so el_runtime.c
|
||||
passes &int directly. GCC 14+ makes that an error under -Wincompatible-pointer-types. Wrap it so
|
||||
the runtime's POSIX-style call sites compile unchanged (defined before the macro so the wrapper
|
||||
itself resolves to the real winsock setsockopt). */
|
||||
static inline int el_setsockopt(SOCKET s, int level, int optname, const void* optval, int optlen) {
|
||||
return setsockopt(s, level, optname, (const char*)optval, optlen);
|
||||
}
|
||||
#define setsockopt(s, l, o, v, n) el_setsockopt((s), (l), (o), (v), (int)(n))
|
||||
|
||||
/* ── winsock init (once, at load) ─────────────────────────────────────────── */
|
||||
static void el__win_net_init(void) {
|
||||
static int inited = 0;
|
||||
if (!inited) { WSADATA w; WSAStartup(MAKEWORD(2, 2), &w); inited = 1; }
|
||||
}
|
||||
__attribute__((constructor)) static void el__win_ctor(void) { el__win_net_init(); }
|
||||
|
||||
/* ── dlsym → GetProcAddress ───────────────────────────────────────────────── */
|
||||
#ifndef RTLD_DEFAULT
|
||||
#define RTLD_DEFAULT ((void*)0)
|
||||
#endif
|
||||
static inline void* el_win_dlsym(void* handle, const char* name) {
|
||||
(void)handle;
|
||||
return (void*)(uintptr_t)GetProcAddress(GetModuleHandleA(NULL), name);
|
||||
}
|
||||
#define dlsym(h, n) el_win_dlsym((h), (n))
|
||||
|
||||
/* ── popen / pclose ───────────────────────────────────────────────────────── */
|
||||
#define popen _popen
|
||||
#define pclose _pclose
|
||||
|
||||
/* ── misc POSIX → Win32 shims ─────────────────────────────────────────────── */
|
||||
#include <direct.h> /* _mkdir */
|
||||
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
||||
#define timegm _mkgmtime /* UTC tm → time_t */
|
||||
|
||||
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
||||
static inline int setenv(const char* name, const char* value, int overwrite) {
|
||||
(void)overwrite;
|
||||
return _putenv_s(name, value ? value : "");
|
||||
}
|
||||
static inline int unsetenv(const char* name) {
|
||||
/* _putenv_s(name, "") sets VAR="" rather than removing it.
|
||||
* SetEnvironmentVariableA(name, NULL) truly deletes it from the Win32
|
||||
* env block; then we sync the CRT cache with _putenv("NAME="). */
|
||||
SetEnvironmentVariableA(name, NULL);
|
||||
size_t len = strlen(name);
|
||||
char *buf = (char*)malloc(len + 2);
|
||||
if (!buf) return -1;
|
||||
memcpy(buf, name, len);
|
||||
buf[len] = '=';
|
||||
buf[len + 1] = '\0';
|
||||
_putenv(buf);
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* nanosleep — not available in MSVC/UCRT; approximate with Sleep(). */
|
||||
static inline int el_nanosleep(const struct timespec *req, struct timespec *rem) {
|
||||
(void)rem;
|
||||
DWORD ms = (DWORD)((req->tv_sec * 1000ULL) + (req->tv_nsec / 1000000ULL));
|
||||
Sleep(ms ? ms : 1);
|
||||
return 0;
|
||||
}
|
||||
#define nanosleep(req, rem) el_nanosleep((req), (rem))
|
||||
|
||||
/* localtime_r/gmtime_r: Windows offers localtime_s/gmtime_s with reversed arg order. */
|
||||
static inline struct tm* localtime_r(const time_t* t, struct tm* out) {
|
||||
return localtime_s(out, t) == 0 ? out : (struct tm*)0;
|
||||
}
|
||||
static inline struct tm* gmtime_r(const time_t* t, struct tm* out) {
|
||||
return gmtime_s(out, t) == 0 ? out : (struct tm*)0;
|
||||
}
|
||||
|
||||
/* ── libcurl: degradable stubs for the curl-less Windows build ─────────────── */
|
||||
/* The curl-less validation build (WITH_CURL=0) links no libcurl. el_runtime.c uses libcurl
|
||||
* unconditionally for its HTTP client / LLM layer; these stubs let it compile and link so the
|
||||
* runtime, HTTP *server*, graph and memory work natively on Windows. Live outbound HTTP/LLM calls
|
||||
* degrade to a runtime error (curl_easy_perform returns an error) — matching the documented
|
||||
* curl-less contract. When HAVE_CURL is defined (WITH_CURL=1) the real <curl/curl.h> is used and
|
||||
* this whole block is compiled out. POSIX never sees this header, so the POSIX build is untouched. */
|
||||
#ifndef HAVE_CURL
|
||||
|
||||
typedef void CURL;
|
||||
typedef int CURLcode;
|
||||
|
||||
#define CURLE_OK 0
|
||||
#define CURLE_HTTP_RETURNED_ERROR 22
|
||||
#define CURL_ERROR_SIZE 256
|
||||
|
||||
/* Option ids: values are irrelevant to the no-op setopt below; kept distinct for readability. */
|
||||
#define CURLOPT_URL 10002
|
||||
#define CURLOPT_WRITEFUNCTION 20011
|
||||
#define CURLOPT_WRITEDATA 10001
|
||||
#define CURLOPT_POSTFIELDS 10015
|
||||
#define CURLOPT_POSTFIELDSIZE 120
|
||||
#define CURLOPT_POST 47
|
||||
#define CURLOPT_HTTPHEADER 10023
|
||||
#define CURLOPT_TIMEOUT_MS 155
|
||||
#define CURLOPT_NOSIGNAL 99
|
||||
#define CURLOPT_USERAGENT 10018
|
||||
#define CURLOPT_FOLLOWLOCATION 52
|
||||
#define CURLOPT_ERRORBUFFER 10010
|
||||
#define CURLOPT_CUSTOMREQUEST 10036
|
||||
#define CURLOPT_FAILONERROR 45
|
||||
|
||||
struct curl_slist { char* data; struct curl_slist* next; };
|
||||
|
||||
static inline struct curl_slist* curl_slist_append(struct curl_slist* list, const char* s) {
|
||||
struct curl_slist* node = (struct curl_slist*)malloc(sizeof(struct curl_slist));
|
||||
if (!node) return list;
|
||||
node->data = s ? strdup(s) : NULL;
|
||||
node->next = NULL;
|
||||
if (!list) return node;
|
||||
struct curl_slist* p = list;
|
||||
while (p->next) p = p->next;
|
||||
p->next = node;
|
||||
return list;
|
||||
}
|
||||
static inline void curl_slist_free_all(struct curl_slist* list) {
|
||||
while (list) { struct curl_slist* n = list->next; free(list->data); free(list); list = n; }
|
||||
}
|
||||
|
||||
static inline CURL* curl_easy_init(void) { return (CURL*)malloc(1); }
|
||||
static inline CURLcode curl_easy_setopt(CURL* h, int opt, ...) { (void)h; (void)opt; return CURLE_OK; }
|
||||
static inline CURLcode curl_easy_perform(CURL* h) { (void)h; return 7 /* CURLE_COULDNT_CONNECT */; }
|
||||
static inline void curl_easy_cleanup(CURL* h) { free(h); }
|
||||
static inline const char* curl_easy_strerror(CURLcode c) {
|
||||
(void)c; return "libcurl not built in (curl-less build)";
|
||||
}
|
||||
|
||||
#endif /* !HAVE_CURL */
|
||||
|
||||
#endif /* EL_PLATFORM_WIN_H */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -758,6 +758,18 @@ el_val_t trace_span_start(el_val_t name);
|
||||
el_val_t trace_span_end(el_val_t span_handle);
|
||||
el_val_t emit_event(el_val_t name, el_val_t duration_ms);
|
||||
|
||||
/* ── Runtime symbols required by the soul modules ──────────────────────────── */
|
||||
/* All implemented in el_runtime.c but omitted from this release header; the soul dist modules
|
||||
* reference them directly, so the public header must export them. Declarations only — mirrors the
|
||||
* mainline el_runtime.h and is platform-independent (no behavioural change to the POSIX build). */
|
||||
typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body);
|
||||
typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, el_val_t body, el_val_t headers);
|
||||
el_val_t el_arena_push(void);
|
||||
el_val_t el_arena_pop(el_val_t mark);
|
||||
void http_serve_async(el_val_t port, el_val_t handler);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_prune_telemetry(el_val_t older_than_ms);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# sandbox — the Neuron STACK sandbox
|
||||
|
||||
**Work on a whole stack at once, not one repo at a time.** `sandbox` assembles every
|
||||
constituent repo of a named stack into **one combined worktree workspace**, wired so
|
||||
they build and run **together**, on an isolated clean base — then tears it all down
|
||||
cleanly. The live soul/engram (`:7770` / `:8742`) are never touched.
|
||||
|
||||
It is the multi-repo sibling of [`nsbx`](./README.md): where `nsbx dev` stands up
|
||||
**one** repo's worktree + an isolated engram, `sandbox` stands up **every** repo of a
|
||||
stack as sibling git worktrees under a single workspace.
|
||||
|
||||
```bash
|
||||
export PATH="$PWD:$PATH" # or symlink `sandbox` onto your PATH
|
||||
|
||||
sandbox neuron-stack tim # el + neuron soul + NeuronUI, assembled together
|
||||
cd ~/Development/neuron-technologies/stack-worktrees/neuron-stack-tim
|
||||
source .stack-env # EL_REPO + PATH now point at the SANDBOX el
|
||||
./build.sh # engram compiles, soul compiles, UI present & buildable
|
||||
sandbox down neuron-stack tim # remove every worktree; live untouched
|
||||
```
|
||||
|
||||
## The two profiles
|
||||
|
||||
### `el-stack` — the whole EL kit
|
||||
The compiler + language + framework + tooling are **all one repo** (`foundation/el`:
|
||||
`lang/` = elc/elb + runtime, `engram/src/server.el`, `elp/` = NLG, `ui/` = the **el-ui
|
||||
framework**, plus `ql`, `ide`, `epm`, `arbor`, `tools`). Its downstream SDK consumers
|
||||
come along so a change to `elc` can be proven end-to-end across the kit.
|
||||
|
||||
| repo | required | role |
|
||||
|------|----------|------|
|
||||
| `foundation/el` | ✓ | elc + elb compiler, el_runtime, engram source, **el-ui framework**, elp/ql/ide/epm tooling |
|
||||
| `engram-language` | | language-faculty reference POC (Python) — being ported into `el/elp` |
|
||||
| `foundation/forge` | | downstream SDK consumer — `make build` |
|
||||
| `foundation/dharma` | | downstream SDK consumer — CGI provenance registry |
|
||||
|
||||
`build.sh` proves it: `elc` compiles a real stack source and `cc` links it against the
|
||||
runtime into a native binary (elc + runtime build together), and — if present — `forge`
|
||||
builds against the freshly-assembled SDK.
|
||||
|
||||
### `neuron-stack` — the full product
|
||||
Substrate + soul + UI. **Engram is not a separate repo** — its source lives inside
|
||||
`foundation/el`.
|
||||
|
||||
| repo | required | role |
|
||||
|------|----------|------|
|
||||
| `foundation/el` | ✓ | substrate: elc + el_runtime + engram source + the `elp` NLG the soul imports |
|
||||
| `neuron` | ✓ | the soul (`:7770`) + engram build; `soul.el` imports `../foundation/el/elp/src/elp.el` |
|
||||
| `products/NeuronUI` | ✓ | the app/UI (Kotlin/Compose desktop client; bundles the soul binary) |
|
||||
| `products/web` | | marketing site + interactive soul-demo |
|
||||
|
||||
`build.sh` proves it: **engram** builds (`elc engram/src/server.el` → `cc … el_runtime.c`
|
||||
→ native binary), the **soul** compiles with its cross-repo `../foundation/el` import
|
||||
resolving to the *sandbox* el, and the **UI** is present with its build entry.
|
||||
|
||||
## Why it works — mirrored-layout wiring
|
||||
|
||||
The repos reference each other by **relative sibling paths** (e.g. the soul imports
|
||||
`../foundation/el/elp/src/elp.el`). So `sandbox` lays every worktree out at its **natural
|
||||
relative path** inside the workspace:
|
||||
|
||||
```
|
||||
stack-worktrees/neuron-stack-tim/
|
||||
├── foundation/el/ ← worktree of foundation/el (the sandbox el)
|
||||
├── neuron/ ← worktree of neuron
|
||||
└── products/NeuronUI/ ← worktree of products/NeuronUI
|
||||
```
|
||||
|
||||
From `neuron/`, `../foundation/el` resolves to `…/neuron-stack-tim/foundation/el` — the
|
||||
**sandbox** copy, never the live tree. No symlinks, no path rewriting: the layout *is*
|
||||
the wiring. `.stack-env` additionally pins `EL_REPO` and prepends the sandbox `elc`/`elb`
|
||||
to `PATH`.
|
||||
|
||||
## Commands
|
||||
|
||||
| command | does |
|
||||
|---------|------|
|
||||
| `sandbox el-stack <name> [--minimal]` | assemble the EL kit (`--minimal` = required repos only) |
|
||||
| `sandbox neuron-stack <name> [--minimal]` | assemble the full product |
|
||||
| `sandbox build <profile> <name>` | run the workspace's combined `build.sh` |
|
||||
| `sandbox status <profile> <name>` | per-repo head + clean/dirty |
|
||||
| `sandbox list` | list assembled workspaces |
|
||||
| `sandbox down <profile> <name> [--delete-branch]` | remove every worktree + drop the workspace (branch kept unless `--delete-branch`) |
|
||||
|
||||
Flags: `--minimal` (required repos only), `--branch B` (branch name; default
|
||||
`sandbox/<profile>-<name>`), `--base REF` (fork point; default each repo's committed
|
||||
HEAD).
|
||||
|
||||
## Rails (always)
|
||||
|
||||
- **Clean base** — worktrees fork off each repo's **committed HEAD**; the dirty state of
|
||||
the live checkout is deliberately *not* carried in.
|
||||
- **Persistent** — the workspace lives under `NSBX_STACK_ROOT` (default
|
||||
`~/Development/neuron-technologies/stack-worktrees`), **never `/tmp`** (ablated on
|
||||
compaction).
|
||||
- **Never touches live** — `sandbox` only does `git worktree` + offline `cc`. It never
|
||||
binds `:8742`/`:7770`, never `launchctl`, never `pkill`. Bringing up an **isolated
|
||||
engram** is delegated, opt-in, to `nsbx` (which guards the live store and refuses the
|
||||
live ports).
|
||||
- **Idempotent & safe** — refuses to clobber an existing workspace; a failed assembly
|
||||
rolls back its partial worktrees; teardown removes worktrees through their origin repo
|
||||
and prunes.
|
||||
- **Own-the-core** — pure bash + `git worktree`. No new dependencies.
|
||||
|
||||
## Env knobs
|
||||
|
||||
`NSBX_STACK_ROOT` (workspace root), `NEURON_DEV_ROOT` (the dir holding all the peer
|
||||
repos, default `~/Development/neuron-technologies`).
|
||||
|
||||
## Isolated engram for `neuron-stack` (opt-in)
|
||||
|
||||
`sandbox` gets the code building together; to run the soul against an **isolated** engram
|
||||
(never live), delegate to `nsbx` from inside the workspace:
|
||||
|
||||
```bash
|
||||
source .stack-env
|
||||
nsbx create $STACK_NAME --source "$EL_REPO" # clone live store onto a non-default port
|
||||
nsbx up $STACK_NAME
|
||||
nsbx status $STACK_NAME # prints the isolated engram URL
|
||||
```
|
||||
Executable
+505
@@ -0,0 +1,505 @@
|
||||
#!/usr/bin/env bash
|
||||
# sandbox — the Neuron STACK sandbox: assemble a WHOLE stack of repos into ONE
|
||||
# combined worktree workspace, wired so they build/run TOGETHER, on an isolated
|
||||
# clean base — so an agent (or Will) can work on the full stack at once instead of
|
||||
# one repo at a time.
|
||||
#
|
||||
# It is the multi-repo generalisation of `nsbx` (this same directory): where
|
||||
# `nsbx dev` stands up ONE repo's worktree + an isolated engram, `sandbox` stands
|
||||
# up EVERY constituent repo of a named stack as sibling git worktrees under a
|
||||
# single workspace, mirroring their on-disk relative layout so the cross-repo
|
||||
# `../foundation/el` imports resolve to the SANDBOX copy — never the live tree.
|
||||
#
|
||||
# sandbox el-stack <name> # elc compiler + EL language + framework + tooling (+ consumers)
|
||||
# sandbox neuron-stack <name> # runtime/soul + engram + app/UI (the full product)
|
||||
# sandbox list # list assembled stack workspaces
|
||||
# sandbox status <profile> <name> # inspect one
|
||||
# sandbox build <profile> <name> # run the workspace's combined build.sh
|
||||
# sandbox down <profile> <name> # tear down: remove every worktree, drop the workspace
|
||||
#
|
||||
# RAILS (always):
|
||||
# * worktrees fork off each repo's COMMITTED HEAD -> a clean, reproducible base
|
||||
# (the dirty state of the live checkout is deliberately NOT carried in).
|
||||
# * the workspace lives at a PERSISTENT path (never /tmp — ablated on compaction).
|
||||
# * NEVER touches the live soul/engram (:7770 / :8742). It only creates git
|
||||
# worktrees + a build script; bringing up an isolated engram is delegated,
|
||||
# opt-in, to `nsbx` (which already guards the live store & ports).
|
||||
# * idempotent & safe: refuses to clobber an existing workspace; teardown removes
|
||||
# worktrees through their origin repo and prunes — branches are kept by default.
|
||||
# * own-the-core: pure bash + git worktree. No new dependencies.
|
||||
set -uo pipefail
|
||||
|
||||
# ---------------------------------------------------------------- constants ----
|
||||
# Root that holds all the peer repos (neuron, foundation/el, products/*, ...).
|
||||
DEV_ROOT="${NEURON_DEV_ROOT:-$HOME/Development/neuron-technologies}"
|
||||
# Where assembled stack workspaces live (persistent; sibling to el-worktrees/).
|
||||
STACK_ROOT="${NSBX_STACK_ROOT:-$DEV_ROOT/stack-worktrees}"
|
||||
EL_REPO_REL="foundation/el"
|
||||
LIVE_ENGRAM_PORT=8742 # live engram — sandbox must never bind it
|
||||
LIVE_SOUL_PORT=7770 # live soul — sandbox must never bind it
|
||||
# nsbx (single-repo isolated-engram tool) lives next to this script.
|
||||
NSBX="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/nsbx"
|
||||
|
||||
C_RED=$'\033[31m'; C_GRN=$'\033[32m'; C_YEL=$'\033[33m'; C_CYN=$'\033[36m'; C_DIM=$'\033[2m'; C_BLD=$'\033[1m'; C_0=$'\033[0m'
|
||||
|
||||
# ---------------------------------------------------------------- helpers ------
|
||||
die(){ printf '%serror:%s %s\n' "$C_RED" "$C_0" "$*" >&2; exit 1; }
|
||||
log(){ printf '%s==>%s %s\n' "$C_BLD" "$C_0" "$*" >&2; }
|
||||
info(){ printf ' %s\n' "$*" >&2; }
|
||||
ok(){ printf ' %s%s%s\n' "$C_GRN" "$*" "$C_0" >&2; }
|
||||
warn(){ printf ' %s%s%s\n' "$C_YEL" "$*" "$C_0" >&2; }
|
||||
need(){ command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"; }
|
||||
|
||||
# ---------------------------------------------------------------- profiles -----
|
||||
# profile_repos <profile> : emit one line per constituent repo:
|
||||
# <relpath-under-DEV_ROOT> | <required|optional> | <role>
|
||||
# The relpath is preserved INSIDE the workspace, so all cross-repo `../foundation/el`
|
||||
# references resolve to the sandbox copy automatically (mirrored-layout wiring).
|
||||
profile_repos(){
|
||||
case "$1" in
|
||||
el-stack)
|
||||
# The compiler+language+framework+tooling are all ONE repo (foundation/el).
|
||||
# Its downstream SDK consumers (forge, dharma) + the language-faculty POC come
|
||||
# along so a change to elc can be proven end-to-end across the kit.
|
||||
cat <<'EOF'
|
||||
foundation/el | required | elc + elb compiler, el_runtime, engram source, el-ui framework, elp/ql/ide/epm tooling
|
||||
engram-language | optional | language-faculty reference POC (Python) — ported into el/elp
|
||||
foundation/forge | optional | downstream SDK consumer — `make build` (imprint forge CLI)
|
||||
foundation/dharma | optional | downstream SDK consumer — CGI provenance registry
|
||||
EOF
|
||||
;;
|
||||
neuron-stack)
|
||||
# The full product: substrate (el) + soul + UI. Engram is NOT a separate repo
|
||||
# (its source lives in foundation/el/engram/src/server.el).
|
||||
cat <<'EOF'
|
||||
foundation/el | required | substrate: elc + el_runtime + engram source + elp NLG the soul imports
|
||||
neuron | required | the soul (:7770) + engram build; soul.el imports ../foundation/el/elp/src/elp.el
|
||||
products/NeuronUI | required | the app/UI (Kotlin/Compose desktop client; bundles the soul binary)
|
||||
products/web | optional | marketing site + interactive soul-demo
|
||||
EOF
|
||||
;;
|
||||
*) return 1;;
|
||||
esac
|
||||
}
|
||||
|
||||
is_profile(){ profile_repos "$1" >/dev/null 2>&1; }
|
||||
|
||||
ws_dir(){ printf '%s/%s-%s' "$STACK_ROOT" "$1" "$2"; } # <root>/<profile>-<name>
|
||||
ws_branch(){ printf 'sandbox/%s-%s' "$1" "$2"; } # branch name used in each repo
|
||||
manifest(){ printf '%s/.stack-manifest.json' "$1"; } # <ws>/.stack-manifest.json
|
||||
|
||||
# ================================================================ up ===========
|
||||
cmd_up(){
|
||||
local profile="$1"; shift
|
||||
local name="" branch="" base_override="" minimal=0
|
||||
[ $# -gt 0 ] && [ "${1#-}" = "$1" ] && { name="$1"; shift; } || die "usage: sandbox $profile <name> [--minimal] [--branch B] [--base REF]"
|
||||
while [ $# -gt 0 ]; do case "$1" in
|
||||
--minimal) minimal=1; shift;;
|
||||
--branch) branch="$2"; shift 2;;
|
||||
--base) base_override="$2"; shift 2;;
|
||||
*) die "unknown flag: $1";;
|
||||
esac; done
|
||||
need git
|
||||
is_profile "$profile" || die "unknown profile: $profile (try: el-stack | neuron-stack)"
|
||||
|
||||
local ws; ws="$(ws_dir "$profile" "$name")"
|
||||
[ -n "$branch" ] || branch="$(ws_branch "$profile" "$name")"
|
||||
|
||||
# -------- pre-flight (fail before creating anything) --------
|
||||
case "$ws" in /tmp/*|/private/tmp/*|/var/tmp/*)
|
||||
die "refusing workspace under a temp dir ($ws) — temp dirs are ablated on compaction; set NSBX_STACK_ROOT to a persistent path";;
|
||||
esac
|
||||
[ -e "$ws" ] && die "workspace already exists: $ws (sandbox down $profile $name first)"
|
||||
|
||||
# resolve + validate every repo, and pick a base sha per repo, BEFORE touching disk
|
||||
local -a rels roles bases origins wts
|
||||
local line rel role_extra role req origin base wt
|
||||
while IFS= read -r line; do
|
||||
[ -z "${line// }" ] && continue
|
||||
rel="$(printf '%s' "$line" | cut -d'|' -f1 | xargs)"
|
||||
req="$(printf '%s' "$line" | cut -d'|' -f2 | xargs)"
|
||||
role="$(printf '%s' "$line" | cut -d'|' -f3- | sed 's/^ *//')"
|
||||
[ "$minimal" -eq 1 ] && [ "$req" = "optional" ] && continue
|
||||
origin="$DEV_ROOT/$rel"
|
||||
git -C "$origin" rev-parse --git-dir >/dev/null 2>&1 || {
|
||||
[ "$req" = "required" ] && die "required repo missing or not a git repo: $origin"
|
||||
warn "skipping optional repo (missing): $rel"; continue; }
|
||||
if [ -n "$base_override" ]; then base="$base_override"; else base="$(git -C "$origin" rev-parse HEAD)"; fi
|
||||
wt="$ws/$rel"
|
||||
[ -e "$wt" ] && die "target worktree path already exists: $wt"
|
||||
rels+=("$rel"); roles+=("$role"); origins+=("$origin"); bases+=("$base"); wts+=("$wt")
|
||||
done < <(profile_repos "$profile")
|
||||
[ "${#rels[@]}" -gt 0 ] || die "no repos resolved for profile $profile"
|
||||
|
||||
log "assembling '$profile' workspace '$name'"
|
||||
info "workspace: $ws"
|
||||
info "branch: $branch (created in each repo, off its committed HEAD)"
|
||||
mkdir -p "$ws"
|
||||
|
||||
# -------- create a worktree per repo (mirrored relpath layout) --------
|
||||
local i n="${#rels[@]}"
|
||||
SB_DONE_WTS=(); SB_DONE_ORIGINS=()
|
||||
for ((i=0; i<n; i++)); do
|
||||
rel="${rels[$i]}"; origin="${origins[$i]}"; base="${bases[$i]}"; wt="${wts[$i]}"
|
||||
mkdir -p "$(dirname "$wt")"
|
||||
local gerr
|
||||
if git -C "$origin" show-ref --verify --quiet "refs/heads/$branch"; then
|
||||
gerr="$(git -C "$origin" worktree add "$wt" "$branch" 2>&1)" \
|
||||
|| { _rollback; die "git worktree add failed for $rel (existing branch $branch):"$'\n'" $gerr"; }
|
||||
else
|
||||
gerr="$(git -C "$origin" worktree add -b "$branch" "$wt" "$base" 2>&1)" \
|
||||
|| { _rollback; die "git worktree add -b $branch failed for $rel (base $base):"$'\n'" $gerr"; }
|
||||
fi
|
||||
SB_DONE_WTS+=("$wt"); SB_DONE_ORIGINS+=("$origin")
|
||||
ok "worktree: $rel -> ${wt#$ws/} (branch $branch @ ${base:0:9})"
|
||||
done
|
||||
|
||||
local el_ws="$ws/$EL_REPO_REL"
|
||||
_write_env "$ws" "$profile" "$name" "$branch" "$el_ws"
|
||||
_write_manifest "$ws" "$profile" "$name" "$branch"
|
||||
_write_build "$ws" "$profile" "$el_ws"
|
||||
_write_readme "$ws" "$profile" "$name" "$branch" "$el_ws"
|
||||
|
||||
# -------- summary --------
|
||||
echo >&2
|
||||
printf '%s STACK WORKSPACE READY — %s / %s%s\n' "$C_BLD" "$profile" "$name" "$C_0" >&2
|
||||
printf ' %-11s %s\n' "workspace" "$ws" >&2
|
||||
printf ' %-11s %s\n' "branch" "$branch (in each repo)" >&2
|
||||
printf ' %-11s %s\n' "repos" "$n worktrees, mirrored layout" >&2
|
||||
echo >&2
|
||||
info "get in: cd $ws && source .stack-env"
|
||||
info "build all: sandbox build $profile $name # (or: cd $ws && ./build.sh)"
|
||||
if [ "$profile" = "neuron-stack" ]; then
|
||||
info "isolated engram (opt-in, via nsbx):"
|
||||
info " nsbx create $profile-$name --source $el_ws && nsbx up $profile-$name"
|
||||
fi
|
||||
info "tear down: sandbox down $profile $name # removes all worktrees; branches kept"
|
||||
}
|
||||
|
||||
# _rollback : remove any worktrees already created this run (globals set by cmd_up)
|
||||
SB_DONE_WTS=(); SB_DONE_ORIGINS=()
|
||||
_rollback(){
|
||||
local j
|
||||
[ "${#SB_DONE_WTS[@]}" -gt 0 ] && warn "rolling back ${#SB_DONE_WTS[@]} partial worktree(s)"
|
||||
for ((j=${#SB_DONE_WTS[@]}-1; j>=0; j--)); do
|
||||
git -C "${SB_DONE_ORIGINS[$j]}" worktree remove --force "${SB_DONE_WTS[$j]}" 2>/dev/null || rm -rf "${SB_DONE_WTS[$j]}"
|
||||
git -C "${SB_DONE_ORIGINS[$j]}" worktree prune 2>/dev/null || true
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- writers ------
|
||||
_write_env(){
|
||||
local ws="$1" profile="$2" name="$3" branch="$4" el_ws="$5"
|
||||
local elc_dir="$el_ws/lang/dist/platform"
|
||||
cat > "$ws/.stack-env" <<ENV
|
||||
# stack env for '$profile/$name' — SOURCE this to work the whole stack together.
|
||||
# Pins EL_REPO + PATH at the SANDBOX copy of foundation/el, so elc/elb/runtime and
|
||||
# every cross-repo ../foundation/el import resolve INSIDE this workspace.
|
||||
# The live mind (:$LIVE_ENGRAM_PORT engram / :$LIVE_SOUL_PORT soul) is deliberately NOT referenced.
|
||||
export STACK_NAME="$profile-$name"
|
||||
export STACK_PROFILE="$profile"
|
||||
export STACK_ROOT_WS="$ws"
|
||||
export EL_REPO="$el_ws"
|
||||
export PATH="$elc_dir:\$PATH" # elc, elb (darwin/linux prebuilt) from the sandbox el
|
||||
ENV
|
||||
if [ "$profile" = "neuron-stack" ]; then
|
||||
cat >> "$ws/.stack-env" <<ENV
|
||||
export NEURON_REPO="$ws/neuron"
|
||||
export NEURONUI_REPO="$ws/products/NeuronUI"
|
||||
# engram/soul are NOT bound here — bring up an ISOLATED engram via nsbx when needed
|
||||
# (nsbx guards the live store & refuses ports :$LIVE_ENGRAM_PORT/:$LIVE_SOUL_PORT):
|
||||
# nsbx create $profile-$name --source \$EL_REPO && nsbx up $profile-$name
|
||||
# nsbx status $profile-$name # prints the isolated engram URL to point the soul at
|
||||
ENV
|
||||
fi
|
||||
# direnv convenience
|
||||
[ -e "$ws/.envrc" ] || printf 'source_env .stack-env 2>/dev/null || source .stack-env\n' > "$ws/.envrc"
|
||||
}
|
||||
|
||||
_write_manifest(){
|
||||
local ws="$1" profile="$2" name="$3" branch="$4"
|
||||
# emit worktree records from git's own worktree list, filtered to this workspace
|
||||
python3 - "$ws" "$profile" "$name" "$branch" "$DEV_ROOT" <<'PY'
|
||||
import json, os, subprocess, sys
|
||||
ws, profile, name, branch, dev = sys.argv[1:6]
|
||||
repos = []
|
||||
for rel in sorted(os.listdir(ws)) if False else []:
|
||||
pass
|
||||
# discover worktrees by walking one level of relpaths we created
|
||||
def git(root, *a):
|
||||
return subprocess.run(["git","-C",root,*a], capture_output=True, text=True).stdout.strip()
|
||||
for dirpath, dirnames, filenames in os.walk(ws):
|
||||
if ".git" in filenames or ".git" in dirnames:
|
||||
rel = os.path.relpath(dirpath, ws)
|
||||
toplevel = git(dirpath, "rev-parse", "--show-toplevel")
|
||||
common = git(dirpath, "rev-parse", "--git-common-dir")
|
||||
origin = os.path.realpath(os.path.join(common, ".."))
|
||||
head = git(dirpath, "rev-parse", "HEAD")
|
||||
repos.append({"rel": rel, "worktree": dirpath, "origin": origin,
|
||||
"branch": branch, "head": head})
|
||||
dirnames[:] = [] # don't descend into a repo
|
||||
repos.sort(key=lambda r: r["rel"])
|
||||
json.dump({"profile": profile, "name": name, "branch": branch,
|
||||
"workspace": ws, "repos": repos},
|
||||
open(os.path.join(ws, ".stack-manifest.json"), "w"), indent=2)
|
||||
PY
|
||||
}
|
||||
|
||||
_write_build(){
|
||||
local ws="$1" profile="$2" el_ws="$3"
|
||||
cat > "$ws/build.sh" <<'BUILD'
|
||||
#!/usr/bin/env bash
|
||||
# build.sh — build the assembled stack together, in dependency order.
|
||||
# Generated by `sandbox`. Run from the workspace root (it sources .stack-env).
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")"; source ./.stack-env
|
||||
say(){ printf '\033[1m==>\033[0m %s\n' "$*"; }
|
||||
ok(){ printf ' \033[32m%s\033[0m\n' "$*"; }
|
||||
bad(){ printf ' \033[31m%s\033[0m\n' "$*"; }
|
||||
|
||||
# locate an elc that runs on THIS machine (darwin-arm64 / linux-amd64), from the sandbox el
|
||||
find_elc(){
|
||||
local d="$EL_REPO/lang/dist/platform"
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Darwin-arm64) echo "$d/elc-darwin-arm64";;
|
||||
Linux-x86_64) echo "$d/elc-linux-amd64";;
|
||||
*) echo "$d/elc";;
|
||||
esac
|
||||
}
|
||||
ELC="$(find_elc)"; [ -x "$ELC" ] || ELC="$EL_REPO/lang/dist/platform/elc"
|
||||
say "elc: $ELC"
|
||||
[ -x "$ELC" ] && ok "$("$ELC" 2>&1 | head -1 || echo present)" || { bad "elc not executable"; exit 1; }
|
||||
|
||||
# canonical runtime C to link (CI-published release copy; ~8 copies exist in-tree)
|
||||
RT="$EL_REPO/lang/releases/v1.0.0-20260501"
|
||||
[ -f "$RT/el_runtime.c" ] || RT="$EL_REPO/lang/el-compiler/runtime"
|
||||
[ -f "$RT/el_runtime.c" ] && ok "el_runtime: $RT/el_runtime.c" || bad "no el_runtime.c found under $EL_REPO/lang"
|
||||
BUILD
|
||||
|
||||
if [ "$profile" = "el-stack" ]; then
|
||||
cat >> "$ws/build.sh" <<'BUILD'
|
||||
|
||||
# ---- EL STACK: prove elc + the el stuff (incl. the el-ui framework) build together ----
|
||||
say "el-ui framework present: $EL_REPO/ui"
|
||||
[ -d "$EL_REPO/ui" ] && ok "framework dir present ($(ls "$EL_REPO/ui" | tr '\n' ' '))" || bad "no ui/ dir"
|
||||
|
||||
# end-to-end compiler proof: elc compiles a real, substantial stack source to C,
|
||||
# then cc links it against the runtime -> a working native binary.
|
||||
B="$(mktemp -d)"
|
||||
say "elc end-to-end: compile engram/src/server.el and link a native binary"
|
||||
if "$ELC" "$EL_REPO/engram/src/server.el" > "$B/x.c" 2>"$B/elc.err"; then
|
||||
ok "elc -> C ($(wc -c <"$B/x.c" | tr -d ' ') bytes)"
|
||||
if cc -std=c11 -O2 -w -I "$RT" -o "$B/x" "$B/x.c" "$RT/el_runtime.c" -lcurl -lpthread -lm 2>"$B/cc.err"; then
|
||||
ok "cc link ok -> native binary $(ls -lh "$B/x" | awk '{print $5}') (elc + runtime build together)"
|
||||
else
|
||||
bad "cc link failed:"; grep -i 'error:' "$B/cc.err" | sort -u | head | sed 's/^/ /'
|
||||
fi
|
||||
else
|
||||
bad "elc compile failed:"; sed 's/^/ /' "$B/elc.err" | head
|
||||
fi
|
||||
|
||||
# optional downstream consumer: forge builds on the SDK (make build) — proves the
|
||||
# freshly-assembled el SDK still compiles a real downstream repo.
|
||||
FORGE="$STACK_ROOT_WS/foundation/forge"
|
||||
if [ -f "$FORGE/Makefile" ]; then
|
||||
say "downstream consumer: foundation/forge (make build)"
|
||||
( cd "$FORGE" && EL_REPO="$EL_REPO" PATH="$EL_REPO/lang/dist/platform:$PATH" make build ) \
|
||||
&& ok "forge built against the sandbox SDK" || bad "forge build failed (see above)"
|
||||
fi
|
||||
say "el-stack build complete"
|
||||
BUILD
|
||||
else
|
||||
cat >> "$ws/build.sh" <<'BUILD'
|
||||
|
||||
# ---- 1) engram (elc engram/src/server.el -> cc engram.c el_runtime.c), from the sandbox el ----
|
||||
say "build engram from $EL_REPO/engram/src/server.el"
|
||||
B="$(mktemp -d)"
|
||||
if "$ELC" "$EL_REPO/engram/src/server.el" > "$B/engram.c" 2>"$B/elc.err"; then
|
||||
ok "elc -> engram.c ($(wc -c <"$B/engram.c" | tr -d ' ') bytes)"
|
||||
if cc -std=c11 -O2 -w -I "$RT" -o "$B/engram" \
|
||||
"$B/engram.c" "$RT/el_runtime.c" -lcurl -lpthread -lm 2>"$B/cc.err"; then
|
||||
ok "engram binary built: $(ls -lh "$B/engram" | awk '{print $5}')"
|
||||
else
|
||||
bad "engram cc link failed:"; grep -i 'error:' "$B/cc.err" | sort -u | head | sed 's/^/ /'
|
||||
fi
|
||||
else
|
||||
bad "engram elc transpile failed:"; sed 's/^/ /' "$B/elc.err"
|
||||
fi
|
||||
|
||||
# ---- 2) soul (imports ../foundation/el/elp/src/elp.el — resolves to SANDBOX el) ----
|
||||
say "soul present + cross-repo import resolves inside the sandbox"
|
||||
[ -f "$NEURON_REPO/soul.el" ] && ok "neuron/soul.el present" || bad "no soul.el"
|
||||
if [ -f "$EL_REPO/elp/src/elp.el" ]; then
|
||||
ok "../foundation/el/elp/src/elp.el resolves -> $EL_REPO/elp/src/elp.el (sandbox copy)"
|
||||
else
|
||||
bad "elp NLG source missing under sandbox el"
|
||||
fi
|
||||
# soul is a heavy single-TU compile; prove elc parses it rather than a full link
|
||||
if "$ELC" "$NEURON_REPO/soul.el" > "$B/soul.c" 2>"$B/soul.err"; then
|
||||
ok "elc compiled soul.el -> $(wc -c <"$B/soul.c" | tr -d ' ') bytes of C (cross-repo imports resolved)"
|
||||
else
|
||||
bad "soul.el elc compile failed:"; sed 's/^/ /' "$B/soul.err" | head
|
||||
fi
|
||||
|
||||
# ---- 3) UI (present + buildable; gradle/JDK21 is heavy so we don't run it here) ----
|
||||
say "app/UI present + buildable"
|
||||
if [ -f "$NEURONUI_REPO/build.sh" ] || [ -f "$NEURONUI_REPO/gradlew" ]; then
|
||||
ok "NeuronUI build entry present (./build.sh / ./gradlew — needs JDK21; run: cd $NEURONUI_REPO && ./gradlew run)"
|
||||
else
|
||||
bad "no NeuronUI build entry"
|
||||
fi
|
||||
say "neuron-stack build complete (engram compiled, soul compiled, UI present & buildable)"
|
||||
BUILD
|
||||
fi
|
||||
chmod +x "$ws/build.sh"
|
||||
}
|
||||
|
||||
_write_readme(){
|
||||
local ws="$1" profile="$2" name="$3" branch="$4" el_ws="$5"
|
||||
cat > "$ws/README.md" <<MD
|
||||
# $profile / $name — combined stack workspace
|
||||
|
||||
Assembled by \`sandbox\`. Every constituent repo is a **git worktree** on branch
|
||||
\`$branch\`, forked off its origin repo's committed HEAD, laid out at its natural
|
||||
relative path so cross-repo \`../foundation/el\` imports resolve **inside this
|
||||
workspace** (the sandbox el), never the live tree.
|
||||
|
||||
## Get in
|
||||
\`\`\`bash
|
||||
cd $ws
|
||||
source .stack-env # EL_REPO + PATH now point at the sandbox el
|
||||
./build.sh # build the stack together (or: sandbox build $profile $name)
|
||||
\`\`\`
|
||||
|
||||
## Layout
|
||||
__STACK_LAYOUT__
|
||||
|
||||
## Isolation
|
||||
- Worktrees only; the live soul/engram (:$LIVE_SOUL_PORT / :$LIVE_ENGRAM_PORT) are never touched.
|
||||
- To run against an **isolated engram**, delegate to \`nsbx\` (guards the live store/ports):
|
||||
\`\`\`bash
|
||||
nsbx create $profile-$name --source \$EL_REPO && nsbx up $profile-$name
|
||||
\`\`\`
|
||||
|
||||
## Tear down
|
||||
\`\`\`bash
|
||||
sandbox down $profile $name # remove every worktree; branch '$branch' kept
|
||||
sandbox down $profile $name --delete-branch
|
||||
\`\`\`
|
||||
MD
|
||||
# fill the layout list from the manifest without embedding backticks in the heredoc
|
||||
python3 - "$ws" <<'PY'
|
||||
import json, os, sys
|
||||
ws = sys.argv[1]
|
||||
d = json.load(open(os.path.join(ws, ".stack-manifest.json")))
|
||||
lines = ["- `%s` <- worktree of %s" % (r["rel"], r["origin"]) for r in d["repos"]]
|
||||
p = os.path.join(ws, "README.md")
|
||||
txt = open(p).read().replace("__STACK_LAYOUT__", "\n".join(lines))
|
||||
open(p, "w").write(txt)
|
||||
PY
|
||||
}
|
||||
|
||||
# ================================================================ down =========
|
||||
cmd_down(){
|
||||
local profile="$1" name="$2"; shift 2 || true
|
||||
local del_branch=0
|
||||
while [ $# -gt 0 ]; do case "$1" in
|
||||
--delete-branch) del_branch=1; shift;;
|
||||
*) die "unknown flag: $1";;
|
||||
esac; done
|
||||
need git
|
||||
local ws; ws="$(ws_dir "$profile" "$name")"
|
||||
[ -d "$ws" ] || die "no such workspace: $ws"
|
||||
local mf; mf="$(manifest "$ws")"
|
||||
[ -f "$mf" ] || die "no manifest in $ws (refusing to guess); remove it by hand if intended"
|
||||
local branch; branch="$(python3 -c "import json;print(json.load(open('$mf'))['branch'])")"
|
||||
|
||||
log "tearing down '$profile/$name' ($ws)"
|
||||
# remove each worktree through its origin repo
|
||||
python3 -c "import json;[print(r['origin']+'\t'+r['worktree']) for r in json.load(open('$mf'))['repos']]" \
|
||||
| while IFS=$'\t' read -r origin wt; do
|
||||
if [ -d "$wt" ]; then
|
||||
git -C "$origin" worktree remove --force "$wt" 2>/dev/null || rm -rf "$wt"
|
||||
git -C "$origin" worktree prune 2>/dev/null || true
|
||||
ok "removed worktree: ${wt#$ws/}"
|
||||
fi
|
||||
if [ "$del_branch" -eq 1 ]; then
|
||||
git -C "$origin" branch -D "$branch" 2>/dev/null && ok "deleted branch $branch in ${origin#$DEV_ROOT/}" || true
|
||||
fi
|
||||
done
|
||||
# drop the (now worktree-free) workspace tree
|
||||
rm -rf "$ws"
|
||||
ok "workspace removed: $ws"
|
||||
[ "$del_branch" -eq 1 ] || info "branch '$branch' kept in each repo (use --delete-branch to drop)"
|
||||
ok "down '$profile/$name' complete (live untouched)"
|
||||
}
|
||||
|
||||
# ================================================================ build ========
|
||||
cmd_build(){
|
||||
local profile="$1" name="$2"; local ws; ws="$(ws_dir "$profile" "$name")"
|
||||
[ -x "$ws/build.sh" ] || die "no build.sh in $ws (is it assembled? sandbox $profile $name)"
|
||||
exec "$ws/build.sh"
|
||||
}
|
||||
|
||||
# ================================================================ list/status ==
|
||||
cmd_list(){
|
||||
[ -d "$STACK_ROOT" ] || { info "no stack workspaces (root $STACK_ROOT absent)"; return 0; }
|
||||
local mf found=0
|
||||
for mf in "$STACK_ROOT"/*/.stack-manifest.json; do
|
||||
[ -f "$mf" ] || continue; found=1
|
||||
python3 -c "import json;d=json.load(open('$mf'));print(' %-22s %-8s %2d repos branch=%s'%(d['profile']+'/'+d['name'],'',len(d['repos']),d['branch']))" 2>/dev/null
|
||||
done
|
||||
[ "$found" -eq 1 ] || info "no assembled stack workspaces under $STACK_ROOT"
|
||||
}
|
||||
|
||||
cmd_status(){
|
||||
local profile="$1" name="$2"; local ws; ws="$(ws_dir "$profile" "$name")"
|
||||
local mf; mf="$(manifest "$ws")"; [ -f "$mf" ] || die "no such workspace: $ws"
|
||||
log "stack '$profile/$name'"; info "workspace: $ws"
|
||||
python3 - "$mf" <<'PY'
|
||||
import json,sys,subprocess
|
||||
d=json.load(open(sys.argv[1]))
|
||||
print(f" branch: {d['branch']}")
|
||||
for r in d['repos']:
|
||||
st=subprocess.run(["git","-C",r["worktree"],"status","--porcelain"],capture_output=True,text=True).stdout
|
||||
n=len([l for l in st.splitlines() if l.strip()])
|
||||
print(f" {r['rel']:<20} {r['head'][:9]} {'clean' if n==0 else str(n)+' changed'}")
|
||||
PY
|
||||
}
|
||||
|
||||
# ================================================================ usage/main ===
|
||||
usage(){ cat >&2 <<EOF
|
||||
${C_BLD}sandbox${C_0} — assemble a WHOLE Neuron stack into one combined worktree workspace,
|
||||
wired to build together on an isolated clean base. Multi-repo sibling of ${C_BLD}nsbx${C_0}.
|
||||
|
||||
${C_CYN}sandbox el-stack <name>${C_0} [--minimal] elc + EL language + el-ui framework + tooling (+ SDK consumers)
|
||||
${C_CYN}sandbox neuron-stack <name>${C_0} [--minimal] runtime/soul + engram + app/UI (the full product)
|
||||
${C_CYN}sandbox build <profile> <name>${C_0} build the assembled stack together (runs its build.sh)
|
||||
${C_CYN}sandbox status <profile> <name>${C_0} inspect one workspace
|
||||
${C_CYN}sandbox list${C_0} list assembled workspaces
|
||||
${C_CYN}sandbox down <profile> <name>${C_0} [--delete-branch] tear down (remove worktrees; branch kept)
|
||||
|
||||
Flags: --minimal only the required repos --branch B branch name --base REF fork point
|
||||
Env: NSBX_STACK_ROOT (workspace root, default \$DEV_ROOT/stack-worktrees) NEURON_DEV_ROOT
|
||||
|
||||
Each constituent repo becomes a git worktree at its natural relpath inside the
|
||||
workspace, so cross-repo ../foundation/el imports resolve to the SANDBOX el. The
|
||||
live soul/engram (:$LIVE_SOUL_PORT / :$LIVE_ENGRAM_PORT) are never touched; isolated-engram
|
||||
bring-up is delegated to nsbx.
|
||||
EOF
|
||||
}
|
||||
|
||||
main(){
|
||||
local cmd="${1:-}"; shift || true
|
||||
case "$cmd" in
|
||||
el-stack|neuron-stack) cmd_up "$cmd" "$@";;
|
||||
up) [ $# -ge 1 ] || die "usage: sandbox up <profile> <name>"; local p="$1"; shift; cmd_up "$p" "$@";;
|
||||
down) [ $# -ge 2 ] || die "usage: sandbox down <profile> <name>"; cmd_down "$@";;
|
||||
build) [ $# -ge 2 ] || die "usage: sandbox build <profile> <name>"; cmd_build "$@";;
|
||||
status) [ $# -ge 2 ] || die "usage: sandbox status <profile> <name>"; cmd_status "$@";;
|
||||
list|ls) cmd_list "$@";;
|
||||
""|-h|--help|help) usage;;
|
||||
*) die "unknown command: $cmd (try: sandbox help)";;
|
||||
esac
|
||||
}
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user