Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2baa0b9a41 | |||
| 6a8b2461cd | |||
| bcb356fe69 | |||
| dd7827059a | |||
| 208e36c899 | |||
| b97ce74d1f | |||
| 155a449c4e | |||
| 4696fd6833 | |||
| 581a351fb1 | |||
| 8ce8656de2 | |||
| 1e49560f1f | |||
| e8f0b5a9de | |||
| 40287c4cfc | |||
| 0481bea44d | |||
| 9d565ca080 | |||
| 4773dd0aa2 | |||
| 6b9d9e6c4a | |||
| b4967af13e | |||
| e3dabe3e08 | |||
| 0a0a2bcb44 | |||
| 2b2a1246e7 | |||
| f78da81aa4 | |||
| 2597a092bb | |||
| 5c41c66a0f |
@@ -214,9 +214,18 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
|
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||||
|
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||||
|
# and the SDK silently never published. Surface failures now.
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||||
|
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||||
gcloud config set project neuron-785695
|
gcloud config set project neuron-785695
|
||||||
|
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||||
|
|
||||||
VERSION="${GITHUB_SHA:0:8}"
|
VERSION="${GITHUB_SHA:0:8}"
|
||||||
|
|
||||||
@@ -268,6 +277,12 @@ jobs:
|
|||||||
# Patches ci-base:dev in-place: pulls the existing image (which has all
|
# Patches ci-base:dev in-place: pulls the existing image (which has all
|
||||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||||
|
#
|
||||||
|
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||||
|
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||||
|
# runner where DinD/Docker availability is fragile. A failure here must
|
||||||
|
# never block or redden the job — the SDK publish above is the deliverable.
|
||||||
|
continue-on-error: true
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
|
|||||||
@@ -212,12 +212,21 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
|
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||||
|
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||||
|
# and the SDK silently never published. Surface failures now.
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||||
|
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||||
apt-get install -y -qq apt-transport-https ca-certificates curl
|
apt-get install -y -qq apt-transport-https ca-certificates curl
|
||||||
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||||
gcloud config set project neuron-785695
|
gcloud config set project neuron-785695
|
||||||
|
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||||
|
|
||||||
VERSION="${GITHUB_SHA:0:8}"
|
VERSION="${GITHUB_SHA:0:8}"
|
||||||
|
|
||||||
@@ -253,6 +262,12 @@ jobs:
|
|||||||
# Patches ci-base:stage in-place: pulls the existing image (which has all
|
# Patches ci-base:stage in-place: pulls the existing image (which has all
|
||||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||||
|
#
|
||||||
|
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||||
|
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||||
|
# runner where DinD/Docker availability is fragile. A failure here must
|
||||||
|
# never block or redden the job — the SDK publish above is the deliverable.
|
||||||
|
continue-on-error: true
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
|
|||||||
@@ -288,12 +288,21 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
|
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||||
|
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||||
|
# and the SDK silently never published. Surface failures now.
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||||
|
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||||
apt-get install -y -qq apt-transport-https ca-certificates curl
|
apt-get install -y -qq apt-transport-https ca-certificates curl
|
||||||
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||||
gcloud config set project neuron-785695
|
gcloud config set project neuron-785695
|
||||||
|
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||||
|
|
||||||
VERSION="${GITHUB_SHA:0:8}"
|
VERSION="${GITHUB_SHA:0:8}"
|
||||||
|
|
||||||
@@ -345,6 +354,12 @@ jobs:
|
|||||||
# Patches ci-base:latest in-place: pulls the existing image (which has all
|
# Patches ci-base:latest in-place: pulls the existing image (which has all
|
||||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||||
|
#
|
||||||
|
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||||
|
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||||
|
# runner where DinD/Docker availability is fragile. A failure here must
|
||||||
|
# never block or redden the job — the SDK publish above is the deliverable.
|
||||||
|
continue-on-error: true
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ jobs:
|
|||||||
# Link to produce the engram binary
|
# Link to produce the engram binary
|
||||||
- name: Link engram binary
|
- name: Link engram binary
|
||||||
run: |
|
run: |
|
||||||
cc -std=c11 -O2 \
|
cc -std=c11 -O2 -DHAVE_CURL \
|
||||||
-I /usr/local/lib/el \
|
-I /usr/local/lib/el \
|
||||||
-o dist/engram \
|
-o dist/engram \
|
||||||
dist/engram.c \
|
dist/engram.c \
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ jobs:
|
|||||||
# Link to produce the engram binary
|
# Link to produce the engram binary
|
||||||
- name: Link engram binary
|
- name: Link engram binary
|
||||||
run: |
|
run: |
|
||||||
cc -std=c11 -O2 \
|
cc -std=c11 -O2 -DHAVE_CURL \
|
||||||
-I /usr/local/lib/el \
|
-I /usr/local/lib/el \
|
||||||
-o dist/engram \
|
-o dist/engram \
|
||||||
dist/engram.c \
|
dist/engram.c \
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ jobs:
|
|||||||
# Link to produce the engram binary
|
# Link to produce the engram binary
|
||||||
- name: Link engram binary
|
- name: Link engram binary
|
||||||
run: |
|
run: |
|
||||||
cc -std=c11 -O2 \
|
cc -std=c11 -O2 -DHAVE_CURL \
|
||||||
-I /usr/local/lib/el \
|
-I /usr/local/lib/el \
|
||||||
-o dist/engram \
|
-o dist/engram \
|
||||||
dist/engram.c \
|
dist/engram.c \
|
||||||
|
|||||||
Vendored
BIN
Binary file not shown.
@@ -17,6 +17,16 @@
|
|||||||
// 4. Append dep to order after all its transitive deps
|
// 4. Append dep to order after all its transitive deps
|
||||||
// 5. Deduplicate: skip already-ordered vessels
|
// 5. Deduplicate: skip already-ordered vessels
|
||||||
|
|
||||||
|
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||||
|
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
|
||||||
|
// give elc the C prototypes so generated install.c compiles cleanly under strict
|
||||||
|
// compilers (gcc>=14 / clang) that reject implicit function declarations.
|
||||||
|
extern fn manifest_name(src: String) -> String // manifest.el
|
||||||
|
extern fn manifest_deps(src: String) -> String // manifest.el
|
||||||
|
extern fn registry_token() -> String // registry.el
|
||||||
|
extern fn registry_find(name: String, version: String) -> String // registry.el
|
||||||
|
extern fn registry_latest_version(name: String) -> String // registry.el
|
||||||
|
|
||||||
// ── Install paths ─────────────────────────────────────────────────────────────
|
// ── Install paths ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// packages_dir returns the root directory for installed vessels.
|
// packages_dir returns the root directory for installed vessels.
|
||||||
|
|||||||
@@ -14,6 +14,15 @@
|
|||||||
// EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies)
|
// EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies)
|
||||||
// EPM_TOKEN — Gitea personal access token (required for publish)
|
// EPM_TOKEN — Gitea personal access token (required for publish)
|
||||||
|
|
||||||
|
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||||
|
// These symbols are defined in sibling epm modules or the El runtime and are
|
||||||
|
// resolved at link time. The `extern fn` decls give elc the C prototype so the
|
||||||
|
// generated registry.c compiles cleanly under strict compilers (gcc>=14 / clang)
|
||||||
|
// that reject implicit function declarations. Signature arity must match the
|
||||||
|
// definition; return/param types are informational (all lower to el_val_t).
|
||||||
|
extern fn config(key: String) -> String // El runtime builtin
|
||||||
|
extern fn read_installed() -> String // install.el
|
||||||
|
|
||||||
// ── Config helpers ────────────────────────────────────────────────────────────
|
// ── Config helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// registry_api_url returns the Gitea API base URL with no trailing slash.
|
// registry_api_url returns the Gitea API base URL with no trailing slash.
|
||||||
|
|||||||
@@ -6,6 +6,15 @@
|
|||||||
// Depends on: registry.el (registry_latest_version, registry_find),
|
// Depends on: registry.el (registry_latest_version, registry_find),
|
||||||
// install.el (read_installed, install_vessel, installed_version)
|
// install.el (read_installed, install_vessel, installed_version)
|
||||||
|
|
||||||
|
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||||
|
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
|
||||||
|
// give elc the C prototypes so generated update.c compiles cleanly under strict
|
||||||
|
// compilers (gcc>=14 / clang) that reject implicit function declarations.
|
||||||
|
extern fn read_installed() -> String // install.el
|
||||||
|
extern fn installed_version(name: String) -> String // install.el
|
||||||
|
extern fn install_vessel(name: String, version: String) -> Bool // install.el
|
||||||
|
extern fn registry_latest_version(name: String) -> String // registry.el
|
||||||
|
|
||||||
// ── Semver helpers ────────────────────────────────────────────────────────────
|
// ── Semver helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// semver_part extracts the Nth dot-separated component from a semver string.
|
// semver_part extracts the Nth dot-separated component from a semver string.
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) {
|
|||||||
#include <direct.h> /* _mkdir */
|
#include <direct.h> /* _mkdir */
|
||||||
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
||||||
#define timegm _mkgmtime /* UTC tm → time_t */
|
#define timegm _mkgmtime /* UTC tm → time_t */
|
||||||
|
#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() (<io.h>) is the equiv */
|
||||||
|
|
||||||
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
||||||
static inline int setenv(const char* name, const char* value, int overwrite) {
|
static inline int setenv(const char* name, const char* value, int overwrite) {
|
||||||
|
|||||||
@@ -710,6 +710,7 @@ static void jb_init(JsonBuf* b);
|
|||||||
static void jb_putc(JsonBuf* b, char c);
|
static void jb_putc(JsonBuf* b, char c);
|
||||||
static void jb_puts(JsonBuf* b, const char* s);
|
static void jb_puts(JsonBuf* b, const char* s);
|
||||||
static void jb_emit_escaped(JsonBuf* b, const char* s);
|
static void jb_emit_escaped(JsonBuf* b, const char* s);
|
||||||
|
static char* jb_finish(JsonBuf* b);
|
||||||
static int looks_like_string(el_val_t v);
|
static int looks_like_string(el_val_t v);
|
||||||
static const char* json_find_key(const char* s, const char* key);
|
static const char* json_find_key(const char* s, const char* key);
|
||||||
static const char* json_skip_value(const char* p);
|
static const char* json_skip_value(const char* p);
|
||||||
@@ -1962,8 +1963,9 @@ void http_serve_async(el_val_t port, el_val_t handler) {
|
|||||||
int sock = socket(AF_INET6, SOCK_STREAM, 0);
|
int sock = socket(AF_INET6, SOCK_STREAM, 0);
|
||||||
if (sock < 0) { perror("socket"); return; }
|
if (sock < 0) { perror("socket"); return; }
|
||||||
int yes = 1; int no = 0;
|
int yes = 1; int no = 0;
|
||||||
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
/* Win32/mingw setsockopt takes optval as (const char*); the cast is portable on POSIX too. */
|
||||||
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no));
|
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));
|
||||||
|
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no));
|
||||||
struct sockaddr_in6 addr;
|
struct sockaddr_in6 addr;
|
||||||
memset(&addr, 0, sizeof(addr));
|
memset(&addr, 0, sizeof(addr));
|
||||||
addr.sin6_family = AF_INET6;
|
addr.sin6_family = AF_INET6;
|
||||||
@@ -2014,7 +2016,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) {
|
|||||||
jb_puts(&out, ",\"body\":");
|
jb_puts(&out, ",\"body\":");
|
||||||
jb_emit_escaped(&out, b);
|
jb_emit_escaped(&out, b);
|
||||||
jb_putc(&out, '}');
|
jb_putc(&out, '}');
|
||||||
return el_wrap_str(out.buf);
|
return el_wrap_str(jb_finish(&out));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ── Filesystem ──────────────────────────────────────────────────────────── */
|
/* ── Filesystem ──────────────────────────────────────────────────────────── */
|
||||||
@@ -2121,7 +2123,7 @@ el_val_t exec_capture(el_val_t cmdv) {
|
|||||||
char buf[4096];
|
char buf[4096];
|
||||||
while (fgets(buf, sizeof(buf), f)) jb_puts(&b, buf);
|
while (fgets(buf, sizeof(buf), f)) jb_puts(&b, buf);
|
||||||
pclose(f);
|
pclose(f);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
// exec — run a shell command via /bin/sh, capture stdout, return as String.
|
// exec — run a shell command via /bin/sh, capture stdout, return as String.
|
||||||
@@ -2145,7 +2147,7 @@ el_val_t exec(el_val_t cmdv) {
|
|||||||
jb_puts(&b, buf);
|
jb_puts(&b, buf);
|
||||||
}
|
}
|
||||||
pclose(f);
|
pclose(f);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
// exec_bg — run a shell command in background, return PID as String.
|
// exec_bg — run a shell command in background, return PID as String.
|
||||||
@@ -3292,6 +3294,23 @@ static void jb_puts(JsonBuf* b, const char* s) {
|
|||||||
b->buf[b->len] = '\0';
|
b->buf[b->len] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* jb_finish — call exactly once, at the point a JsonBuf's buffer is handed
|
||||||
|
* off as an el_val_t return value (via el_wrap_str). JsonBuf allocates through
|
||||||
|
* raw malloc/realloc (jb_init/jb_reserve), NOT el_strdup/el_strbuf, so unlike
|
||||||
|
* those helpers it was never registered with the per-request arena — every
|
||||||
|
* JsonBuf-built JSON string leaked unconditionally, whether or not a request
|
||||||
|
* arena was active, because nothing ever freed it. el_arena_track() is a
|
||||||
|
* documented no-op when no arena is active, so this is safe to call from any
|
||||||
|
* context (HTTP request handler, CLI arena-scope, or no arena at all — the
|
||||||
|
* last case still leaks, same as before, until the caller's context is itself
|
||||||
|
* arena-scoped). Do NOT call this on a JsonBuf whose ->buf is consumed by
|
||||||
|
* another builder (jb_puts(&other, x.buf)) and freed manually — only on the
|
||||||
|
* one actually returned via el_wrap_str. */
|
||||||
|
static char* jb_finish(JsonBuf* b) {
|
||||||
|
el_arena_track(b->buf);
|
||||||
|
return b->buf;
|
||||||
|
}
|
||||||
|
|
||||||
static void jb_emit_escaped(JsonBuf* b, const char* s) {
|
static void jb_emit_escaped(JsonBuf* b, const char* s) {
|
||||||
jb_putc(b, '"');
|
jb_putc(b, '"');
|
||||||
const unsigned char* p = (const unsigned char*)s;
|
const unsigned char* p = (const unsigned char*)s;
|
||||||
@@ -3400,7 +3419,7 @@ static void jb_emit_value(JsonBuf* b, el_val_t v) {
|
|||||||
el_val_t json_stringify(el_val_t v) {
|
el_val_t json_stringify(el_val_t v) {
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_emit_value(&b, v);
|
jb_emit_value(&b, v);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ── JSON substring accessors ────────────────────────────────────────────── */
|
/* ── JSON substring accessors ────────────────────────────────────────────── */
|
||||||
@@ -3587,7 +3606,7 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) {
|
|||||||
jb_putc(&b, ':');
|
jb_putc(&b, ':');
|
||||||
jb_puts(&b, raw_val);
|
jb_puts(&b, raw_val);
|
||||||
jb_putc(&b, '}');
|
jb_putc(&b, '}');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
const char* existing = json_find_key(json, k);
|
const char* existing = json_find_key(json, k);
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
@@ -3601,7 +3620,7 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) {
|
|||||||
b.buf[b.len] = '\0';
|
b.buf[b.len] = '\0';
|
||||||
jb_puts(&b, raw_val);
|
jb_puts(&b, raw_val);
|
||||||
jb_puts(&b, end);
|
jb_puts(&b, end);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
/* Insert before closing '}'. Find last '}' */
|
/* Insert before closing '}'. Find last '}' */
|
||||||
size_t jl = strlen(json);
|
size_t jl = strlen(json);
|
||||||
@@ -3633,7 +3652,7 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) {
|
|||||||
jb_puts(&b, raw_val);
|
jb_puts(&b, raw_val);
|
||||||
/* Append from close_idx onward */
|
/* Append from close_idx onward */
|
||||||
jb_puts(&b, json + close_idx);
|
jb_puts(&b, json + close_idx);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t json_array_len(el_val_t json_str) {
|
el_val_t json_array_len(el_val_t json_str) {
|
||||||
@@ -3771,7 +3790,7 @@ el_val_t json_build_object(el_val_t kvs) {
|
|||||||
jb_putc(&b, '"');
|
jb_putc(&b, '"');
|
||||||
}
|
}
|
||||||
jb_putc(&b, '}');
|
jb_putc(&b, '}');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* json_build_array — build a JSON array from a list of raw JSON values.
|
/* json_build_array — build a JSON array from a list of raw JSON values.
|
||||||
@@ -3789,7 +3808,7 @@ el_val_t json_build_array(el_val_t items) {
|
|||||||
jb_puts(&b, vs);
|
jb_puts(&b, vs);
|
||||||
}
|
}
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ── Time ────────────────────────────────────────────────────────────────── */
|
/* ── Time ────────────────────────────────────────────────────────────────── */
|
||||||
@@ -5193,7 +5212,7 @@ el_val_t state_keys(void) {
|
|||||||
}
|
}
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
pthread_mutex_unlock(&_state_mu);
|
pthread_mutex_unlock(&_state_mu);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Returns 1 (true) if the key is present in the state store, else 0 (false). */
|
/* Returns 1 (true) if the key is present in the state store, else 0 (false). */
|
||||||
@@ -5399,7 +5418,7 @@ el_val_t str_format(el_val_t fmt, el_val_t data) {
|
|||||||
jb_putc(&b, *p);
|
jb_putc(&b, *p);
|
||||||
p++;
|
p++;
|
||||||
}
|
}
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t str_lower(el_val_t s) { return str_to_lower(s); }
|
el_val_t str_lower(el_val_t s) { return str_to_lower(s); }
|
||||||
@@ -5815,7 +5834,7 @@ el_val_t list_join(el_val_t listv, el_val_t sep) {
|
|||||||
jb_puts(&b, tmp);
|
jb_puts(&b, tmp);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t list_range(el_val_t start, el_val_t end) {
|
el_val_t list_range(el_val_t start, el_val_t end) {
|
||||||
@@ -6465,12 +6484,12 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
|
|||||||
n->id = engram_new_id();
|
n->id = engram_new_id();
|
||||||
const char* c = EL_CSTR(content);
|
const char* c = EL_CSTR(content);
|
||||||
const char* nt = EL_CSTR(node_type);
|
const char* nt = EL_CSTR(node_type);
|
||||||
n->content = el_strdup(c ? c : "");
|
n->content = el_strdup_persist(c ? c : "");
|
||||||
n->node_type = el_strdup(nt && *nt ? nt : "Memory");
|
n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory");
|
||||||
n->label = engram_first_n_chars(c, 60);
|
{ char* _lb = engram_first_n_chars(c, 60); n->label = el_strdup_persist(_lb); } /* persist: stored field must outlive request arena */
|
||||||
n->tier = el_strdup("Working");
|
n->tier = el_strdup_persist("Working");
|
||||||
n->tags = el_strdup("");
|
n->tags = el_strdup_persist("");
|
||||||
n->metadata = el_strdup("{}");
|
n->metadata = el_strdup_persist("{}");
|
||||||
n->salience = engram_decode_score(salience);
|
n->salience = engram_decode_score(salience);
|
||||||
if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5;
|
if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5;
|
||||||
n->importance = 0.5;
|
n->importance = 0.5;
|
||||||
@@ -6597,11 +6616,11 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
|
|||||||
const char* lb = EL_CSTR(label);
|
const char* lb = EL_CSTR(label);
|
||||||
const char* tg = EL_CSTR(tags);
|
const char* tg = EL_CSTR(tags);
|
||||||
const char* st = EL_CSTR(status);
|
const char* st = EL_CSTR(status);
|
||||||
n->content = el_strdup(c ? c : "");
|
n->content = el_strdup_persist(c ? c : "");
|
||||||
n->node_type = el_strdup(nt && *nt ? nt : "Memory");
|
n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory");
|
||||||
n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : ""));
|
n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : ""));
|
||||||
n->tier = el_strdup("Working");
|
n->tier = el_strdup_persist("Working");
|
||||||
n->tags = el_strdup(tg ? tg : "");
|
n->tags = el_strdup_persist(tg ? tg : "");
|
||||||
if (st && *st) {
|
if (st && *st) {
|
||||||
/* Minimal metadata payload: {"status":"..."}. Keep it cheap so
|
/* Minimal metadata payload: {"status":"..."}. Keep it cheap so
|
||||||
* callers using `status` don't pay JSON parse cost on every read. */
|
* callers using `status` don't pay JSON parse cost on every read. */
|
||||||
@@ -6610,7 +6629,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
|
|||||||
snprintf(meta, sl, "{\"status\":\"%s\"}", st);
|
snprintf(meta, sl, "{\"status\":\"%s\"}", st);
|
||||||
n->metadata = meta;
|
n->metadata = meta;
|
||||||
} else {
|
} else {
|
||||||
n->metadata = el_strdup("{}");
|
n->metadata = el_strdup_persist("{}");
|
||||||
}
|
}
|
||||||
n->salience = engram_decode_score(salience);
|
n->salience = engram_decode_score(salience);
|
||||||
n->importance = engram_decode_score(certainty);
|
n->importance = engram_decode_score(certainty);
|
||||||
@@ -6808,6 +6827,312 @@ static int istr_contains(const char* hay, const char* needle) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── Tokenized query matching ───────────────────────────────────────────
|
||||||
|
* The engram query surface (search / activate / goal-bias) historically
|
||||||
|
* matched the ENTIRE raw query string as a single case-insensitive
|
||||||
|
* substring via istr_contains(field, q). That is Ctrl-F, not search:
|
||||||
|
* a multi-word query like "windows msi signing" only matched a node whose
|
||||||
|
* text contained that exact contiguous run, so real multi-word queries
|
||||||
|
* returned zero. istr_contains stays as the per-TOKEN primitive; these
|
||||||
|
* helpers split the query on whitespace and match ANY token, then rank by
|
||||||
|
* how many DISTINCT tokens a node covers. Single-token queries are a strict
|
||||||
|
* special case (score is 0 or 1) so single-word callers never regress. */
|
||||||
|
#define ENGRAM_MAX_QTOKENS 32
|
||||||
|
#define ENGRAM_QTOK_LEN 256
|
||||||
|
|
||||||
|
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
|
||||||
|
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
|
||||||
|
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
|
||||||
|
static int engram_tokenize_query(const char* q,
|
||||||
|
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
|
||||||
|
int n = 0;
|
||||||
|
if (!q) return 0;
|
||||||
|
const char* p = q;
|
||||||
|
while (*p && n < maxtok) {
|
||||||
|
while (*p && isspace((unsigned char)*p)) p++;
|
||||||
|
if (!*p) break;
|
||||||
|
char buf[ENGRAM_QTOK_LEN];
|
||||||
|
size_t tl = 0;
|
||||||
|
while (*p && !isspace((unsigned char)*p)) {
|
||||||
|
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
|
||||||
|
p++;
|
||||||
|
}
|
||||||
|
buf[tl] = '\0';
|
||||||
|
if (tl == 0) continue;
|
||||||
|
int dup = 0;
|
||||||
|
for (int s = 0; s < n; s++) {
|
||||||
|
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
|
||||||
|
}
|
||||||
|
if (dup) continue;
|
||||||
|
memcpy(toks[n], buf, tl + 1);
|
||||||
|
n++;
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Count how many of the ntok distinct query tokens appear (case-insensitive)
|
||||||
|
* in the node's content, label, or tags. 0 == no match. */
|
||||||
|
static int engram_node_match_score(const EngramNode* n,
|
||||||
|
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||||
|
int score = 0;
|
||||||
|
for (int t = 0; t < ntok; t++) {
|
||||||
|
if (istr_contains(n->content, toks[t]) ||
|
||||||
|
istr_contains(n->label, toks[t]) ||
|
||||||
|
istr_contains(n->tags, toks[t]))
|
||||||
|
score++;
|
||||||
|
}
|
||||||
|
return score;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Rank entry: distinct-token match count (primary, desc) then salience
|
||||||
|
* (tiebreak, desc). */
|
||||||
|
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||||
|
static int engram_rank_cmp(const void* a, const void* b) {
|
||||||
|
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||||
|
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||||
|
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
|
||||||
|
if (ea->salience < eb->salience) return 1;
|
||||||
|
if (ea->salience > eb->salience) return -1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ══════════════════════════════════════════════════════════════════════════
|
||||||
|
* SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
|
||||||
|
* ──────────────────────────────────────────────────────────────────────────
|
||||||
|
* Augments the lexical (istr_contains) matcher with dense-vector retrieval.
|
||||||
|
* Node content and the query are embedded through a local Ollama server;
|
||||||
|
* nodes are ranked by cosine similarity and UNIONED with lexical hits. This
|
||||||
|
* lets a paraphrase query surface a node whose words never appear in it.
|
||||||
|
*
|
||||||
|
* DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot
|
||||||
|
* runtime probe of the embedding endpoint. If curl is not compiled in, or
|
||||||
|
* Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns
|
||||||
|
* "no semantic signal" and callers fall back to pure lexical behaviour —
|
||||||
|
* byte-for-byte the pre-existing search.
|
||||||
|
*
|
||||||
|
* CACHE. Node embeddings are computed lazily on first use and cached in
|
||||||
|
* process memory keyed by node id, with an FNV-1a content hash for
|
||||||
|
* invalidation (edited content re-embeds). The query is embedded once per
|
||||||
|
* search call. This is what "avoid re-embedding the whole graph every query"
|
||||||
|
* buys us: a warm cache serves cosine from RAM. (A cold process still pays
|
||||||
|
* O(N) embed calls the first time each node is scanned — persisting the cache
|
||||||
|
* to a snapshot sidecar is the documented next step, not done here.)
|
||||||
|
*
|
||||||
|
* nomic task prefixes ("search_query:" / "search_document:") are applied
|
||||||
|
* because nomic-embed-text is trained with them; they materially improve
|
||||||
|
* retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48).
|
||||||
|
*
|
||||||
|
* ENV:
|
||||||
|
* ENGRAM_SEMANTIC "0" disables; unset/other = auto-probe
|
||||||
|
* ENGRAM_EMBED_URL default http://localhost:11434/api/embeddings
|
||||||
|
* ENGRAM_EMBED_MODEL default nomic-embed-text
|
||||||
|
* ENGRAM_SEMANTIC_MIN cosine threshold for a pure-semantic match (def 0.6)
|
||||||
|
* ════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
static double engram_semantic_min(void) {
|
||||||
|
static double v = -1.0;
|
||||||
|
if (v >= 0.0) return v;
|
||||||
|
const char* s = getenv("ENGRAM_SEMANTIC_MIN");
|
||||||
|
double d = 0.6;
|
||||||
|
if (s && *s) { char* e = NULL; double t = strtod(s, &e);
|
||||||
|
if (e != s && t >= 0.0 && t <= 1.0) d = t; }
|
||||||
|
v = d; return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef HAVE_CURL
|
||||||
|
|
||||||
|
typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry;
|
||||||
|
static EngramEmbEntry* g_emb_items = NULL;
|
||||||
|
static int64_t g_emb_count = 0, g_emb_cap = 0;
|
||||||
|
static int g_emb_state = 0; /* 0=unprobed, 1=available, -1=disabled */
|
||||||
|
|
||||||
|
static uint64_t engram_fnv1a(const char* s) {
|
||||||
|
uint64_t h = 1469598103934665603ULL;
|
||||||
|
if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||||
|
h ^= *p; h *= 1099511628211ULL;
|
||||||
|
}
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */
|
||||||
|
static float* engram_parse_embedding(const char* json, int* out_dim) {
|
||||||
|
if (!json) return NULL;
|
||||||
|
const char* p = strstr(json, "\"embedding\"");
|
||||||
|
if (!p) return NULL;
|
||||||
|
p = strchr(p, '[');
|
||||||
|
if (!p) return NULL;
|
||||||
|
p++;
|
||||||
|
int cap = 1024, n = 0;
|
||||||
|
float* v = malloc((size_t)cap * sizeof(float));
|
||||||
|
if (!v) return NULL;
|
||||||
|
while (*p && *p != ']') {
|
||||||
|
while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++;
|
||||||
|
if (*p == ']' || !*p) break;
|
||||||
|
char* e = NULL;
|
||||||
|
double d = strtod(p, &e);
|
||||||
|
if (e == p) break;
|
||||||
|
if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float));
|
||||||
|
if (!nv) { free(v); return NULL; } v = nv; }
|
||||||
|
v[n++] = (float)d;
|
||||||
|
p = e;
|
||||||
|
}
|
||||||
|
if (n == 0) { free(v); return NULL; }
|
||||||
|
*out_dim = n;
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */
|
||||||
|
static char* engram_json_escape(const char* src) {
|
||||||
|
if (!src) src = "";
|
||||||
|
size_t n = strlen(src);
|
||||||
|
char* out = malloc(n * 2 + 1);
|
||||||
|
if (!out) return NULL;
|
||||||
|
size_t j = 0;
|
||||||
|
for (size_t i = 0; i < n; i++) {
|
||||||
|
unsigned char c = (unsigned char)src[i];
|
||||||
|
if (c == '"') { out[j++] = '\\'; out[j++] = '"'; }
|
||||||
|
else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; }
|
||||||
|
else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; }
|
||||||
|
else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; }
|
||||||
|
else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; }
|
||||||
|
else if (c < 0x20) { /* drop other control bytes */ }
|
||||||
|
else { out[j++] = (char)c; }
|
||||||
|
}
|
||||||
|
out[j] = '\0';
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */
|
||||||
|
static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) {
|
||||||
|
if (!text) return NULL;
|
||||||
|
const char* url = getenv("ENGRAM_EMBED_URL");
|
||||||
|
if (!url || !*url) url = "http://localhost:11434/api/embeddings";
|
||||||
|
const char* model = getenv("ENGRAM_EMBED_MODEL");
|
||||||
|
if (!model || !*model) model = "nomic-embed-text";
|
||||||
|
/* Bound content length to keep latency/memory sane on huge nodes. */
|
||||||
|
char* trunc = NULL;
|
||||||
|
size_t maxlen = 8192;
|
||||||
|
if (strlen(text) > maxlen) {
|
||||||
|
trunc = malloc(maxlen + 1);
|
||||||
|
if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; }
|
||||||
|
}
|
||||||
|
char* esc_prefix = engram_json_escape(prefix ? prefix : "");
|
||||||
|
char* esc = engram_json_escape(text);
|
||||||
|
free(trunc);
|
||||||
|
if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; }
|
||||||
|
size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64;
|
||||||
|
char* body = malloc(blen);
|
||||||
|
if (!body) { free(esc); free(esc_prefix); return NULL; }
|
||||||
|
snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc);
|
||||||
|
free(esc); free(esc_prefix);
|
||||||
|
|
||||||
|
CURL* c = curl_easy_init();
|
||||||
|
if (!c) { free(body); return NULL; }
|
||||||
|
HttpBuf rb; httpbuf_init(&rb);
|
||||||
|
struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json");
|
||||||
|
char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
|
||||||
|
curl_easy_setopt(c, CURLOPT_URL, url);
|
||||||
|
curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
|
||||||
|
curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
|
||||||
|
curl_easy_setopt(c, CURLOPT_POST, 1L);
|
||||||
|
curl_easy_setopt(c, CURLOPT_POSTFIELDS, body);
|
||||||
|
curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body));
|
||||||
|
curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
|
||||||
|
curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
|
||||||
|
curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
|
||||||
|
curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
|
||||||
|
CURLcode rc = curl_easy_perform(c);
|
||||||
|
curl_slist_free_all(h);
|
||||||
|
curl_easy_cleanup(c);
|
||||||
|
free(body);
|
||||||
|
if (rc != CURLE_OK) { free(rb.data); return NULL; }
|
||||||
|
float* v = engram_parse_embedding(rb.data, out_dim);
|
||||||
|
free(rb.data);
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* One-shot probe: is semantic search available? Caches the verdict. */
|
||||||
|
static int engram_semantic_enabled(void) {
|
||||||
|
if (g_emb_state != 0) return g_emb_state == 1;
|
||||||
|
const char* s = getenv("ENGRAM_SEMANTIC");
|
||||||
|
if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; }
|
||||||
|
int dim = 0;
|
||||||
|
float* v = engram_embed_raw("search_query: ", "probe", &dim);
|
||||||
|
if (v && dim > 0) { free(v); g_emb_state = 1; return 1; }
|
||||||
|
free(v);
|
||||||
|
g_emb_state = -1; return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */
|
||||||
|
static float* engram_embed_query(const char* q, int* dim) {
|
||||||
|
if (!engram_semantic_enabled()) return NULL;
|
||||||
|
if (!q || !*q) return NULL;
|
||||||
|
return engram_embed_raw("search_query: ", q, dim);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */
|
||||||
|
static const float* engram_node_vec(EngramNode* n, int* out_dim) {
|
||||||
|
if (!n || !n->id) return NULL;
|
||||||
|
uint64_t h = engram_fnv1a(n->content);
|
||||||
|
for (int64_t i = 0; i < g_emb_count; i++) {
|
||||||
|
if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) {
|
||||||
|
if (g_emb_items[i].hash == h && g_emb_items[i].vec) {
|
||||||
|
*out_dim = g_emb_items[i].dim; return g_emb_items[i].vec;
|
||||||
|
}
|
||||||
|
/* content changed → re-embed in place */
|
||||||
|
int dim = 0;
|
||||||
|
float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
|
||||||
|
if (!v) return NULL;
|
||||||
|
free(g_emb_items[i].vec);
|
||||||
|
g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h;
|
||||||
|
*out_dim = dim; return v;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int dim = 0;
|
||||||
|
float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
|
||||||
|
if (!v) return NULL;
|
||||||
|
if (g_emb_count >= g_emb_cap) {
|
||||||
|
int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256;
|
||||||
|
EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry));
|
||||||
|
if (!ni) { free(v); return NULL; }
|
||||||
|
g_emb_items = ni; g_emb_cap = nc;
|
||||||
|
}
|
||||||
|
g_emb_items[g_emb_count].id = strdup(n->id);
|
||||||
|
g_emb_items[g_emb_count].hash = h;
|
||||||
|
g_emb_items[g_emb_count].vec = v;
|
||||||
|
g_emb_items[g_emb_count].dim = dim;
|
||||||
|
g_emb_count++;
|
||||||
|
*out_dim = dim; return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
static double engram_cosine(const float* a, const float* b, int dim) {
|
||||||
|
double dot = 0, na = 0, nb = 0;
|
||||||
|
for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i];
|
||||||
|
na += (double)a[i] * a[i];
|
||||||
|
nb += (double)b[i] * b[i]; }
|
||||||
|
if (na <= 0 || nb <= 0) return 0.0;
|
||||||
|
return dot / (sqrt(na) * sqrt(nb));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */
|
||||||
|
static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
|
||||||
|
if (!qvec || qdim <= 0) return 0.0;
|
||||||
|
int ndim = 0;
|
||||||
|
const float* nv = engram_node_vec(n, &ndim);
|
||||||
|
if (!nv || ndim != qdim) return 0.0;
|
||||||
|
return engram_cosine(qvec, nv, qdim);
|
||||||
|
}
|
||||||
|
|
||||||
|
#else /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical.
|
||||||
|
* Only the two boundary functions the always-compiled search/activate
|
||||||
|
* code calls are stubbed; the query embed always yields NULL so every
|
||||||
|
* cosine is 0 and every caller collapses to lexical-only. */
|
||||||
|
static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; }
|
||||||
|
static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
|
||||||
|
(void)n; (void)qvec; (void)qdim; return 0.0;
|
||||||
|
}
|
||||||
|
#endif /* HAVE_CURL */
|
||||||
|
|
||||||
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||||
EngramStore* g = engram_get();
|
EngramStore* g = engram_get();
|
||||||
const char* q = EL_CSTR(query);
|
const char* q = EL_CSTR(query);
|
||||||
@@ -6815,21 +7140,45 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
|||||||
if (lim <= 0) lim = 100;
|
if (lim <= 0) lim = 100;
|
||||||
el_val_t lst = el_list_empty();
|
el_val_t lst = el_list_empty();
|
||||||
if (!q || !*q) return lst;
|
if (!q || !*q) return lst;
|
||||||
int64_t found = 0;
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||||
|
if (ntok == 0) return lst;
|
||||||
|
/* Semantic augmentation: embed the query once; a node is a hit if it covers
|
||||||
|
* >=1 query token (tokenized-lexical, #66) OR its cosine clears the
|
||||||
|
* threshold (#67). qvec is NULL (cosine 0) when semantic is unavailable →
|
||||||
|
* pure tokenized-lexical, byte-identical to the lexical-only behaviour. */
|
||||||
|
int qdim = 0;
|
||||||
|
float* qvec = engram_embed_query(q, &qdim);
|
||||||
|
double sem_min = engram_semantic_min();
|
||||||
|
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||||
|
if (!hits) { free(qvec); return lst; }
|
||||||
|
int64_t nhits = 0;
|
||||||
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
||||||
* shape output but are invisible to introspection ("what do you
|
* shape output but are invisible to introspection ("what do you
|
||||||
* know about yourself"). They still surface via engram_activate
|
* know about yourself"). They still surface via engram_activate
|
||||||
* + engram_compile_layered_json — that's the legitimate path. */
|
* + engram_compile_layered_json — that's the legitimate path. */
|
||||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||||
if (istr_contains(n->content, q) ||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
istr_contains(n->label, q) ||
|
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
||||||
istr_contains(n->tags, q)) {
|
if (sc > 0 || sem >= sem_min) {
|
||||||
lst = el_list_append(lst, engram_node_to_map(n));
|
hits[nhits].idx = i;
|
||||||
found++;
|
hits[nhits].score = sc;
|
||||||
|
hits[nhits].salience = n->salience;
|
||||||
|
nhits++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/* Rank by distinct tokens matched (desc) then salience (desc), then cap.
|
||||||
|
* Pure-semantic hits (token score 0) sort after every lexical hit — a
|
||||||
|
* lexical ∪ semantic union with lexical precedence. */
|
||||||
|
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||||
|
int64_t end = nhits < lim ? nhits : lim;
|
||||||
|
for (int64_t k = 0; k < end; k++) {
|
||||||
|
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
||||||
|
}
|
||||||
|
free(hits);
|
||||||
|
free(qvec);
|
||||||
return lst;
|
return lst;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6883,10 +7232,10 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
|
|||||||
EngramEdge* e = &g->edges[g->edge_count];
|
EngramEdge* e = &g->edges[g->edge_count];
|
||||||
memset(e, 0, sizeof(*e));
|
memset(e, 0, sizeof(*e));
|
||||||
e->id = engram_new_id();
|
e->id = engram_new_id();
|
||||||
e->from_id = el_strdup(f);
|
e->from_id = el_strdup_persist(f);
|
||||||
e->to_id = el_strdup(t);
|
e->to_id = el_strdup_persist(t);
|
||||||
e->relation = el_strdup(r && *r ? r : "associate");
|
e->relation = el_strdup_persist(r && *r ? r : "associate");
|
||||||
e->metadata = el_strdup("{}");
|
e->metadata = el_strdup_persist("{}");
|
||||||
e->weight = engram_decode_score(weight);
|
e->weight = engram_decode_score(weight);
|
||||||
if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5;
|
if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5;
|
||||||
e->confidence = 1.0;
|
e->confidence = 1.0;
|
||||||
@@ -7106,10 +7455,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
|
|||||||
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
||||||
if (!query || !*query) return 1.0;
|
if (!query || !*query) return 1.0;
|
||||||
double bias = 1.0;
|
double bias = 1.0;
|
||||||
/* Direct lexical overlap: node content/label/tags share text with query. */
|
/* Direct lexical overlap, graded by token coverage: a node covering all
|
||||||
if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
|
* query tokens gets the full +0.5; partial coverage gets a proportional
|
||||||
istr_contains(n->tags, query)) {
|
* share. Single-token queries → full +0.5 on match, identical to before. */
|
||||||
bias += 0.5;
|
{
|
||||||
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
|
int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
|
||||||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
|
if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
|
||||||
}
|
}
|
||||||
/* Node-type resonance with query intent. */
|
/* Node-type resonance with query intent. */
|
||||||
int technical_query = istr_contains(query, "code") ||
|
int technical_query = istr_contains(query, "code") ||
|
||||||
@@ -7175,14 +7528,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
|||||||
if (!seeds) {
|
if (!seeds) {
|
||||||
free(best_bg); free(best_hops); free(reached); return out;
|
free(best_bg); free(best_hops); free(reached); return out;
|
||||||
}
|
}
|
||||||
|
/* Tokenized + semantic seeding: a node seeds if it covers >=1 query token
|
||||||
|
* (tokenized-lexical, #66) OR its cosine clears the threshold (#67). A
|
||||||
|
* lexical seed's activation is scaled by token coverage (fraction of
|
||||||
|
* distinct query tokens covered) so a node matching all words seeds more
|
||||||
|
* strongly than one matching a single word; single-word queries → coverage
|
||||||
|
* 1.0. A pure-semantic seed (no token match) is instead down-weighted by
|
||||||
|
* its cosine so paraphrase matches spread without overpowering exact seeds.
|
||||||
|
* q_vec is NULL (cosine 0) when semantic is unavailable → the seed set is
|
||||||
|
* exactly the tokenized-lexical one. q_vec is freed right after this loop
|
||||||
|
* so the many downstream early-returns need no cleanup change. */
|
||||||
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||||
|
int q_dim = 0;
|
||||||
|
float* q_vec = engram_embed_query(q, &q_dim);
|
||||||
|
double q_sem_min = engram_semantic_min();
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
if (istr_contains(n->content, q) ||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
istr_contains(n->label, q) ||
|
double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
|
||||||
istr_contains(n->tags, q)) {
|
if (sc > 0 || sem >= q_sem_min) {
|
||||||
double tdecay = engram_temporal_decay(n, now_ms);
|
double tdecay = engram_temporal_decay(n, now_ms);
|
||||||
double dampen = engram_activation_dampen(n);
|
double dampen = engram_activation_dampen(n);
|
||||||
double act = n->salience * tdecay * dampen;
|
double act = n->salience * tdecay * dampen;
|
||||||
|
if (sc > 0) act *= (ntok > 0 ? (double)sc / (double)ntok : 1.0);
|
||||||
|
else act *= sem; /* pure-semantic seed: down-weight by cosine */
|
||||||
seeds[seed_count].idx = i;
|
seeds[seed_count].idx = i;
|
||||||
seeds[seed_count].act = act;
|
seeds[seed_count].act = act;
|
||||||
seeds[seed_count].created_at = n->created_at;
|
seeds[seed_count].created_at = n->created_at;
|
||||||
@@ -7192,6 +7562,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
|||||||
reached[i] = 1;
|
reached[i] = 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
free(q_vec);
|
||||||
/* Compute mean seed created_at for temporal proximity bonus. */
|
/* Compute mean seed created_at for temporal proximity bonus. */
|
||||||
int64_t seed_epoch = 0;
|
int64_t seed_epoch = 0;
|
||||||
if (seed_count > 0) {
|
if (seed_count > 0) {
|
||||||
@@ -7519,11 +7890,11 @@ el_val_t engram_save(el_val_t path) {
|
|||||||
/* Helper: extract a string field from a JSON object substring. */
|
/* Helper: extract a string field from a JSON object substring. */
|
||||||
static char* eg_get_str_field(const char* obj, const char* key) {
|
static char* eg_get_str_field(const char* obj, const char* key) {
|
||||||
const char* p = json_find_key(obj, key);
|
const char* p = json_find_key(obj, key);
|
||||||
if (!p) return el_strdup("");
|
if (!p) return el_strdup_persist("");
|
||||||
if (*p != '"') return el_strdup("");
|
if (*p != '"') return el_strdup_persist("");
|
||||||
JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 };
|
JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 };
|
||||||
char* out = jp_parse_string_raw(&jp);
|
char* out = jp_parse_string_raw(&jp);
|
||||||
if (jp.err) { free(out); return el_strdup(""); }
|
if (jp.err) { free(out); return el_strdup_persist(""); }
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7597,7 +7968,7 @@ el_val_t engram_load(el_val_t path) {
|
|||||||
nn->tier = eg_get_str_field(obj, "tier");
|
nn->tier = eg_get_str_field(obj, "tier");
|
||||||
nn->tags = eg_get_str_field(obj, "tags");
|
nn->tags = eg_get_str_field(obj, "tags");
|
||||||
nn->metadata = eg_get_str_field(obj, "metadata");
|
nn->metadata = eg_get_str_field(obj, "metadata");
|
||||||
if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup("{}"); }
|
if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); }
|
||||||
nn->salience = eg_get_num_field(obj, "salience");
|
nn->salience = eg_get_num_field(obj, "salience");
|
||||||
nn->importance = eg_get_num_field(obj, "importance");
|
nn->importance = eg_get_num_field(obj, "importance");
|
||||||
nn->confidence = eg_get_num_field(obj, "confidence");
|
nn->confidence = eg_get_num_field(obj, "confidence");
|
||||||
@@ -7648,7 +8019,7 @@ el_val_t engram_load(el_val_t path) {
|
|||||||
ee->to_id = eg_get_str_field(obj, "to_id");
|
ee->to_id = eg_get_str_field(obj, "to_id");
|
||||||
ee->relation = eg_get_str_field(obj, "relation");
|
ee->relation = eg_get_str_field(obj, "relation");
|
||||||
ee->metadata = eg_get_str_field(obj, "metadata");
|
ee->metadata = eg_get_str_field(obj, "metadata");
|
||||||
if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup("{}"); }
|
if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); }
|
||||||
ee->weight = eg_get_num_field(obj, "weight");
|
ee->weight = eg_get_num_field(obj, "weight");
|
||||||
ee->confidence = eg_get_num_field(obj, "confidence");
|
ee->confidence = eg_get_num_field(obj, "confidence");
|
||||||
ee->created_at = eg_get_int_field(obj, "created_at");
|
ee->created_at = eg_get_int_field(obj, "created_at");
|
||||||
@@ -7740,7 +8111,36 @@ el_val_t engram_get_node_json(el_val_t id) {
|
|||||||
if (!n) return el_wrap_str(el_strdup("{}"));
|
if (!n) return el_wrap_str(el_strdup("{}"));
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
engram_emit_node_json(&b, n);
|
engram_emit_node_json(&b, n);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* engram_get_node_by_label — find the first node whose label field exactly
|
||||||
|
* matches the given string. Returns the node as a JSON object string, or "{}"
|
||||||
|
* if no match is found.
|
||||||
|
*
|
||||||
|
* Used by chat.el to retrieve well-known nodes (e.g. "conv:history",
|
||||||
|
* "session:summary") by their stable label rather than by ID, which is immune
|
||||||
|
* to vector index drift across restarts.
|
||||||
|
*
|
||||||
|
* Exact match (strcmp, not istr_contains) because labels like "conv:history"
|
||||||
|
* must not collide with nodes whose content happens to contain that substring.
|
||||||
|
*
|
||||||
|
* Backported verbatim (idiom-adapted to jb_finish) from release runtime
|
||||||
|
* v1.0.0-20260501 to unblock the soul regen link: chat.el references this
|
||||||
|
* native but the current runtime lacked its definition. */
|
||||||
|
el_val_t engram_get_node_by_label(el_val_t label) {
|
||||||
|
const char* lbl = EL_CSTR(label);
|
||||||
|
if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}"));
|
||||||
|
EngramStore* g = engram_get();
|
||||||
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
|
EngramNode* n = &g->nodes[i];
|
||||||
|
if (n->label && strcmp(n->label, lbl) == 0) {
|
||||||
|
JsonBuf b; jb_init(&b);
|
||||||
|
engram_emit_node_json(&b, n);
|
||||||
|
return el_wrap_str(jb_finish(&b));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return el_wrap_str(el_strdup("{}"));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||||
@@ -7750,25 +8150,53 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
|||||||
if (lim <= 0) lim = 100;
|
if (lim <= 0) lim = 100;
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
int first = 1;
|
if (q && *q && g->node_count > 0) {
|
||||||
int64_t found = 0;
|
/* Collect candidates from the UNION of tokenized-lexical and semantic
|
||||||
if (q && *q) {
|
* matches, score each, rank by score, emit the top `lim`. A node is a
|
||||||
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
* candidate if it covers >=1 query token (tokenized-lexical, #66) OR its
|
||||||
EngramNode* n = &g->nodes[i];
|
* query cosine clears the threshold (#67). Lexical score is the distinct
|
||||||
/* Filter transparent layers — same as engram_search. */
|
* token count (>=1), so any lexical hit outranks a pure-semantic hit
|
||||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
* (cosine < 1); pure-semantic hits are scored by cosine alone. When
|
||||||
if (istr_contains(n->content, q) ||
|
* semantic is unavailable qvec is NULL, sem is 0, only tokenized-lexical
|
||||||
istr_contains(n->label, q) ||
|
* hits are collected, and the stable insertion sort preserves order. */
|
||||||
istr_contains(n->tags, q)) {
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
if (!first) jb_putc(&b, ',');
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||||
engram_emit_node_json(&b, n);
|
int qdim = 0;
|
||||||
first = 0;
|
float* qvec = engram_embed_query(q, &qdim);
|
||||||
found++;
|
double sem_min = engram_semantic_min();
|
||||||
|
typedef struct { int64_t idx; double score; } Cand;
|
||||||
|
Cand* cand = malloc((size_t)g->node_count * sizeof(Cand));
|
||||||
|
if (cand) {
|
||||||
|
int64_t nc = 0;
|
||||||
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
|
EngramNode* n = &g->nodes[i];
|
||||||
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
|
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
||||||
|
if (sc > 0 || sem >= sem_min) {
|
||||||
|
cand[nc].idx = i;
|
||||||
|
cand[nc].score = (double)sc + sem;
|
||||||
|
nc++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
/* Insertion sort by score desc; stable for equal scores. */
|
||||||
|
for (int64_t i = 1; i < nc; i++) {
|
||||||
|
Cand k = cand[i]; int64_t j = i - 1;
|
||||||
|
while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; }
|
||||||
|
cand[j + 1] = k;
|
||||||
|
}
|
||||||
|
int first = 1;
|
||||||
|
for (int64_t i = 0; i < nc && i < lim; i++) {
|
||||||
|
if (!first) jb_putc(&b, ',');
|
||||||
|
engram_emit_node_json(&b, &g->nodes[cand[i].idx]);
|
||||||
|
first = 0;
|
||||||
|
}
|
||||||
|
free(cand);
|
||||||
}
|
}
|
||||||
|
free(qvec);
|
||||||
}
|
}
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
|
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
|
||||||
@@ -7777,9 +8205,9 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
|
|||||||
int64_t off = (int64_t)offset; if (off < 0) off = 0;
|
int64_t off = (int64_t)offset; if (off < 0) off = 0;
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
|
||||||
int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
|
int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
|
||||||
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
|
||||||
/* Skip transparent layers — introspection filter, same as engram_scan_nodes. */
|
/* Skip transparent layers — introspection filter, same as engram_scan_nodes. */
|
||||||
int64_t live = 0;
|
int64_t live = 0;
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
@@ -7797,7 +8225,7 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
|
|||||||
}
|
}
|
||||||
free(idx);
|
free(idx);
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* engram_scan_nodes_by_type_json — filter by node_type before paginating.
|
/* engram_scan_nodes_by_type_json — filter by node_type before paginating.
|
||||||
@@ -7813,9 +8241,9 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_
|
|||||||
int64_t off = (int64_t)offset; if (off < 0) off = 0;
|
int64_t off = (int64_t)offset; if (off < 0) off = 0;
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
|
||||||
int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
|
int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
|
||||||
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
|
||||||
int64_t live = 0;
|
int64_t live = 0;
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
|
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
|
||||||
@@ -7834,7 +8262,7 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_
|
|||||||
}
|
}
|
||||||
free(idx);
|
free(idx);
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
|
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
|
||||||
@@ -7849,7 +8277,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
|
|||||||
int allow_in = (strcmp(dir, "in") == 0) || (strcmp(dir, "both") == 0);
|
int allow_in = (strcmp(dir, "in") == 0) || (strcmp(dir, "both") == 0);
|
||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
|
||||||
|
|
||||||
/* Frontier of (node_id, hops). Cap to a sane size. */
|
/* Frontier of (node_id, hops). Cap to a sane size. */
|
||||||
char** frontier = calloc(1024, sizeof(char*));
|
char** frontier = calloc(1024, sizeof(char*));
|
||||||
@@ -7859,7 +8287,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
|
|||||||
int64_t vc = 0;
|
int64_t vc = 0;
|
||||||
if (!frontier || !frontier_h || !visited) {
|
if (!frontier || !frontier_h || !visited) {
|
||||||
free(frontier); free(frontier_h); free(visited);
|
free(frontier); free(frontier_h); free(visited);
|
||||||
jb_putc(&b, ']'); return el_wrap_str(b.buf);
|
jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
/* Use plain strdup (not el_strdup) so arena doesn't track these pointers.
|
/* Use plain strdup (not el_strdup) so arena doesn't track these pointers.
|
||||||
* The BFS loop manually frees them below — arena would double-free them. */
|
* The BFS loop manually frees them below — arena would double-free them. */
|
||||||
@@ -7902,7 +8330,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
|
|||||||
for (int64_t i = 0; i < vc; i++) free(visited[i]);
|
for (int64_t i = 0; i < vc; i++) free(visited[i]);
|
||||||
free(frontier); free(frontier_h); free(visited);
|
free(frontier); free(frontier_h); free(visited);
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
|
el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
|
||||||
@@ -7943,7 +8371,7 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t engram_stats_json(void) {
|
el_val_t engram_stats_json(void) {
|
||||||
@@ -7963,7 +8391,7 @@ el_val_t engram_list_layers_json(void) {
|
|||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
/* Build a sorted index over live layers. */
|
/* Build a sorted index over live layers. */
|
||||||
size_t* idx = malloc((g->layer_count + 1) * sizeof(size_t));
|
size_t* idx = malloc((g->layer_count + 1) * sizeof(size_t));
|
||||||
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
|
||||||
size_t live = 0;
|
size_t live = 0;
|
||||||
for (size_t i = 0; i < g->layer_count; i++) {
|
for (size_t i = 0; i < g->layer_count; i++) {
|
||||||
if (g->layers[i].name) idx[live++] = i;
|
if (g->layers[i].name) idx[live++] = i;
|
||||||
@@ -8000,7 +8428,7 @@ el_val_t engram_list_layers_json(void) {
|
|||||||
}
|
}
|
||||||
free(idx);
|
free(idx);
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* engram_compile_layered_json — produce a prompt-ready context block split
|
/* engram_compile_layered_json — produce a prompt-ready context block split
|
||||||
@@ -8086,7 +8514,7 @@ el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth) {
|
|||||||
free(b.buf);
|
free(b.buf);
|
||||||
return el_wrap_str(el_strdup(""));
|
return el_wrap_str(el_strdup(""));
|
||||||
}
|
}
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* engram_query_range — temporal range query.
|
/* engram_query_range — temporal range query.
|
||||||
@@ -8131,7 +8559,7 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) {
|
|||||||
}
|
}
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
free(idx);
|
free(idx);
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* engram_load_merge — like engram_load but WITHOUT resetting the store.
|
/* engram_load_merge — like engram_load but WITHOUT resetting the store.
|
||||||
@@ -8382,7 +8810,7 @@ el_val_t engram_wm_top_json(el_val_t n_v) {
|
|||||||
}
|
}
|
||||||
free(idx);
|
free(idx);
|
||||||
jb_putc(&b, ']');
|
jb_putc(&b, ']');
|
||||||
return el_wrap_str(b.buf);
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef HAVE_CURL
|
#ifdef HAVE_CURL
|
||||||
@@ -8778,13 +9206,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr
|
|||||||
engram_grow_nodes();
|
engram_grow_nodes();
|
||||||
EngramNode* n = &g->nodes[g->node_count];
|
EngramNode* n = &g->nodes[g->node_count];
|
||||||
memset(n, 0, sizeof(*n));
|
memset(n, 0, sizeof(*n));
|
||||||
n->id = el_strdup(self_id);
|
n->id = el_strdup_persist(self_id);
|
||||||
n->content = el_strdup(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)");
|
n->content = el_strdup_persist(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)");
|
||||||
n->node_type = el_strdup("DharmaSelf");
|
n->node_type = el_strdup_persist("DharmaSelf");
|
||||||
n->label = el_strdup("dharma:self");
|
n->label = el_strdup_persist("dharma:self");
|
||||||
n->tier = el_strdup("Working");
|
n->tier = el_strdup_persist("Working");
|
||||||
n->tags = el_strdup("dharma");
|
n->tags = el_strdup_persist("dharma");
|
||||||
n->metadata = el_strdup("{}");
|
n->metadata = el_strdup_persist("{}");
|
||||||
n->salience = 1.0; n->importance = 1.0; n->confidence = 1.0;
|
n->salience = 1.0; n->importance = 1.0; n->confidence = 1.0;
|
||||||
int64_t now = engram_now_ms();
|
int64_t now = engram_now_ms();
|
||||||
n->created_at = now; n->updated_at = now; n->last_activated = now;
|
n->created_at = now; n->updated_at = now; n->last_activated = now;
|
||||||
@@ -8795,13 +9223,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr
|
|||||||
engram_grow_nodes();
|
engram_grow_nodes();
|
||||||
EngramNode* n = &g->nodes[g->node_count];
|
EngramNode* n = &g->nodes[g->node_count];
|
||||||
memset(n, 0, sizeof(*n));
|
memset(n, 0, sizeof(*n));
|
||||||
n->id = el_strdup(peer_node);
|
n->id = el_strdup_persist(peer_node);
|
||||||
n->content = el_strdup(peer_base);
|
n->content = el_strdup_persist(peer_base);
|
||||||
n->node_type = el_strdup("DharmaPeer");
|
n->node_type = el_strdup_persist("DharmaPeer");
|
||||||
n->label = el_strdup(peer_node);
|
n->label = el_strdup_persist(peer_node);
|
||||||
n->tier = el_strdup("Working");
|
n->tier = el_strdup_persist("Working");
|
||||||
n->tags = el_strdup("dharma");
|
n->tags = el_strdup_persist("dharma");
|
||||||
n->metadata = el_strdup("{}");
|
n->metadata = el_strdup_persist("{}");
|
||||||
n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0;
|
n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0;
|
||||||
int64_t now = engram_now_ms();
|
int64_t now = engram_now_ms();
|
||||||
n->created_at = now; n->updated_at = now; n->last_activated = now;
|
n->created_at = now; n->updated_at = now; n->last_activated = now;
|
||||||
@@ -8813,10 +9241,10 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr
|
|||||||
EngramEdge* e = &g->edges[g->edge_count];
|
EngramEdge* e = &g->edges[g->edge_count];
|
||||||
memset(e, 0, sizeof(*e));
|
memset(e, 0, sizeof(*e));
|
||||||
e->id = engram_new_id();
|
e->id = engram_new_id();
|
||||||
e->from_id = el_strdup(self_id);
|
e->from_id = el_strdup_persist(self_id);
|
||||||
e->to_id = el_strdup(peer_node);
|
e->to_id = el_strdup_persist(peer_node);
|
||||||
e->relation = el_strdup("dharma-relation");
|
e->relation = el_strdup_persist("dharma-relation");
|
||||||
e->metadata = el_strdup("{}");
|
e->metadata = el_strdup_persist("{}");
|
||||||
e->weight = 0.0;
|
e->weight = 0.0;
|
||||||
e->confidence = 1.0;
|
e->confidence = 1.0;
|
||||||
int64_t now = engram_now_ms();
|
int64_t now = engram_now_ms();
|
||||||
@@ -9109,7 +9537,7 @@ static el_val_t llm_extract_text(el_val_t resp_val) {
|
|||||||
free(obj);
|
free(obj);
|
||||||
p = end;
|
p = end;
|
||||||
}
|
}
|
||||||
return el_wrap_str(out.buf);
|
return el_wrap_str(jb_finish(&out));
|
||||||
}
|
}
|
||||||
|
|
||||||
el_val_t llm_call(el_val_t model, el_val_t prompt) {
|
el_val_t llm_call(el_val_t model, el_val_t prompt) {
|
||||||
|
|||||||
@@ -632,6 +632,7 @@ el_val_t engram_load(el_val_t path);
|
|||||||
* can pass results straight through without round-tripping ElList/ElMap
|
* can pass results straight through without round-tripping ElList/ElMap
|
||||||
* through json_stringify. */
|
* through json_stringify. */
|
||||||
el_val_t engram_get_node_json(el_val_t id);
|
el_val_t engram_get_node_json(el_val_t id);
|
||||||
|
el_val_t engram_get_node_by_label(el_val_t label);
|
||||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||||
|
|||||||
@@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path) { return engram_save
|
|||||||
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
||||||
|
|
||||||
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
||||||
|
el_val_t __engram_get_node_by_label(el_val_t label) { return engram_get_node_by_label(label); }
|
||||||
|
|
||||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
||||||
return engram_search_json(query, limit);
|
return engram_search_json(query, limit);
|
||||||
|
|||||||
@@ -226,6 +226,7 @@ el_val_t __engram_activate(el_val_t query, el_val_t depth);
|
|||||||
el_val_t __engram_save(el_val_t path);
|
el_val_t __engram_save(el_val_t path);
|
||||||
el_val_t __engram_load(el_val_t path);
|
el_val_t __engram_load(el_val_t path);
|
||||||
el_val_t __engram_get_node_json(el_val_t id);
|
el_val_t __engram_get_node_json(el_val_t id);
|
||||||
|
el_val_t __engram_get_node_by_label(el_val_t label);
|
||||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
||||||
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||||
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||||
|
|||||||
@@ -2670,6 +2670,7 @@ fn builtin_arity(name: String) -> Int {
|
|||||||
if str_eq(name, "engram_save") { return 1 }
|
if str_eq(name, "engram_save") { return 1 }
|
||||||
if str_eq(name, "engram_load") { return 1 }
|
if str_eq(name, "engram_load") { return 1 }
|
||||||
if str_eq(name, "engram_get_node_json") { return 1 }
|
if str_eq(name, "engram_get_node_json") { return 1 }
|
||||||
|
if str_eq(name, "engram_get_node_by_label") { return 1 }
|
||||||
if str_eq(name, "engram_search_json") { return 2 }
|
if str_eq(name, "engram_search_json") { return 2 }
|
||||||
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
||||||
if str_eq(name, "engram_neighbors_json") { return 3 }
|
if str_eq(name, "engram_neighbors_json") { return 3 }
|
||||||
|
|||||||
@@ -23,10 +23,29 @@ fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn tok_kind(tokens: [Any], pos: Int) -> String {
|
fn tok_kind(tokens: [Any], pos: Int) -> String {
|
||||||
|
// Out-of-range reads must report the Eof sentinel so every `== "Eof"`
|
||||||
|
// termination guard in the parser fires. Without this, reading past the
|
||||||
|
// single trailing Eof token returns runtime null (el_list_get OOB -> 0),
|
||||||
|
// which matches no delimiter, letting inner parse loops append AST nodes
|
||||||
|
// forever on malformed input -> unbounded allocation -> OOM.
|
||||||
|
let n: Int = native_list_len(tokens) / 2
|
||||||
|
if pos < 0 {
|
||||||
|
return "Eof"
|
||||||
|
}
|
||||||
|
if pos >= n {
|
||||||
|
return "Eof"
|
||||||
|
}
|
||||||
native_list_get(tokens, pos * 2)
|
native_list_get(tokens, pos * 2)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn tok_value(tokens: [Any], pos: Int) -> String {
|
fn tok_value(tokens: [Any], pos: Int) -> String {
|
||||||
|
let n: Int = native_list_len(tokens) / 2
|
||||||
|
if pos < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if pos >= n {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
native_list_get(tokens, pos * 2 + 1)
|
native_list_get(tokens, pos * 2 + 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -35,7 +54,12 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
|
|||||||
if k == kind {
|
if k == kind {
|
||||||
return pos + 1
|
return pos + 1
|
||||||
}
|
}
|
||||||
// On mismatch just advance; error recovery is best-effort
|
// On mismatch, error recovery is best-effort. But never step PAST the Eof
|
||||||
|
// sentinel: once at Eof a mismatch means the input ended early, and
|
||||||
|
// advancing would run the cursor off the token list.
|
||||||
|
if k == "Eof" {
|
||||||
|
return pos
|
||||||
|
}
|
||||||
pos + 1
|
pos + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user