Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bcb356fe69 | |||
| dd7827059a | |||
| 155a449c4e | |||
| 4696fd6833 | |||
| 581a351fb1 | |||
| 8ce8656de2 | |||
| 1e49560f1f | |||
| e8f0b5a9de | |||
| 40287c4cfc | |||
| 0481bea44d | |||
| 9d565ca080 | |||
| 4773dd0aa2 | |||
| 6b9d9e6c4a | |||
| e3dabe3e08 | |||
| 2b2a1246e7 |
@@ -214,9 +214,18 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
|
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||||
|
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||||
|
# and the SDK silently never published. Surface failures now.
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||||
|
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||||
gcloud config set project neuron-785695
|
gcloud config set project neuron-785695
|
||||||
|
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||||
|
|
||||||
VERSION="${GITHUB_SHA:0:8}"
|
VERSION="${GITHUB_SHA:0:8}"
|
||||||
|
|
||||||
@@ -268,6 +277,12 @@ jobs:
|
|||||||
# Patches ci-base:dev in-place: pulls the existing image (which has all
|
# Patches ci-base:dev in-place: pulls the existing image (which has all
|
||||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||||
|
#
|
||||||
|
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||||
|
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||||
|
# runner where DinD/Docker availability is fragile. A failure here must
|
||||||
|
# never block or redden the job — the SDK publish above is the deliverable.
|
||||||
|
continue-on-error: true
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
|
|||||||
@@ -212,12 +212,21 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
|
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||||
|
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||||
|
# and the SDK silently never published. Surface failures now.
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||||
|
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||||
apt-get install -y -qq apt-transport-https ca-certificates curl
|
apt-get install -y -qq apt-transport-https ca-certificates curl
|
||||||
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||||
gcloud config set project neuron-785695
|
gcloud config set project neuron-785695
|
||||||
|
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||||
|
|
||||||
VERSION="${GITHUB_SHA:0:8}"
|
VERSION="${GITHUB_SHA:0:8}"
|
||||||
|
|
||||||
@@ -253,6 +262,12 @@ jobs:
|
|||||||
# Patches ci-base:stage in-place: pulls the existing image (which has all
|
# Patches ci-base:stage in-place: pulls the existing image (which has all
|
||||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||||
|
#
|
||||||
|
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||||
|
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||||
|
# runner where DinD/Docker availability is fragile. A failure here must
|
||||||
|
# never block or redden the job — the SDK publish above is the deliverable.
|
||||||
|
continue-on-error: true
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
|
|||||||
@@ -288,12 +288,21 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
run: |
|
run: |
|
||||||
|
# Fail loudly: previously this step had no `set -e`, so an auth or
|
||||||
|
# upload failure was swallowed (step exited 0 on the trailing echo)
|
||||||
|
# and the SDK silently never published. Surface failures now.
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${GCP_SA_KEY:-}" ]; then
|
||||||
|
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
|
||||||
apt-get install -y -qq apt-transport-https ca-certificates curl
|
apt-get install -y -qq apt-transport-https ca-certificates curl
|
||||||
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||||
apt-get update -qq && apt-get install -y google-cloud-cli
|
apt-get update -qq && apt-get install -y google-cloud-cli
|
||||||
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
|
||||||
gcloud config set project neuron-785695
|
gcloud config set project neuron-785695
|
||||||
|
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
|
||||||
|
|
||||||
VERSION="${GITHUB_SHA:0:8}"
|
VERSION="${GITHUB_SHA:0:8}"
|
||||||
|
|
||||||
@@ -345,6 +354,12 @@ jobs:
|
|||||||
# Patches ci-base:latest in-place: pulls the existing image (which has all
|
# Patches ci-base:latest in-place: pulls the existing image (which has all
|
||||||
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
|
||||||
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
|
||||||
|
#
|
||||||
|
# continue-on-error: this is a CI-cache optimization, NOT the release
|
||||||
|
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
|
||||||
|
# runner where DinD/Docker availability is fragile. A failure here must
|
||||||
|
# never block or redden the job — the SDK publish above is the deliverable.
|
||||||
|
continue-on-error: true
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
env:
|
env:
|
||||||
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
|
||||||
|
|||||||
@@ -17,6 +17,16 @@
|
|||||||
// 4. Append dep to order after all its transitive deps
|
// 4. Append dep to order after all its transitive deps
|
||||||
// 5. Deduplicate: skip already-ordered vessels
|
// 5. Deduplicate: skip already-ordered vessels
|
||||||
|
|
||||||
|
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||||
|
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
|
||||||
|
// give elc the C prototypes so generated install.c compiles cleanly under strict
|
||||||
|
// compilers (gcc>=14 / clang) that reject implicit function declarations.
|
||||||
|
extern fn manifest_name(src: String) -> String // manifest.el
|
||||||
|
extern fn manifest_deps(src: String) -> String // manifest.el
|
||||||
|
extern fn registry_token() -> String // registry.el
|
||||||
|
extern fn registry_find(name: String, version: String) -> String // registry.el
|
||||||
|
extern fn registry_latest_version(name: String) -> String // registry.el
|
||||||
|
|
||||||
// ── Install paths ─────────────────────────────────────────────────────────────
|
// ── Install paths ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// packages_dir returns the root directory for installed vessels.
|
// packages_dir returns the root directory for installed vessels.
|
||||||
|
|||||||
@@ -14,6 +14,15 @@
|
|||||||
// EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies)
|
// EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies)
|
||||||
// EPM_TOKEN — Gitea personal access token (required for publish)
|
// EPM_TOKEN — Gitea personal access token (required for publish)
|
||||||
|
|
||||||
|
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||||
|
// These symbols are defined in sibling epm modules or the El runtime and are
|
||||||
|
// resolved at link time. The `extern fn` decls give elc the C prototype so the
|
||||||
|
// generated registry.c compiles cleanly under strict compilers (gcc>=14 / clang)
|
||||||
|
// that reject implicit function declarations. Signature arity must match the
|
||||||
|
// definition; return/param types are informational (all lower to el_val_t).
|
||||||
|
extern fn config(key: String) -> String // El runtime builtin
|
||||||
|
extern fn read_installed() -> String // install.el
|
||||||
|
|
||||||
// ── Config helpers ────────────────────────────────────────────────────────────
|
// ── Config helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// registry_api_url returns the Gitea API base URL with no trailing slash.
|
// registry_api_url returns the Gitea API base URL with no trailing slash.
|
||||||
|
|||||||
@@ -6,6 +6,15 @@
|
|||||||
// Depends on: registry.el (registry_latest_version, registry_find),
|
// Depends on: registry.el (registry_latest_version, registry_find),
|
||||||
// install.el (read_installed, install_vessel, installed_version)
|
// install.el (read_installed, install_vessel, installed_version)
|
||||||
|
|
||||||
|
// ── Cross-module forward declarations ─────────────────────────────────────────
|
||||||
|
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
|
||||||
|
// give elc the C prototypes so generated update.c compiles cleanly under strict
|
||||||
|
// compilers (gcc>=14 / clang) that reject implicit function declarations.
|
||||||
|
extern fn read_installed() -> String // install.el
|
||||||
|
extern fn installed_version(name: String) -> String // install.el
|
||||||
|
extern fn install_vessel(name: String, version: String) -> Bool // install.el
|
||||||
|
extern fn registry_latest_version(name: String) -> String // registry.el
|
||||||
|
|
||||||
// ── Semver helpers ────────────────────────────────────────────────────────────
|
// ── Semver helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// semver_part extracts the Nth dot-separated component from a semver string.
|
// semver_part extracts the Nth dot-separated component from a semver string.
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) {
|
|||||||
#include <direct.h> /* _mkdir */
|
#include <direct.h> /* _mkdir */
|
||||||
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
||||||
#define timegm _mkgmtime /* UTC tm → time_t */
|
#define timegm _mkgmtime /* UTC tm → time_t */
|
||||||
|
#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() (<io.h>) is the equiv */
|
||||||
|
|
||||||
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
||||||
static inline int setenv(const char* name, const char* value, int overwrite) {
|
static inline int setenv(const char* name, const char* value, int overwrite) {
|
||||||
|
|||||||
@@ -1963,8 +1963,9 @@ void http_serve_async(el_val_t port, el_val_t handler) {
|
|||||||
int sock = socket(AF_INET6, SOCK_STREAM, 0);
|
int sock = socket(AF_INET6, SOCK_STREAM, 0);
|
||||||
if (sock < 0) { perror("socket"); return; }
|
if (sock < 0) { perror("socket"); return; }
|
||||||
int yes = 1; int no = 0;
|
int yes = 1; int no = 0;
|
||||||
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
/* Win32/mingw setsockopt takes optval as (const char*); the cast is portable on POSIX too. */
|
||||||
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no));
|
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));
|
||||||
|
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no));
|
||||||
struct sockaddr_in6 addr;
|
struct sockaddr_in6 addr;
|
||||||
memset(&addr, 0, sizeof(addr));
|
memset(&addr, 0, sizeof(addr));
|
||||||
addr.sin6_family = AF_INET6;
|
addr.sin6_family = AF_INET6;
|
||||||
@@ -6826,6 +6827,75 @@ static int istr_contains(const char* hay, const char* needle) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── Tokenized query matching ───────────────────────────────────────────
|
||||||
|
* The engram query surface (search / activate / goal-bias) historically
|
||||||
|
* matched the ENTIRE raw query string as a single case-insensitive
|
||||||
|
* substring via istr_contains(field, q). That is Ctrl-F, not search:
|
||||||
|
* a multi-word query like "windows msi signing" only matched a node whose
|
||||||
|
* text contained that exact contiguous run, so real multi-word queries
|
||||||
|
* returned zero. istr_contains stays as the per-TOKEN primitive; these
|
||||||
|
* helpers split the query on whitespace and match ANY token, then rank by
|
||||||
|
* how many DISTINCT tokens a node covers. Single-token queries are a strict
|
||||||
|
* special case (score is 0 or 1) so single-word callers never regress. */
|
||||||
|
#define ENGRAM_MAX_QTOKENS 32
|
||||||
|
#define ENGRAM_QTOK_LEN 256
|
||||||
|
|
||||||
|
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
|
||||||
|
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
|
||||||
|
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
|
||||||
|
static int engram_tokenize_query(const char* q,
|
||||||
|
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
|
||||||
|
int n = 0;
|
||||||
|
if (!q) return 0;
|
||||||
|
const char* p = q;
|
||||||
|
while (*p && n < maxtok) {
|
||||||
|
while (*p && isspace((unsigned char)*p)) p++;
|
||||||
|
if (!*p) break;
|
||||||
|
char buf[ENGRAM_QTOK_LEN];
|
||||||
|
size_t tl = 0;
|
||||||
|
while (*p && !isspace((unsigned char)*p)) {
|
||||||
|
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
|
||||||
|
p++;
|
||||||
|
}
|
||||||
|
buf[tl] = '\0';
|
||||||
|
if (tl == 0) continue;
|
||||||
|
int dup = 0;
|
||||||
|
for (int s = 0; s < n; s++) {
|
||||||
|
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
|
||||||
|
}
|
||||||
|
if (dup) continue;
|
||||||
|
memcpy(toks[n], buf, tl + 1);
|
||||||
|
n++;
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Count how many of the ntok distinct query tokens appear (case-insensitive)
|
||||||
|
* in the node's content, label, or tags. 0 == no match. */
|
||||||
|
static int engram_node_match_score(const EngramNode* n,
|
||||||
|
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||||
|
int score = 0;
|
||||||
|
for (int t = 0; t < ntok; t++) {
|
||||||
|
if (istr_contains(n->content, toks[t]) ||
|
||||||
|
istr_contains(n->label, toks[t]) ||
|
||||||
|
istr_contains(n->tags, toks[t]))
|
||||||
|
score++;
|
||||||
|
}
|
||||||
|
return score;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Rank entry: distinct-token match count (primary, desc) then salience
|
||||||
|
* (tiebreak, desc). */
|
||||||
|
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||||
|
static int engram_rank_cmp(const void* a, const void* b) {
|
||||||
|
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||||
|
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||||
|
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
|
||||||
|
if (ea->salience < eb->salience) return 1;
|
||||||
|
if (ea->salience > eb->salience) return -1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* ══════════════════════════════════════════════════════════════════════════
|
/* ══════════════════════════════════════════════════════════════════════════
|
||||||
* SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
|
* SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
|
||||||
* ──────────────────────────────────────────────────────────────────────────
|
* ──────────────────────────────────────────────────────────────────────────
|
||||||
@@ -7070,29 +7140,44 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
|||||||
if (lim <= 0) lim = 100;
|
if (lim <= 0) lim = 100;
|
||||||
el_val_t lst = el_list_empty();
|
el_val_t lst = el_list_empty();
|
||||||
if (!q || !*q) return lst;
|
if (!q || !*q) return lst;
|
||||||
/* Semantic augmentation: embed the query once; a node matches if it is a
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
* lexical hit OR its cosine similarity clears the threshold. qvec is NULL
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||||
* (and cosine 0) whenever semantic search is unavailable → pure lexical. */
|
if (ntok == 0) return lst;
|
||||||
|
/* Semantic augmentation: embed the query once; a node is a hit if it covers
|
||||||
|
* >=1 query token (tokenized-lexical, #66) OR its cosine clears the
|
||||||
|
* threshold (#67). qvec is NULL (cosine 0) when semantic is unavailable →
|
||||||
|
* pure tokenized-lexical, byte-identical to the lexical-only behaviour. */
|
||||||
int qdim = 0;
|
int qdim = 0;
|
||||||
float* qvec = engram_embed_query(q, &qdim);
|
float* qvec = engram_embed_query(q, &qdim);
|
||||||
double sem_min = engram_semantic_min();
|
double sem_min = engram_semantic_min();
|
||||||
int64_t found = 0;
|
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||||
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
if (!hits) { free(qvec); return lst; }
|
||||||
|
int64_t nhits = 0;
|
||||||
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
||||||
* shape output but are invisible to introspection ("what do you
|
* shape output but are invisible to introspection ("what do you
|
||||||
* know about yourself"). They still surface via engram_activate
|
* know about yourself"). They still surface via engram_activate
|
||||||
* + engram_compile_layered_json — that's the legitimate path. */
|
* + engram_compile_layered_json — that's the legitimate path. */
|
||||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||||
int lex = istr_contains(n->content, q) ||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
istr_contains(n->label, q) ||
|
|
||||||
istr_contains(n->tags, q);
|
|
||||||
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
||||||
if (lex || sem >= sem_min) {
|
if (sc > 0 || sem >= sem_min) {
|
||||||
lst = el_list_append(lst, engram_node_to_map(n));
|
hits[nhits].idx = i;
|
||||||
found++;
|
hits[nhits].score = sc;
|
||||||
|
hits[nhits].salience = n->salience;
|
||||||
|
nhits++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/* Rank by distinct tokens matched (desc) then salience (desc), then cap.
|
||||||
|
* Pure-semantic hits (token score 0) sort after every lexical hit — a
|
||||||
|
* lexical ∪ semantic union with lexical precedence. */
|
||||||
|
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||||
|
int64_t end = nhits < lim ? nhits : lim;
|
||||||
|
for (int64_t k = 0; k < end; k++) {
|
||||||
|
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
||||||
|
}
|
||||||
|
free(hits);
|
||||||
free(qvec);
|
free(qvec);
|
||||||
return lst;
|
return lst;
|
||||||
}
|
}
|
||||||
@@ -7370,10 +7455,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
|
|||||||
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
||||||
if (!query || !*query) return 1.0;
|
if (!query || !*query) return 1.0;
|
||||||
double bias = 1.0;
|
double bias = 1.0;
|
||||||
/* Direct lexical overlap: node content/label/tags share text with query. */
|
/* Direct lexical overlap, graded by token coverage: a node covering all
|
||||||
if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
|
* query tokens gets the full +0.5; partial coverage gets a proportional
|
||||||
istr_contains(n->tags, query)) {
|
* share. Single-token queries → full +0.5 on match, identical to before. */
|
||||||
bias += 0.5;
|
{
|
||||||
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
|
int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
|
||||||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
|
if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
|
||||||
}
|
}
|
||||||
/* Node-type resonance with query intent. */
|
/* Node-type resonance with query intent. */
|
||||||
int technical_query = istr_contains(query, "code") ||
|
int technical_query = istr_contains(query, "code") ||
|
||||||
@@ -7439,28 +7528,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
|||||||
if (!seeds) {
|
if (!seeds) {
|
||||||
free(best_bg); free(best_hops); free(reached); return out;
|
free(best_bg); free(best_hops); free(reached); return out;
|
||||||
}
|
}
|
||||||
/* Semantic seed augmentation: embed the query once; a node becomes a seed
|
/* Tokenized + semantic seeding: a node seeds if it covers >=1 query token
|
||||||
* if it lexically matches OR its cosine clears the threshold. Semantic-only
|
* (tokenized-lexical, #66) OR its cosine clears the threshold (#67). A
|
||||||
* seeds enter at reduced strength (scaled by cosine) so pure paraphrase
|
* lexical seed's activation is scaled by token coverage (fraction of
|
||||||
* matches spread activation without overpowering exact lexical seeds.
|
* distinct query tokens covered) so a node matching all words seeds more
|
||||||
* qvec is NULL (cosine 0) when semantic search is unavailable → the seed
|
* strongly than one matching a single word; single-word queries → coverage
|
||||||
* set is exactly the pre-existing lexical one. qvec is freed right after
|
* 1.0. A pure-semantic seed (no token match) is instead down-weighted by
|
||||||
* this loop so the many downstream early-returns need no cleanup change. */
|
* its cosine so paraphrase matches spread without overpowering exact seeds.
|
||||||
|
* q_vec is NULL (cosine 0) when semantic is unavailable → the seed set is
|
||||||
|
* exactly the tokenized-lexical one. q_vec is freed right after this loop
|
||||||
|
* so the many downstream early-returns need no cleanup change. */
|
||||||
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||||
int q_dim = 0;
|
int q_dim = 0;
|
||||||
float* q_vec = engram_embed_query(q, &q_dim);
|
float* q_vec = engram_embed_query(q, &q_dim);
|
||||||
double q_sem_min = engram_semantic_min();
|
double q_sem_min = engram_semantic_min();
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
int lex = istr_contains(n->content, q) ||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
istr_contains(n->label, q) ||
|
|
||||||
istr_contains(n->tags, q);
|
|
||||||
double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
|
double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
|
||||||
if (lex || sem >= q_sem_min) {
|
if (sc > 0 || sem >= q_sem_min) {
|
||||||
double tdecay = engram_temporal_decay(n, now_ms);
|
double tdecay = engram_temporal_decay(n, now_ms);
|
||||||
double dampen = engram_activation_dampen(n);
|
double dampen = engram_activation_dampen(n);
|
||||||
double act = n->salience * tdecay * dampen;
|
double act = n->salience * tdecay * dampen;
|
||||||
/* Down-weight pure-semantic seeds by their cosine strength. */
|
if (sc > 0) act *= (ntok > 0 ? (double)sc / (double)ntok : 1.0);
|
||||||
if (!lex) act *= sem;
|
else act *= sem; /* pure-semantic seed: down-weight by cosine */
|
||||||
seeds[seed_count].idx = i;
|
seeds[seed_count].idx = i;
|
||||||
seeds[seed_count].act = act;
|
seeds[seed_count].act = act;
|
||||||
seeds[seed_count].created_at = n->created_at;
|
seeds[seed_count].created_at = n->created_at;
|
||||||
@@ -8022,6 +8114,35 @@ el_val_t engram_get_node_json(el_val_t id) {
|
|||||||
return el_wrap_str(jb_finish(&b));
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* engram_get_node_by_label — find the first node whose label field exactly
|
||||||
|
* matches the given string. Returns the node as a JSON object string, or "{}"
|
||||||
|
* if no match is found.
|
||||||
|
*
|
||||||
|
* Used by chat.el to retrieve well-known nodes (e.g. "conv:history",
|
||||||
|
* "session:summary") by their stable label rather than by ID, which is immune
|
||||||
|
* to vector index drift across restarts.
|
||||||
|
*
|
||||||
|
* Exact match (strcmp, not istr_contains) because labels like "conv:history"
|
||||||
|
* must not collide with nodes whose content happens to contain that substring.
|
||||||
|
*
|
||||||
|
* Backported verbatim (idiom-adapted to jb_finish) from release runtime
|
||||||
|
* v1.0.0-20260501 to unblock the soul regen link: chat.el references this
|
||||||
|
* native but the current runtime lacked its definition. */
|
||||||
|
el_val_t engram_get_node_by_label(el_val_t label) {
|
||||||
|
const char* lbl = EL_CSTR(label);
|
||||||
|
if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}"));
|
||||||
|
EngramStore* g = engram_get();
|
||||||
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
|
EngramNode* n = &g->nodes[i];
|
||||||
|
if (n->label && strcmp(n->label, lbl) == 0) {
|
||||||
|
JsonBuf b; jb_init(&b);
|
||||||
|
engram_emit_node_json(&b, n);
|
||||||
|
return el_wrap_str(jb_finish(&b));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return el_wrap_str(el_strdup("{}"));
|
||||||
|
}
|
||||||
|
|
||||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||||
EngramStore* g = engram_get();
|
EngramStore* g = engram_get();
|
||||||
const char* q = EL_CSTR(query);
|
const char* q = EL_CSTR(query);
|
||||||
@@ -8030,16 +8151,16 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
|||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
if (q && *q && g->node_count > 0) {
|
if (q && *q && g->node_count > 0) {
|
||||||
/* Collect candidates from the UNION of lexical and semantic matches,
|
/* Collect candidates from the UNION of tokenized-lexical and semantic
|
||||||
* score each, rank by score, then emit the top `lim`. A node is a
|
* matches, score each, rank by score, emit the top `lim`. A node is a
|
||||||
* candidate if it lexically matches OR its query cosine clears the
|
* candidate if it covers >=1 query token (tokenized-lexical, #66) OR its
|
||||||
* threshold. Lexical hits get a base of 1.0 (so they always outrank a
|
* query cosine clears the threshold (#67). Lexical score is the distinct
|
||||||
* pure-semantic hit, whose cosine is in [0,1)), refined by cosine;
|
* token count (>=1), so any lexical hit outranks a pure-semantic hit
|
||||||
* pure-semantic hits are scored by cosine alone.
|
* (cosine < 1); pure-semantic hits are scored by cosine alone. When
|
||||||
*
|
* semantic is unavailable qvec is NULL, sem is 0, only tokenized-lexical
|
||||||
* When semantic search is unavailable, qvec is NULL, sem is 0, only
|
* hits are collected, and the stable insertion sort preserves order. */
|
||||||
* lexical hits are collected (score 1.0), and the stable insertion
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
||||||
* sort preserves node order — identical to the pre-existing search. */
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
||||||
int qdim = 0;
|
int qdim = 0;
|
||||||
float* qvec = engram_embed_query(q, &qdim);
|
float* qvec = engram_embed_query(q, &qdim);
|
||||||
double sem_min = engram_semantic_min();
|
double sem_min = engram_semantic_min();
|
||||||
@@ -8050,13 +8171,11 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
|||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||||
int lex = istr_contains(n->content, q) ||
|
int sc = engram_node_match_score(n, toks, ntok);
|
||||||
istr_contains(n->label, q) ||
|
|
||||||
istr_contains(n->tags, q);
|
|
||||||
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
||||||
if (lex || sem >= sem_min) {
|
if (sc > 0 || sem >= sem_min) {
|
||||||
cand[nc].idx = i;
|
cand[nc].idx = i;
|
||||||
cand[nc].score = (lex ? 1.0 : 0.0) + sem;
|
cand[nc].score = (double)sc + sem;
|
||||||
nc++;
|
nc++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -632,6 +632,7 @@ el_val_t engram_load(el_val_t path);
|
|||||||
* can pass results straight through without round-tripping ElList/ElMap
|
* can pass results straight through without round-tripping ElList/ElMap
|
||||||
* through json_stringify. */
|
* through json_stringify. */
|
||||||
el_val_t engram_get_node_json(el_val_t id);
|
el_val_t engram_get_node_json(el_val_t id);
|
||||||
|
el_val_t engram_get_node_by_label(el_val_t label);
|
||||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||||
|
|||||||
@@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path) { return engram_save
|
|||||||
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
||||||
|
|
||||||
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
||||||
|
el_val_t __engram_get_node_by_label(el_val_t label) { return engram_get_node_by_label(label); }
|
||||||
|
|
||||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
||||||
return engram_search_json(query, limit);
|
return engram_search_json(query, limit);
|
||||||
|
|||||||
@@ -226,6 +226,7 @@ el_val_t __engram_activate(el_val_t query, el_val_t depth);
|
|||||||
el_val_t __engram_save(el_val_t path);
|
el_val_t __engram_save(el_val_t path);
|
||||||
el_val_t __engram_load(el_val_t path);
|
el_val_t __engram_load(el_val_t path);
|
||||||
el_val_t __engram_get_node_json(el_val_t id);
|
el_val_t __engram_get_node_json(el_val_t id);
|
||||||
|
el_val_t __engram_get_node_by_label(el_val_t label);
|
||||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
||||||
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||||
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||||
|
|||||||
@@ -2670,6 +2670,7 @@ fn builtin_arity(name: String) -> Int {
|
|||||||
if str_eq(name, "engram_save") { return 1 }
|
if str_eq(name, "engram_save") { return 1 }
|
||||||
if str_eq(name, "engram_load") { return 1 }
|
if str_eq(name, "engram_load") { return 1 }
|
||||||
if str_eq(name, "engram_get_node_json") { return 1 }
|
if str_eq(name, "engram_get_node_json") { return 1 }
|
||||||
|
if str_eq(name, "engram_get_node_by_label") { return 1 }
|
||||||
if str_eq(name, "engram_search_json") { return 2 }
|
if str_eq(name, "engram_search_json") { return 2 }
|
||||||
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
||||||
if str_eq(name, "engram_neighbors_json") { return 3 }
|
if str_eq(name, "engram_neighbors_json") { return 3 }
|
||||||
|
|||||||
Reference in New Issue
Block a user