Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6b9d9e6c4a | |||
| 2b2a1246e7 |
@@ -6826,75 +6826,6 @@ static int istr_contains(const char* hay, const char* needle) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ── Tokenized query matching ───────────────────────────────────────────
|
|
||||||
* The engram query surface (search / activate / goal-bias) historically
|
|
||||||
* matched the ENTIRE raw query string as a single case-insensitive
|
|
||||||
* substring via istr_contains(field, q). That is Ctrl-F, not search:
|
|
||||||
* a multi-word query like "windows msi signing" only matched a node whose
|
|
||||||
* text contained that exact contiguous run, so real multi-word queries
|
|
||||||
* returned zero. istr_contains stays as the per-TOKEN primitive; these
|
|
||||||
* helpers split the query on whitespace and match ANY token, then rank by
|
|
||||||
* how many DISTINCT tokens a node covers. Single-token queries are a strict
|
|
||||||
* special case (score is 0 or 1) so single-word callers never regress. */
|
|
||||||
#define ENGRAM_MAX_QTOKENS 32
|
|
||||||
#define ENGRAM_QTOK_LEN 256
|
|
||||||
|
|
||||||
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
|
|
||||||
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
|
|
||||||
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
|
|
||||||
static int engram_tokenize_query(const char* q,
|
|
||||||
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
|
|
||||||
int n = 0;
|
|
||||||
if (!q) return 0;
|
|
||||||
const char* p = q;
|
|
||||||
while (*p && n < maxtok) {
|
|
||||||
while (*p && isspace((unsigned char)*p)) p++;
|
|
||||||
if (!*p) break;
|
|
||||||
char buf[ENGRAM_QTOK_LEN];
|
|
||||||
size_t tl = 0;
|
|
||||||
while (*p && !isspace((unsigned char)*p)) {
|
|
||||||
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
|
|
||||||
p++;
|
|
||||||
}
|
|
||||||
buf[tl] = '\0';
|
|
||||||
if (tl == 0) continue;
|
|
||||||
int dup = 0;
|
|
||||||
for (int s = 0; s < n; s++) {
|
|
||||||
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
|
|
||||||
}
|
|
||||||
if (dup) continue;
|
|
||||||
memcpy(toks[n], buf, tl + 1);
|
|
||||||
n++;
|
|
||||||
}
|
|
||||||
return n;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Count how many of the ntok distinct query tokens appear (case-insensitive)
|
|
||||||
* in the node's content, label, or tags. 0 == no match. */
|
|
||||||
static int engram_node_match_score(const EngramNode* n,
|
|
||||||
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
|
||||||
int score = 0;
|
|
||||||
for (int t = 0; t < ntok; t++) {
|
|
||||||
if (istr_contains(n->content, toks[t]) ||
|
|
||||||
istr_contains(n->label, toks[t]) ||
|
|
||||||
istr_contains(n->tags, toks[t]))
|
|
||||||
score++;
|
|
||||||
}
|
|
||||||
return score;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Rank entry: distinct-token match count (primary, desc) then salience
|
|
||||||
* (tiebreak, desc). */
|
|
||||||
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
|
||||||
static int engram_rank_cmp(const void* a, const void* b) {
|
|
||||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
|
||||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
|
||||||
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
|
|
||||||
if (ea->salience < eb->salience) return 1;
|
|
||||||
if (ea->salience > eb->salience) return -1;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||||
EngramStore* g = engram_get();
|
EngramStore* g = engram_get();
|
||||||
const char* q = EL_CSTR(query);
|
const char* q = EL_CSTR(query);
|
||||||
@@ -6902,34 +6833,21 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
|||||||
if (lim <= 0) lim = 100;
|
if (lim <= 0) lim = 100;
|
||||||
el_val_t lst = el_list_empty();
|
el_val_t lst = el_list_empty();
|
||||||
if (!q || !*q) return lst;
|
if (!q || !*q) return lst;
|
||||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
int64_t found = 0;
|
||||||
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
||||||
if (ntok == 0) return lst;
|
|
||||||
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
|
||||||
if (!hits) return lst;
|
|
||||||
int64_t nhits = 0;
|
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
||||||
* shape output but are invisible to introspection ("what do you
|
* shape output but are invisible to introspection ("what do you
|
||||||
* know about yourself"). They still surface via engram_activate
|
* know about yourself"). They still surface via engram_activate
|
||||||
* + engram_compile_layered_json — that's the legitimate path. */
|
* + engram_compile_layered_json — that's the legitimate path. */
|
||||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||||
int sc = engram_node_match_score(n, toks, ntok);
|
if (istr_contains(n->content, q) ||
|
||||||
if (sc > 0) {
|
istr_contains(n->label, q) ||
|
||||||
hits[nhits].idx = i;
|
istr_contains(n->tags, q)) {
|
||||||
hits[nhits].score = sc;
|
lst = el_list_append(lst, engram_node_to_map(n));
|
||||||
hits[nhits].salience = n->salience;
|
found++;
|
||||||
nhits++;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
|
|
||||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
|
||||||
int64_t end = nhits < lim ? nhits : lim;
|
|
||||||
for (int64_t k = 0; k < end; k++) {
|
|
||||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
|
||||||
}
|
|
||||||
free(hits);
|
|
||||||
return lst;
|
return lst;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7206,14 +7124,10 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
|
|||||||
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
||||||
if (!query || !*query) return 1.0;
|
if (!query || !*query) return 1.0;
|
||||||
double bias = 1.0;
|
double bias = 1.0;
|
||||||
/* Direct lexical overlap, graded by token coverage: a node covering all
|
/* Direct lexical overlap: node content/label/tags share text with query. */
|
||||||
* query tokens gets the full +0.5; partial coverage gets a proportional
|
if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
|
||||||
* share. Single-token queries → full +0.5 on match, identical to before. */
|
istr_contains(n->tags, query)) {
|
||||||
{
|
bias += 0.5;
|
||||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
|
||||||
int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
|
|
||||||
int sc = engram_node_match_score(n, toks, ntok);
|
|
||||||
if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
|
|
||||||
}
|
}
|
||||||
/* Node-type resonance with query intent. */
|
/* Node-type resonance with query intent. */
|
||||||
int technical_query = istr_contains(query, "code") ||
|
int technical_query = istr_contains(query, "code") ||
|
||||||
@@ -7279,21 +7193,14 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
|||||||
if (!seeds) {
|
if (!seeds) {
|
||||||
free(best_bg); free(best_hops); free(reached); return out;
|
free(best_bg); free(best_hops); free(reached); return out;
|
||||||
}
|
}
|
||||||
/* Tokenize once: a node seeds if it matches ANY query token, and its seed
|
|
||||||
* activation is scaled by token coverage (fraction of distinct query
|
|
||||||
* tokens it contains) so a node matching all words seeds more strongly
|
|
||||||
* than one matching a single word. Single-word queries → coverage 1.0,
|
|
||||||
* identical to the prior whole-query behavior. */
|
|
||||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
|
||||||
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
EngramNode* n = &g->nodes[i];
|
EngramNode* n = &g->nodes[i];
|
||||||
int sc = engram_node_match_score(n, toks, ntok);
|
if (istr_contains(n->content, q) ||
|
||||||
if (sc > 0) {
|
istr_contains(n->label, q) ||
|
||||||
|
istr_contains(n->tags, q)) {
|
||||||
double tdecay = engram_temporal_decay(n, now_ms);
|
double tdecay = engram_temporal_decay(n, now_ms);
|
||||||
double dampen = engram_activation_dampen(n);
|
double dampen = engram_activation_dampen(n);
|
||||||
double cover = ntok > 0 ? (double)sc / (double)ntok : 1.0;
|
double act = n->salience * tdecay * dampen;
|
||||||
double act = n->salience * tdecay * dampen * cover;
|
|
||||||
seeds[seed_count].idx = i;
|
seeds[seed_count].idx = i;
|
||||||
seeds[seed_count].act = act;
|
seeds[seed_count].act = act;
|
||||||
seeds[seed_count].created_at = n->created_at;
|
seeds[seed_count].created_at = n->created_at;
|
||||||
@@ -7854,6 +7761,35 @@ el_val_t engram_get_node_json(el_val_t id) {
|
|||||||
return el_wrap_str(jb_finish(&b));
|
return el_wrap_str(jb_finish(&b));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* engram_get_node_by_label — find the first node whose label field exactly
|
||||||
|
* matches the given string. Returns the node as a JSON object string, or "{}"
|
||||||
|
* if no match is found.
|
||||||
|
*
|
||||||
|
* Used by chat.el to retrieve well-known nodes (e.g. "conv:history",
|
||||||
|
* "session:summary") by their stable label rather than by ID, which is immune
|
||||||
|
* to vector index drift across restarts.
|
||||||
|
*
|
||||||
|
* Exact match (strcmp, not istr_contains) because labels like "conv:history"
|
||||||
|
* must not collide with nodes whose content happens to contain that substring.
|
||||||
|
*
|
||||||
|
* Backported verbatim (idiom-adapted to jb_finish) from release runtime
|
||||||
|
* v1.0.0-20260501 to unblock the soul regen link: chat.el references this
|
||||||
|
* native but the current runtime lacked its definition. */
|
||||||
|
el_val_t engram_get_node_by_label(el_val_t label) {
|
||||||
|
const char* lbl = EL_CSTR(label);
|
||||||
|
if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}"));
|
||||||
|
EngramStore* g = engram_get();
|
||||||
|
for (int64_t i = 0; i < g->node_count; i++) {
|
||||||
|
EngramNode* n = &g->nodes[i];
|
||||||
|
if (n->label && strcmp(n->label, lbl) == 0) {
|
||||||
|
JsonBuf b; jb_init(&b);
|
||||||
|
engram_emit_node_json(&b, n);
|
||||||
|
return el_wrap_str(jb_finish(&b));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return el_wrap_str(el_strdup("{}"));
|
||||||
|
}
|
||||||
|
|
||||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||||
EngramStore* g = engram_get();
|
EngramStore* g = engram_get();
|
||||||
const char* q = EL_CSTR(query);
|
const char* q = EL_CSTR(query);
|
||||||
@@ -7862,36 +7798,19 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
|||||||
JsonBuf b; jb_init(&b);
|
JsonBuf b; jb_init(&b);
|
||||||
jb_putc(&b, '[');
|
jb_putc(&b, '[');
|
||||||
int first = 1;
|
int first = 1;
|
||||||
|
int64_t found = 0;
|
||||||
if (q && *q) {
|
if (q && *q) {
|
||||||
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
||||||
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
EngramNode* n = &g->nodes[i];
|
||||||
if (ntok > 0) {
|
/* Filter transparent layers — same as engram_search. */
|
||||||
EngramRankEntry* hits =
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||||
malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
if (istr_contains(n->content, q) ||
|
||||||
if (hits) {
|
istr_contains(n->label, q) ||
|
||||||
int64_t nhits = 0;
|
istr_contains(n->tags, q)) {
|
||||||
for (int64_t i = 0; i < g->node_count; i++) {
|
if (!first) jb_putc(&b, ',');
|
||||||
EngramNode* n = &g->nodes[i];
|
engram_emit_node_json(&b, n);
|
||||||
/* Filter transparent layers — same as engram_search. */
|
first = 0;
|
||||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
found++;
|
||||||
int sc = engram_node_match_score(n, toks, ntok);
|
|
||||||
if (sc > 0) {
|
|
||||||
hits[nhits].idx = i;
|
|
||||||
hits[nhits].score = sc;
|
|
||||||
hits[nhits].salience = n->salience;
|
|
||||||
nhits++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/* Rank by distinct tokens matched (desc) then salience (desc). */
|
|
||||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry),
|
|
||||||
engram_rank_cmp);
|
|
||||||
int64_t end = nhits < lim ? nhits : lim;
|
|
||||||
for (int64_t k = 0; k < end; k++) {
|
|
||||||
if (!first) jb_putc(&b, ',');
|
|
||||||
engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
|
|
||||||
first = 0;
|
|
||||||
}
|
|
||||||
free(hits);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -632,6 +632,7 @@ el_val_t engram_load(el_val_t path);
|
|||||||
* can pass results straight through without round-tripping ElList/ElMap
|
* can pass results straight through without round-tripping ElList/ElMap
|
||||||
* through json_stringify. */
|
* through json_stringify. */
|
||||||
el_val_t engram_get_node_json(el_val_t id);
|
el_val_t engram_get_node_json(el_val_t id);
|
||||||
|
el_val_t engram_get_node_by_label(el_val_t label);
|
||||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||||
|
|||||||
@@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path) { return engram_save
|
|||||||
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
el_val_t __engram_load(el_val_t path) { return engram_load(path); }
|
||||||
|
|
||||||
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); }
|
||||||
|
el_val_t __engram_get_node_by_label(el_val_t label) { return engram_get_node_by_label(label); }
|
||||||
|
|
||||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
|
||||||
return engram_search_json(query, limit);
|
return engram_search_json(query, limit);
|
||||||
|
|||||||
@@ -226,6 +226,7 @@ el_val_t __engram_activate(el_val_t query, el_val_t depth);
|
|||||||
el_val_t __engram_save(el_val_t path);
|
el_val_t __engram_save(el_val_t path);
|
||||||
el_val_t __engram_load(el_val_t path);
|
el_val_t __engram_load(el_val_t path);
|
||||||
el_val_t __engram_get_node_json(el_val_t id);
|
el_val_t __engram_get_node_json(el_val_t id);
|
||||||
|
el_val_t __engram_get_node_by_label(el_val_t label);
|
||||||
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
el_val_t __engram_search_json(el_val_t query, el_val_t limit);
|
||||||
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||||
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||||
|
|||||||
@@ -2670,6 +2670,7 @@ fn builtin_arity(name: String) -> Int {
|
|||||||
if str_eq(name, "engram_save") { return 1 }
|
if str_eq(name, "engram_save") { return 1 }
|
||||||
if str_eq(name, "engram_load") { return 1 }
|
if str_eq(name, "engram_load") { return 1 }
|
||||||
if str_eq(name, "engram_get_node_json") { return 1 }
|
if str_eq(name, "engram_get_node_json") { return 1 }
|
||||||
|
if str_eq(name, "engram_get_node_by_label") { return 1 }
|
||||||
if str_eq(name, "engram_search_json") { return 2 }
|
if str_eq(name, "engram_search_json") { return 2 }
|
||||||
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
||||||
if str_eq(name, "engram_neighbors_json") { return 3 }
|
if str_eq(name, "engram_neighbors_json") { return 3 }
|
||||||
|
|||||||
@@ -23,29 +23,10 @@ fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn tok_kind(tokens: [Any], pos: Int) -> String {
|
fn tok_kind(tokens: [Any], pos: Int) -> String {
|
||||||
// Out-of-range reads must report the Eof sentinel so every `== "Eof"`
|
|
||||||
// termination guard in the parser fires. Without this, reading past the
|
|
||||||
// single trailing Eof token returns runtime null (el_list_get OOB -> 0),
|
|
||||||
// which matches no delimiter, letting inner parse loops append AST nodes
|
|
||||||
// forever on malformed input -> unbounded allocation -> OOM.
|
|
||||||
let n: Int = native_list_len(tokens) / 2
|
|
||||||
if pos < 0 {
|
|
||||||
return "Eof"
|
|
||||||
}
|
|
||||||
if pos >= n {
|
|
||||||
return "Eof"
|
|
||||||
}
|
|
||||||
native_list_get(tokens, pos * 2)
|
native_list_get(tokens, pos * 2)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn tok_value(tokens: [Any], pos: Int) -> String {
|
fn tok_value(tokens: [Any], pos: Int) -> String {
|
||||||
let n: Int = native_list_len(tokens) / 2
|
|
||||||
if pos < 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
if pos >= n {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
native_list_get(tokens, pos * 2 + 1)
|
native_list_get(tokens, pos * 2 + 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,12 +35,7 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
|
|||||||
if k == kind {
|
if k == kind {
|
||||||
return pos + 1
|
return pos + 1
|
||||||
}
|
}
|
||||||
// On mismatch, error recovery is best-effort. But never step PAST the Eof
|
// On mismatch just advance; error recovery is best-effort
|
||||||
// sentinel: once at Eof a mismatch means the input ended early, and
|
|
||||||
// advancing would run the cursor off the token list.
|
|
||||||
if k == "Eof" {
|
|
||||||
return pos
|
|
||||||
}
|
|
||||||
pos + 1
|
pos + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user