Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Tim Lingo
|
bef36a5e3a |
+330
-125
@@ -1205,6 +1205,18 @@ el_val_t handle_api_memory_update(el_val_t body);
|
||||
el_val_t handle_api_cultivate(el_val_t body);
|
||||
el_val_t handle_api_list_typed(el_val_t node_type, el_val_t path, el_val_t body);
|
||||
el_val_t handle_api_consolidate(el_val_t body);
|
||||
el_val_t audit_pct1(el_val_t num, el_val_t den);
|
||||
el_val_t audit_finding(el_val_t name, el_val_t measured, el_val_t note);
|
||||
el_val_t audit_str_at(el_val_t s, el_val_t start, el_val_t maxlen);
|
||||
el_val_t audit_rel_count(el_val_t edges, el_val_t rel);
|
||||
el_val_t audit_owner_stats(el_val_t url);
|
||||
el_val_t audit_divergence(void);
|
||||
el_val_t audit_edge_typing(el_val_t edges, el_val_t total_edges, el_val_t node_total);
|
||||
el_val_t audit_orphans_dangling(el_val_t edges, el_val_t total_edges, el_val_t node_total, el_val_t edge_cap, el_val_t node_cap);
|
||||
el_val_t audit_pillar(el_val_t key, el_val_t id);
|
||||
el_val_t audit_self_model(void);
|
||||
el_val_t audit_deferred(void);
|
||||
el_val_t handle_api_structural_audit(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t session_title_from_message(el_val_t message);
|
||||
el_val_t session_make_content(el_val_t id, el_val_t title, el_val_t created_at, el_val_t updated_at, el_val_t folder);
|
||||
el_val_t session_exists(el_val_t session_id);
|
||||
@@ -30219,6 +30231,193 @@ el_val_t handle_api_consolidate(el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_pct1(el_val_t num, el_val_t den) {
|
||||
if (den <= 0) {
|
||||
return EL_STR("null");
|
||||
}
|
||||
el_val_t neg = (num < 0);
|
||||
el_val_t a = ({ el_val_t _if_result_653 = 0; if (neg) { _if_result_653 = ((0 - num)); } else { _if_result_653 = (num); } _if_result_653; });
|
||||
el_val_t tenths = ((a * 1000) / den);
|
||||
el_val_t whole = (tenths / 10);
|
||||
el_val_t frac = (tenths - (whole * 10));
|
||||
el_val_t sign = ({ el_val_t _if_result_654 = 0; if (neg) { _if_result_654 = (EL_STR("-")); } else { _if_result_654 = (EL_STR("")); } _if_result_654; });
|
||||
return el_str_concat(el_str_concat(el_str_concat(sign, int_to_str(whole)), EL_STR(".")), int_to_str(frac));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_finding(el_val_t name, el_val_t measured, el_val_t note) {
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"finding\":\""), name), EL_STR("\"")), EL_STR(",\"measured\":{")), measured), EL_STR("}")), EL_STR(",\"note\":\"")), api_json_escape(note)), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_str_at(el_val_t s, el_val_t start, el_val_t maxlen) {
|
||||
el_val_t n = str_len(s);
|
||||
if ((start < 0) || (start >= n)) {
|
||||
return EL_STR("");
|
||||
}
|
||||
el_val_t end_guess = (start + maxlen);
|
||||
el_val_t stop = ({ el_val_t _if_result_655 = 0; if ((end_guess > n)) { _if_result_655 = (n); } else { _if_result_655 = (end_guess); } _if_result_655; });
|
||||
el_val_t win = str_slice(s, start, stop);
|
||||
el_val_t q = str_index_of(win, EL_STR("\""));
|
||||
if (q < 0) {
|
||||
return EL_STR("");
|
||||
}
|
||||
return str_slice(win, 0, q);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_rel_count(el_val_t edges, el_val_t rel) {
|
||||
return str_count(edges, el_str_concat(el_str_concat(EL_STR("\"relation\":\""), rel), EL_STR("\"")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_owner_stats(el_val_t url) {
|
||||
if (str_eq(url, EL_STR(""))) {
|
||||
return EL_STR("");
|
||||
}
|
||||
return http_get(el_str_concat(url, EL_STR("/api/stats")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_divergence(void) {
|
||||
el_val_t rt_nodes = engram_node_count();
|
||||
el_val_t rt_edges = engram_edge_count();
|
||||
el_val_t url = wt_engram_url();
|
||||
if (str_eq(url, EL_STR(""))) {
|
||||
return audit_finding(EL_STR("owner_runtime_divergence"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\"runtime_nodes\":"), int_to_str(rt_nodes)), EL_STR(",\"runtime_edges\":")), int_to_str(rt_edges)), EL_STR(",\"owner\":\"none\",\"owner_reachable\":false")), el_str_concat(el_str_concat(el_str_concat(EL_STR("No HTTP persistence owner is configured, so this soul IS the owner "), EL_STR("(file mode) and divergence is not defined. This check only has ")), EL_STR("meaning when ENGRAM_URL points at a separate engram that owns the ")), EL_STR("canonical store.")));
|
||||
}
|
||||
el_val_t stats = audit_owner_stats(url);
|
||||
el_val_t owner_nc_raw = json_get_raw(stats, EL_STR("node_count"));
|
||||
if (str_eq(stats, EL_STR("")) || str_eq(owner_nc_raw, EL_STR(""))) {
|
||||
return audit_finding(EL_STR("owner_runtime_divergence"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\"runtime_nodes\":"), int_to_str(rt_nodes)), EL_STR(",\"runtime_edges\":")), int_to_str(rt_edges)), EL_STR(",\"owner\":\"")), api_json_escape(url)), EL_STR("\",\"owner_reachable\":false")), EL_STR(",\"owner_reply\":\"")), api_json_escape(api_utf8_trunc(stats, 200))), EL_STR("\"")), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("The persistence owner at "), url), EL_STR(" did not return a node_count ")), EL_STR("from GET /api/stats. Divergence is UNKNOWN, NOT ZERO \xe2\x80\x94 an owner ")), EL_STR("that cannot be read is exactly the condition under which the ")), EL_STR("runtime's own count means least, and reporting 0 for the owner ")), EL_STR("would manufacture a total-loss reading out of a network error. ")), EL_STR("Reported as a finding rather than raised as an error so the rest ")), EL_STR("of the audit still returns; the owner's raw reply is in ")), EL_STR("owner_reply.")));
|
||||
}
|
||||
el_val_t ow_nodes = json_get_int(stats, EL_STR("node_count"));
|
||||
el_val_t ow_edges = json_get_int(stats, EL_STR("edge_count"));
|
||||
el_val_t d_nodes = (rt_nodes - ow_nodes);
|
||||
el_val_t d_edges = (rt_edges - ow_edges);
|
||||
el_val_t prev_raw = state_get(EL_STR("audit_prev_node_delta"));
|
||||
el_val_t prev = str_to_int(prev_raw);
|
||||
el_val_t abs_now = ({ el_val_t _if_result_656 = 0; if ((d_nodes < 0)) { _if_result_656 = ((0 - d_nodes)); } else { _if_result_656 = (d_nodes); } _if_result_656; });
|
||||
el_val_t abs_prev = ({ el_val_t _if_result_657 = 0; if ((prev < 0)) { _if_result_657 = ((0 - prev)); } else { _if_result_657 = (prev); } _if_result_657; });
|
||||
el_val_t trend = ({ el_val_t _if_result_658 = 0; if (str_eq(prev_raw, EL_STR(""))) { _if_result_658 = (EL_STR("no_prior_audit")); } else { _if_result_658 = (({ el_val_t _if_result_659 = 0; if ((abs_now > abs_prev)) { _if_result_659 = (EL_STR("growing")); } else { _if_result_659 = (({ el_val_t _if_result_660 = 0; if ((abs_now < abs_prev)) { _if_result_660 = (EL_STR("shrinking")); } else { _if_result_660 = (EL_STR("flat")); } _if_result_660; })); } _if_result_659; })); } _if_result_658; });
|
||||
state_set(EL_STR("audit_prev_node_delta"), int_to_str(d_nodes));
|
||||
state_set(EL_STR("audit_prev_ts"), int_to_str(time_now()));
|
||||
el_val_t note_head = ({ el_val_t _if_result_661 = 0; if ((d_nodes == 0)) { _if_result_661 = (EL_STR("Runtime and owner agree on node count.")); } else { _if_result_661 = (el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("Runtime holds "), int_to_str(d_nodes)), EL_STR(" nodes (")), audit_pct1(d_nodes, rt_nodes)), EL_STR("% of its own graph) that the persistence owner does not report. Nodes ")), EL_STR("that exist only in runtime memory do not survive a restart."))); } _if_result_661; });
|
||||
return audit_finding(EL_STR("owner_runtime_divergence"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\"runtime_nodes\":"), int_to_str(rt_nodes)), EL_STR(",\"runtime_edges\":")), int_to_str(rt_edges)), EL_STR(",\"owner\":\"")), api_json_escape(url)), EL_STR("\",\"owner_reachable\":true")), EL_STR(",\"owner_nodes\":")), int_to_str(ow_nodes)), EL_STR(",\"owner_edges\":")), int_to_str(ow_edges)), EL_STR(",\"node_delta\":")), int_to_str(d_nodes)), EL_STR(",\"edge_delta\":")), int_to_str(d_edges)), EL_STR(",\"node_delta_pct_of_runtime\":")), audit_pct1(d_nodes, rt_nodes)), EL_STR(",\"trend_vs_previous_audit\":\"")), trend), EL_STR("\"")), EL_STR(",\"previous_node_delta\":")), ({ el_val_t _if_result_662 = 0; if (str_eq(prev_raw, EL_STR(""))) { _if_result_662 = (EL_STR("null")); } else { _if_result_662 = (int_to_str(prev)); } _if_result_662; })), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(note_head, EL_STR(" Trend against the previous audit recorded in this soul's ")), EL_STR("state: ")), trend), EL_STR(". This is the comparison whose absence let a ")), EL_STR("~24,000-node loss run for weeks with every boot reporting green.")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_edge_typing(el_val_t edges, el_val_t total_edges, el_val_t node_total) {
|
||||
el_val_t c_sup = audit_rel_count(edges, EL_STR("Supersedes"));
|
||||
el_val_t c_cau = audit_rel_count(edges, EL_STR("Causes"));
|
||||
el_val_t c_con = audit_rel_count(edges, EL_STR("Contains"));
|
||||
el_val_t c_ref = audit_rel_count(edges, EL_STR("References"));
|
||||
el_val_t c_ctr = audit_rel_count(edges, EL_STR("Contradicts"));
|
||||
el_val_t c_exe = audit_rel_count(edges, EL_STR("Exemplifies"));
|
||||
el_val_t c_act = audit_rel_count(edges, EL_STR("Activates"));
|
||||
el_val_t c_tmp = audit_rel_count(edges, EL_STR("TemporallyPrecedes"));
|
||||
el_val_t typed = (((((((c_sup + c_cau) + c_con) + c_ref) + c_ctr) + c_exe) + c_act) + c_tmp);
|
||||
el_val_t l_sup = audit_rel_count(edges, EL_STR("supersedes"));
|
||||
el_val_t l_cau = audit_rel_count(edges, EL_STR("causes"));
|
||||
el_val_t l_con = audit_rel_count(edges, EL_STR("contains"));
|
||||
el_val_t l_ref = audit_rel_count(edges, EL_STR("references"));
|
||||
el_val_t l_ctr = audit_rel_count(edges, EL_STR("contradicts"));
|
||||
el_val_t l_exe = audit_rel_count(edges, EL_STR("exemplifies"));
|
||||
el_val_t l_act = audit_rel_count(edges, EL_STR("activates"));
|
||||
el_val_t l_tmp = audit_rel_count(edges, EL_STR("temporallyPrecedes"));
|
||||
el_val_t near = (((((((l_sup + l_cau) + l_con) + l_ref) + l_ctr) + l_exe) + l_act) + l_tmp);
|
||||
el_val_t untyped = (total_edges - typed);
|
||||
return audit_finding(EL_STR("typed_edge_distribution"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\"total_edges\":"), int_to_str(total_edges)), EL_STR(",\"total_nodes\":")), int_to_str(node_total)), EL_STR(",\"edges_per_100_nodes\":")), audit_pct1(total_edges, node_total)), EL_STR(",\"claim10_typed\":")), int_to_str(typed)), EL_STR(",\"claim10_typed_pct\":")), audit_pct1(typed, total_edges)), EL_STR(",\"outside_claim10_vocabulary\":")), int_to_str(untyped)), EL_STR(",\"lowercase_near_miss\":")), int_to_str(near)), EL_STR(",\"by_relation\":{")), EL_STR("\"Supersedes\":")), int_to_str(c_sup)), EL_STR(",\"Causes\":")), int_to_str(c_cau)), EL_STR(",\"Contains\":")), int_to_str(c_con)), EL_STR(",\"References\":")), int_to_str(c_ref)), EL_STR(",\"Contradicts\":")), int_to_str(c_ctr)), EL_STR(",\"Exemplifies\":")), int_to_str(c_exe)), EL_STR(",\"Activates\":")), int_to_str(c_act)), EL_STR(",\"TemporallyPrecedes\":")), int_to_str(c_tmp)), EL_STR("}")), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("Only "), int_to_str(typed)), EL_STR(" of ")), int_to_str(total_edges)), EL_STR(" edges use the claim-10 causal vocabulary; the remainder are ad-hoc ")), EL_STR("relation strings, which is why the graph's causal claims cannot yet ")), EL_STR("be checked for internal consistency \xe2\x80\x94 an untyped edge asserts ")), EL_STR("association, not causation. ")), int_to_str(near)), EL_STR(" edges use a ")), EL_STR("lowercase spelling of a claim-10 relation: those are near-misses the ")), EL_STR("write paths could be corrected to emit, not genuinely foreign types.")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_orphans_dangling(el_val_t edges, el_val_t total_edges, el_val_t node_total, el_val_t edge_cap, el_val_t node_cap) {
|
||||
el_val_t n_take = ({ el_val_t _if_result_663 = 0; if ((node_total < node_cap)) { _if_result_663 = (node_total); } else { _if_result_663 = (node_cap); } _if_result_663; });
|
||||
el_val_t n_stride = ({ el_val_t _if_result_664 = 0; if ((n_take > 0)) { _if_result_664 = ((node_total / n_take)); } else { _if_result_664 = (1); } _if_result_664; });
|
||||
n_stride = ({ el_val_t _if_result_665 = 0; if ((n_stride < 1)) { _if_result_665 = (1); } else { _if_result_665 = (n_stride); } _if_result_665; });
|
||||
el_val_t orphans = 0;
|
||||
el_val_t n_checked = 0;
|
||||
el_val_t j = 0;
|
||||
while (j < n_take) {
|
||||
el_val_t one = engram_scan_nodes_json(1, (j * n_stride));
|
||||
el_val_t nid = json_get(json_array_get(one, 0), EL_STR("id"));
|
||||
if (!str_eq(nid, EL_STR(""))) {
|
||||
el_val_t nbrs = engram_neighbors_json(nid, 1, EL_STR("both"));
|
||||
el_val_t deg = json_array_len(nbrs);
|
||||
orphans = ({ el_val_t _if_result_666 = 0; if ((deg == 0)) { _if_result_666 = ((orphans + 1)); } else { _if_result_666 = (orphans); } _if_result_666; });
|
||||
n_checked = (n_checked + 1);
|
||||
}
|
||||
j = (j + 1);
|
||||
}
|
||||
el_val_t from_pos = str_index_of_all(edges, EL_STR("\"from_id\":\""));
|
||||
el_val_t to_pos = str_index_of_all(edges, EL_STR("\"to_id\":\""));
|
||||
el_val_t nf = len(from_pos);
|
||||
el_val_t nt = len(to_pos);
|
||||
el_val_t ne = ({ el_val_t _if_result_667 = 0; if ((nf < nt)) { _if_result_667 = (nf); } else { _if_result_667 = (nt); } _if_result_667; });
|
||||
el_val_t e_take = ({ el_val_t _if_result_668 = 0; if ((ne < edge_cap)) { _if_result_668 = (ne); } else { _if_result_668 = (edge_cap); } _if_result_668; });
|
||||
el_val_t e_stride = ({ el_val_t _if_result_669 = 0; if ((e_take > 0)) { _if_result_669 = ((ne / e_take)); } else { _if_result_669 = (1); } _if_result_669; });
|
||||
e_stride = ({ el_val_t _if_result_670 = 0; if ((e_stride < 1)) { _if_result_670 = (1); } else { _if_result_670 = (e_stride); } _if_result_670; });
|
||||
el_val_t dangling = 0;
|
||||
el_val_t e_checked = 0;
|
||||
el_val_t i = 0;
|
||||
while ((i < ne) && (e_checked < e_take)) {
|
||||
el_val_t fid = audit_str_at(edges, (get(from_pos, i) + 11), 96);
|
||||
el_val_t tid = audit_str_at(edges, (get(to_pos, i) + 9), 96);
|
||||
el_val_t f_gone = str_eq(engram_get_node_json(fid), EL_STR("{}"));
|
||||
el_val_t t_gone = ({ el_val_t _if_result_671 = 0; if (f_gone) { _if_result_671 = (1); } else { _if_result_671 = (str_eq(engram_get_node_json(tid), EL_STR("{}"))); } _if_result_671; });
|
||||
dangling = ({ el_val_t _if_result_672 = 0; if ((f_gone || t_gone)) { _if_result_672 = ((dangling + 1)); } else { _if_result_672 = (dangling); } _if_result_672; });
|
||||
e_checked = (e_checked + 1);
|
||||
i = (i + e_stride);
|
||||
}
|
||||
el_val_t orphan_est = ({ el_val_t _if_result_673 = 0; if ((n_checked > 0)) { _if_result_673 = (((orphans * node_total) / n_checked)); } else { _if_result_673 = (0); } _if_result_673; });
|
||||
el_val_t dangle_est = ({ el_val_t _if_result_674 = 0; if ((e_checked > 0)) { _if_result_674 = (((dangling * total_edges) / e_checked)); } else { _if_result_674 = (0); } _if_result_674; });
|
||||
el_val_t exhaustive_n = ({ el_val_t _if_result_675 = 0; if ((n_checked >= node_total)) { _if_result_675 = (EL_STR("true")); } else { _if_result_675 = (EL_STR("false")); } _if_result_675; });
|
||||
el_val_t exhaustive_e = ({ el_val_t _if_result_676 = 0; if ((e_checked >= ne)) { _if_result_676 = (EL_STR("true")); } else { _if_result_676 = (EL_STR("false")); } _if_result_676; });
|
||||
return audit_finding(EL_STR("orphans_and_dangling_edges"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\"nodes_population\":"), int_to_str(node_total)), EL_STR(",\"nodes_sampled\":")), int_to_str(n_checked)), EL_STR(",\"nodes_sample_exhaustive\":")), exhaustive_n), EL_STR(",\"orphans_in_sample\":")), int_to_str(orphans)), EL_STR(",\"orphan_rate_pct\":")), audit_pct1(orphans, n_checked)), EL_STR(",\"orphans_extrapolated\":")), int_to_str(orphan_est)), EL_STR(",\"edges_population\":")), int_to_str(total_edges)), EL_STR(",\"edges_sampled\":")), int_to_str(e_checked)), EL_STR(",\"edges_sample_exhaustive\":")), exhaustive_e), EL_STR(",\"dangling_in_sample\":")), int_to_str(dangling)), EL_STR(",\"dangling_rate_pct\":")), audit_pct1(dangling, e_checked)), EL_STR(",\"dangling_extrapolated\":")), int_to_str(dangle_est)), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("Orphan = zero RESOLVABLE edges, so a node whose only edges dangle counts "), EL_STR("as an orphan; either way it is unreachable by traversal. Dangling = an ")), EL_STR("edge with an endpoint id that resolves to no node. Both are uniform ")), EL_STR("stride samples over the whole population, not the head of the list; ")), EL_STR("the extrapolations are estimates and are labelled as such. Pass ")), EL_STR("?node_sample= / ?edge_sample= at or above the population size to run ")), EL_STR("either check exhaustively. A high orphan rate is a characterization, ")), EL_STR("not a verdict: an accumulating store legitimately holds unlinked ")), EL_STR("material. It becomes a defect when the write paths were SUPPOSED to ")), EL_STR("link and did not.")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_pillar(el_val_t key, el_val_t id) {
|
||||
el_val_t node = engram_get_node_json(id);
|
||||
el_val_t present = (!str_eq(node, EL_STR("{}")) && !str_eq(node, EL_STR("")));
|
||||
if (!present) {
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\""), key), EL_STR("\":{\"id\":\"")), id), EL_STR("\",\"present\":false")), EL_STR(",\"content_length\":0,\"degree\":0}"));
|
||||
}
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t deg = json_array_len(engram_neighbors_json(id, 1, EL_STR("both")));
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\""), key), EL_STR("\":{\"id\":\"")), id), EL_STR("\",\"present\":true")), EL_STR(",\"label\":\"")), api_json_escape(json_get(node, EL_STR("label")))), EL_STR("\"")), EL_STR(",\"tier\":\"")), api_json_escape(json_get(node, EL_STR("tier")))), EL_STR("\"")), EL_STR(",\"content_length\":")), int_to_str(str_len(content))), EL_STR(",\"degree\":")), int_to_str(deg)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_self_model(void) {
|
||||
el_val_t dna = audit_pillar(EL_STR("intellectual_dna"), EL_STR("kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6"));
|
||||
el_val_t val = audit_pillar(EL_STR("values_hub"), EL_STR("kn-5b606390-a52d-4ca2-8e0e-eba141d13440"));
|
||||
el_val_t phi = audit_pillar(EL_STR("memory_philosophy"), EL_STR("kn-dcfe04b3-3702-4cac-b6f0-ecb4db837eee"));
|
||||
el_val_t root = audit_pillar(EL_STR("self_root"), EL_STR("kn-efeb4a5b-5aff-4759-8a97-7233099be6ee"));
|
||||
return audit_finding(EL_STR("self_model_connectivity"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("\"pillars\":{"), dna), EL_STR(",")), val), EL_STR(",")), phi), EL_STR(",")), root), EL_STR("}")), el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("The three identity pillars plus the self root. `degree` counts nodes "), EL_STR("reachable in one hop in either direction \xe2\x80\x94 the self-model's connection ")), EL_STR("to the rest of the graph. present:false on any pillar is the condition ")), EL_STR("that ran undetected for weeks; content_length distinguishes a pillar ")), EL_STR("that is present from one that is present but hollowed out. The patent ")), EL_STR("also asks whether the self-model makes ACCURATE PREDICTIONS about the ")), EL_STR("system's own behavior; that half needs Prediction nodes and is deferred ")), EL_STR("with the rest of stage 1b below.")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t audit_deferred(void) {
|
||||
el_val_t preds = json_array_len(api_or_empty(engram_scan_nodes_by_type_json(EL_STR("Prediction"), 50, 0)));
|
||||
el_val_t wonders = json_array_len(api_or_empty(engram_scan_nodes_by_type_json(EL_STR("WonderQuestion"), 50, 0)));
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("[{\"deferred\":\"value_execution_record_consistency\""), EL_STR(",\"stage\":\"1b\"")), EL_STR(",\"measured\":{\"prediction_nodes_found\":")), int_to_str(preds)), EL_STR("}")), EL_STR(",\"reason\":\"")), api_json_escape(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("The patent asks whether the execution history SUPPORTS the stated "), EL_STR("values or shows systematic conflict. That requires execution ")), EL_STR("records tied to value nodes and predictions to score them against. ")), EL_STR("Prediction nodes found (capped at 50): ")), int_to_str(preds)), EL_STR(". Asserting value/execution coherence on that population would be ")), EL_STR("a fabricated result, which is worse than a stated gap.")))), EL_STR("\"}")), EL_STR(",{\"deferred\":\"wonder_manifest_authenticity\"")), EL_STR(",\"stage\":\"1b\"")), EL_STR(",\"measured\":{\"wonder_question_nodes_found\":")), int_to_str(wonders)), EL_STR("}")), EL_STR(",\"reason\":\"")), api_json_escape(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("The patent asks whether pull weights CORRELATE WITH GENUINE "), EL_STR("PREDICTION UNCERTAINTY or are uniform/externally assigned \xe2\x80\x94 a ")), EL_STR("correlation between two populations. WonderQuestion nodes readable ")), EL_STR("by type (capped at 50): ")), int_to_str(wonders)), EL_STR(", against ")), int_to_str(preds)), EL_STR(" Prediction nodes. There is a known write/read ")), EL_STR("node-type mismatch on the wonder path; until that is fixed and both ")), EL_STR("populations exist, any correlation reported here would be noise.")))), EL_STR("\"}]"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t handle_api_structural_audit(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t node_total = engram_node_count();
|
||||
el_val_t edge_total = engram_edge_count();
|
||||
el_val_t want_edges = !str_eq(api_query_param(path, EL_STR("edges")), EL_STR("0"));
|
||||
el_val_t edge_cap = api_query_int(path, EL_STR("edge_sample"), 3000);
|
||||
el_val_t node_cap = api_query_int(path, EL_STR("node_sample"), 300);
|
||||
el_val_t divergence = audit_divergence();
|
||||
el_val_t self_model = audit_self_model();
|
||||
el_val_t edge_part = ({ el_val_t _if_result_677 = 0; if (want_edges) { el_val_t scratch_dir = env(EL_STR("TMPDIR")); el_val_t scratch_base = ({ el_val_t _if_result_678 = 0; if (str_eq(scratch_dir, EL_STR(""))) { _if_result_678 = (EL_STR("/tmp")); } else { _if_result_678 = (scratch_dir); } _if_result_678; }); el_val_t snap_path = el_str_concat(el_str_concat(el_str_concat(scratch_base, EL_STR("/soul-audit-export-")), state_get(EL_STR("soul_cgi_id"))), EL_STR(".json")); el_val_t saved = engram_save(snap_path); _if_result_677 = (({ el_val_t _if_result_679 = 0; if ((saved == 0)) { _if_result_679 = (el_str_concat(EL_STR(","), audit_finding(EL_STR("typed_edge_distribution"), EL_STR("\"available\":false"), el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("Could not export the graph to "), snap_path), EL_STR(" for edge analysis, ")), EL_STR("so edge typing and the dangling-edge sample were not run. ")), EL_STR("Reported as a gap, not as zero findings."))))); } else { el_val_t snap = wt_read(snap_path); el_val_t edges_raw = json_get_raw(snap, EL_STR("edges")); el_val_t edges = ({ el_val_t _if_result_680 = 0; if (str_eq(edges_raw, EL_STR(""))) { _if_result_680 = (EL_STR("[]")); } else { _if_result_680 = (edges_raw); } _if_result_680; }); _if_result_679 = (el_str_concat(el_str_concat(el_str_concat(EL_STR(","), audit_edge_typing(edges, edge_total, node_total)), EL_STR(",")), audit_orphans_dangling(edges, edge_total, node_total, edge_cap, node_cap))); } _if_result_679; })); } else { _if_result_677 = (EL_STR("")); } _if_result_677; });
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"audit\":\"structural\",\"stage\":1"), EL_STR(",\"spec\":\"CGI provisional 05-detailed-description.md, Stage 1: Structural audit 430\"")), EL_STR(",\"assessment\":\"coherence_assessment_432\"")), EL_STR(",\"assessment_kind\":\"annotated_characterization\"")), EL_STR(",\"score\":null")), EL_STR(",\"score_note\":\"By design. The specification calls for an annotated characterization of the graph's structural properties, not a binary score. Read the findings.\"")), EL_STR(",\"cgi_id\":\"")), api_json_escape(state_get(EL_STR("soul_cgi_id")))), EL_STR("\"")), EL_STR(",\"ts_ms\":")), int_to_str(time_now())), EL_STR(",\"findings\":[")), divergence), EL_STR(",")), self_model), edge_part), EL_STR("]")), EL_STR(",\"deferred\":")), audit_deferred()), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t session_title_from_message(el_val_t message) {
|
||||
if (str_eq(message, EL_STR(""))) {
|
||||
return EL_STR("New conversation");
|
||||
@@ -30264,7 +30463,7 @@ el_val_t session_exists(el_val_t session_id) {
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t sid = json_get(content, EL_STR("id"));
|
||||
el_val_t is_match = (str_eq(label, EL_STR("session:meta")) && str_eq(sid, session_id));
|
||||
found = ({ el_val_t _if_result_653 = 0; if (is_match) { _if_result_653 = (1); } else { _if_result_653 = (found); } _if_result_653; });
|
||||
found = ({ el_val_t _if_result_681 = 0; if (is_match) { _if_result_681 = (1); } else { _if_result_681 = (found); } _if_result_681; });
|
||||
i = (i + 1);
|
||||
}
|
||||
return found;
|
||||
@@ -30275,7 +30474,7 @@ el_val_t session_create(el_val_t body) {
|
||||
el_val_t ts = time_now();
|
||||
el_val_t id = uuid_v4();
|
||||
el_val_t title_req = json_get(body, EL_STR("title"));
|
||||
el_val_t title = ({ el_val_t _if_result_654 = 0; if (str_eq(title_req, EL_STR(""))) { _if_result_654 = (EL_STR("New conversation")); } else { _if_result_654 = (title_req); } _if_result_654; });
|
||||
el_val_t title = ({ el_val_t _if_result_682 = 0; if (str_eq(title_req, EL_STR(""))) { _if_result_682 = (EL_STR("New conversation")); } else { _if_result_682 = (title_req); } _if_result_682; });
|
||||
el_val_t folder = json_get(body, EL_STR("folder"));
|
||||
el_val_t content = session_make_content(id, title, ts, ts, folder);
|
||||
el_val_t tags = EL_STR("[\"session\",\"session:meta\",\"Conversation\"]");
|
||||
@@ -30287,7 +30486,7 @@ el_val_t session_create(el_val_t body) {
|
||||
state_set(el_str_concat(EL_STR("session_pending_first_msg_"), id), EL_STR("1"));
|
||||
el_val_t existing_idx = state_get(EL_STR("session_index"));
|
||||
el_val_t idx_entry = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"title\":\"")), json_safe(title)), EL_STR("\",\"folder\":\"")), json_safe(folder)), EL_STR("\",\"created_at\":")), int_to_str(ts)), EL_STR(",\"updated_at\":")), int_to_str(ts)), EL_STR(",\"last_message\":\"\"}"));
|
||||
el_val_t new_idx = ({ el_val_t _if_result_655 = 0; if (str_eq(existing_idx, EL_STR(""))) { _if_result_655 = (el_str_concat(el_str_concat(EL_STR("["), idx_entry), EL_STR("]"))); } else { el_val_t inner = str_slice(existing_idx, 1, (str_len(existing_idx) - 1)); _if_result_655 = (el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("["), idx_entry), EL_STR(",")), inner), EL_STR("]"))); } _if_result_655; });
|
||||
el_val_t new_idx = ({ el_val_t _if_result_683 = 0; if (str_eq(existing_idx, EL_STR(""))) { _if_result_683 = (el_str_concat(el_str_concat(EL_STR("["), idx_entry), EL_STR("]"))); } else { el_val_t inner = str_slice(existing_idx, 1, (str_len(existing_idx) - 1)); _if_result_683 = (el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("["), idx_entry), EL_STR(",")), inner), EL_STR("]"))); } _if_result_683; });
|
||||
state_set(EL_STR("session_index"), new_idx);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\"")), EL_STR(",\"title\":\"")), json_safe(title)), EL_STR("\"")), EL_STR(",\"folder\":\"")), json_safe(folder)), EL_STR("\"")), EL_STR(",\"node_id\":\"")), node_id), EL_STR("\"")), EL_STR(",\"created_at\":")), int_to_str(ts)), EL_STR("}"));
|
||||
return 0;
|
||||
@@ -30324,16 +30523,16 @@ el_val_t session_list(void) {
|
||||
el_val_t is_session = (str_eq(label, EL_STR("session:meta")) && str_eq(node_type, EL_STR("Conversation")));
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t sess_id = json_get(content, EL_STR("id"));
|
||||
el_val_t eff_id = ({ el_val_t _if_result_656 = 0; if (str_eq(sess_id, EL_STR(""))) { _if_result_656 = (json_get(node, EL_STR("id"))); } else { _if_result_656 = (sess_id); } _if_result_656; });
|
||||
el_val_t eff_id = ({ el_val_t _if_result_684 = 0; if (str_eq(sess_id, EL_STR(""))) { _if_result_684 = (json_get(node, EL_STR("id"))); } else { _if_result_684 = (sess_id); } _if_result_684; });
|
||||
el_val_t title_inner = json_get(content, EL_STR("title"));
|
||||
el_val_t eff_title = ({ el_val_t _if_result_657 = 0; if (str_eq(title_inner, EL_STR(""))) { _if_result_657 = (EL_STR("New conversation")); } else { _if_result_657 = (title_inner); } _if_result_657; });
|
||||
el_val_t eff_title = ({ el_val_t _if_result_685 = 0; if (str_eq(title_inner, EL_STR(""))) { _if_result_685 = (EL_STR("New conversation")); } else { _if_result_685 = (title_inner); } _if_result_685; });
|
||||
el_val_t folder_inner = json_get(content, EL_STR("folder"));
|
||||
el_val_t created_inner = json_get(content, EL_STR("created_at"));
|
||||
el_val_t updated_inner = json_get(content, EL_STR("updated_at"));
|
||||
el_val_t eff_created = ({ el_val_t _if_result_658 = 0; if (str_eq(created_inner, EL_STR(""))) { _if_result_658 = (EL_STR("0")); } else { _if_result_658 = (created_inner); } _if_result_658; });
|
||||
el_val_t eff_updated = ({ el_val_t _if_result_659 = 0; if (str_eq(updated_inner, EL_STR(""))) { _if_result_659 = (eff_created); } else { _if_result_659 = (updated_inner); } _if_result_659; });
|
||||
el_val_t entry = ({ el_val_t _if_result_660 = 0; if (is_session) { _if_result_660 = (el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), json_safe(eff_id)), EL_STR("\"")), EL_STR(",\"title\":\"")), json_safe(eff_title)), EL_STR("\"")), EL_STR(",\"folder\":\"")), json_safe(folder_inner)), EL_STR("\"")), EL_STR(",\"last_message\":\"\"")), EL_STR(",\"created_at\":")), eff_created), EL_STR(",\"updated_at\":")), eff_updated), EL_STR("}"))); } else { _if_result_660 = (EL_STR("")); } _if_result_660; });
|
||||
out = ({ el_val_t _if_result_661 = 0; if (!str_eq(entry, EL_STR(""))) { _if_result_661 = (({ el_val_t _if_result_662 = 0; if (str_eq(out, EL_STR(""))) { _if_result_662 = (entry); } else { _if_result_662 = (el_str_concat(el_str_concat(out, EL_STR(",")), entry)); } _if_result_662; })); } else { _if_result_661 = (out); } _if_result_661; });
|
||||
el_val_t eff_created = ({ el_val_t _if_result_686 = 0; if (str_eq(created_inner, EL_STR(""))) { _if_result_686 = (EL_STR("0")); } else { _if_result_686 = (created_inner); } _if_result_686; });
|
||||
el_val_t eff_updated = ({ el_val_t _if_result_687 = 0; if (str_eq(updated_inner, EL_STR(""))) { _if_result_687 = (eff_created); } else { _if_result_687 = (updated_inner); } _if_result_687; });
|
||||
el_val_t entry = ({ el_val_t _if_result_688 = 0; if (is_session) { _if_result_688 = (el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), json_safe(eff_id)), EL_STR("\"")), EL_STR(",\"title\":\"")), json_safe(eff_title)), EL_STR("\"")), EL_STR(",\"folder\":\"")), json_safe(folder_inner)), EL_STR("\"")), EL_STR(",\"last_message\":\"\"")), EL_STR(",\"created_at\":")), eff_created), EL_STR(",\"updated_at\":")), eff_updated), EL_STR("}"))); } else { _if_result_688 = (EL_STR("")); } _if_result_688; });
|
||||
out = ({ el_val_t _if_result_689 = 0; if (!str_eq(entry, EL_STR(""))) { _if_result_689 = (({ el_val_t _if_result_690 = 0; if (str_eq(out, EL_STR(""))) { _if_result_690 = (entry); } else { _if_result_690 = (el_str_concat(el_str_concat(out, EL_STR(",")), entry)); } _if_result_690; })); } else { _if_result_689 = (out); } _if_result_689; });
|
||||
i = (i + 1);
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("["), out), EL_STR("]"));
|
||||
@@ -30351,7 +30550,7 @@ el_val_t session_get(el_val_t session_id) {
|
||||
el_val_t meta_created = EL_STR("0");
|
||||
el_val_t meta_updated = EL_STR("0");
|
||||
el_val_t found = 0;
|
||||
el_val_t total = ({ el_val_t _if_result_663 = 0; if (str_eq(results, EL_STR(""))) { _if_result_663 = (0); } else { _if_result_663 = (json_array_len(results)); } _if_result_663; });
|
||||
el_val_t total = ({ el_val_t _if_result_691 = 0; if (str_eq(results, EL_STR(""))) { _if_result_691 = (0); } else { _if_result_691 = (json_array_len(results)); } _if_result_691; });
|
||||
el_val_t i = 0;
|
||||
while (i < total) {
|
||||
el_val_t node = json_array_get(results, i);
|
||||
@@ -30359,17 +30558,17 @@ el_val_t session_get(el_val_t session_id) {
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t sid = json_get(content, EL_STR("id"));
|
||||
el_val_t is_match = ((str_eq(label, EL_STR("session:meta")) && str_eq(sid, session_id)) && !found);
|
||||
found = ({ el_val_t _if_result_664 = 0; if (is_match) { _if_result_664 = (1); } else { _if_result_664 = (found); } _if_result_664; });
|
||||
meta_title = ({ el_val_t _if_result_665 = 0; if (is_match) { _if_result_665 = (json_get(content, EL_STR("title"))); } else { _if_result_665 = (meta_title); } _if_result_665; });
|
||||
meta_folder = ({ el_val_t _if_result_666 = 0; if (is_match) { _if_result_666 = (json_get(content, EL_STR("folder"))); } else { _if_result_666 = (meta_folder); } _if_result_666; });
|
||||
found = ({ el_val_t _if_result_692 = 0; if (is_match) { _if_result_692 = (1); } else { _if_result_692 = (found); } _if_result_692; });
|
||||
meta_title = ({ el_val_t _if_result_693 = 0; if (is_match) { _if_result_693 = (json_get(content, EL_STR("title"))); } else { _if_result_693 = (meta_title); } _if_result_693; });
|
||||
meta_folder = ({ el_val_t _if_result_694 = 0; if (is_match) { _if_result_694 = (json_get(content, EL_STR("folder"))); } else { _if_result_694 = (meta_folder); } _if_result_694; });
|
||||
el_val_t meta_created_raw = json_get(content, EL_STR("created_at"));
|
||||
meta_created = ({ el_val_t _if_result_667 = 0; if ((is_match && !str_eq(meta_created_raw, EL_STR("")))) { _if_result_667 = (meta_created_raw); } else { _if_result_667 = (meta_created); } _if_result_667; });
|
||||
meta_created = ({ el_val_t _if_result_695 = 0; if ((is_match && !str_eq(meta_created_raw, EL_STR("")))) { _if_result_695 = (meta_created_raw); } else { _if_result_695 = (meta_created); } _if_result_695; });
|
||||
el_val_t meta_updated_raw = json_get(content, EL_STR("updated_at"));
|
||||
meta_updated = ({ el_val_t _if_result_668 = 0; if ((is_match && !str_eq(meta_updated_raw, EL_STR("")))) { _if_result_668 = (meta_updated_raw); } else { _if_result_668 = (meta_updated); } _if_result_668; });
|
||||
meta_updated = ({ el_val_t _if_result_696 = 0; if ((is_match && !str_eq(meta_updated_raw, EL_STR("")))) { _if_result_696 = (meta_updated_raw); } else { _if_result_696 = (meta_updated); } _if_result_696; });
|
||||
i = (i + 1);
|
||||
}
|
||||
el_val_t state_hist = state_get(el_str_concat(EL_STR("session_hist_"), session_id));
|
||||
el_val_t hist_raw = ({ el_val_t _if_result_669 = 0; if (str_eq(state_hist, EL_STR(""))) { el_val_t engram_hist = engram_search_json(el_str_concat(EL_STR("session:messages:"), session_id), 3); _if_result_669 = (({ el_val_t _if_result_670 = 0; if (str_eq(engram_hist, EL_STR(""))) { _if_result_670 = (EL_STR("[]")); } else { _if_result_670 = (({ el_val_t _if_result_671 = 0; if (str_eq(engram_hist, EL_STR("[]"))) { _if_result_671 = (EL_STR("[]")); } else { el_val_t h_node = json_array_get(engram_hist, 0); el_val_t h_content = json_get(h_node, EL_STR("content")); _if_result_671 = (({ el_val_t _if_result_672 = 0; if (str_starts_with(h_content, EL_STR("["))) { _if_result_672 = (h_content); } else { _if_result_672 = (EL_STR("[]")); } _if_result_672; })); } _if_result_671; })); } _if_result_670; })); } else { _if_result_669 = (state_hist); } _if_result_669; });
|
||||
el_val_t hist_raw = ({ el_val_t _if_result_697 = 0; if (str_eq(state_hist, EL_STR(""))) { el_val_t engram_hist = engram_search_json(el_str_concat(EL_STR("session:messages:"), session_id), 3); _if_result_697 = (({ el_val_t _if_result_698 = 0; if (str_eq(engram_hist, EL_STR(""))) { _if_result_698 = (EL_STR("[]")); } else { _if_result_698 = (({ el_val_t _if_result_699 = 0; if (str_eq(engram_hist, EL_STR("[]"))) { _if_result_699 = (EL_STR("[]")); } else { el_val_t h_node = json_array_get(engram_hist, 0); el_val_t h_content = json_get(h_node, EL_STR("content")); _if_result_699 = (({ el_val_t _if_result_700 = 0; if (str_starts_with(h_content, EL_STR("["))) { _if_result_700 = (h_content); } else { _if_result_700 = (EL_STR("[]")); } _if_result_700; })); } _if_result_699; })); } _if_result_698; })); } else { _if_result_697 = (state_hist); } _if_result_697; });
|
||||
el_val_t safe_title = json_safe(meta_title);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), session_id), EL_STR("\"")), EL_STR(",\"title\":\"")), safe_title), EL_STR("\"")), EL_STR(",\"folder\":\"")), json_safe(meta_folder)), EL_STR("\"")), EL_STR(",\"created_at\":")), meta_created), EL_STR(",\"updated_at\":")), meta_updated), EL_STR(",\"messages\":")), hist_raw), EL_STR("}"));
|
||||
return 0;
|
||||
@@ -30380,7 +30579,7 @@ el_val_t session_delete(el_val_t session_id) {
|
||||
return EL_STR("{\"error\":\"session_id is required\"}");
|
||||
}
|
||||
el_val_t results = engram_search_json(el_str_concat(EL_STR("session:meta "), session_id), 10);
|
||||
el_val_t total = ({ el_val_t _if_result_673 = 0; if (str_eq(results, EL_STR(""))) { _if_result_673 = (0); } else { _if_result_673 = (json_array_len(results)); } _if_result_673; });
|
||||
el_val_t total = ({ el_val_t _if_result_701 = 0; if (str_eq(results, EL_STR(""))) { _if_result_701 = (0); } else { _if_result_701 = (json_array_len(results)); } _if_result_701; });
|
||||
el_val_t deleted_meta = 0;
|
||||
el_val_t i = 0;
|
||||
while (i < total) {
|
||||
@@ -30390,11 +30589,11 @@ el_val_t session_delete(el_val_t session_id) {
|
||||
el_val_t sid = json_get(content, EL_STR("id"));
|
||||
el_val_t is_match = (str_eq(label, EL_STR("session:meta")) && str_eq(sid, session_id));
|
||||
el_val_t node_id = json_get(node, EL_STR("id"));
|
||||
deleted_meta = ({ el_val_t _if_result_674 = 0; if ((is_match && !str_eq(node_id, EL_STR("")))) { (void)(engram_forget(node_id)); _if_result_674 = ((deleted_meta + 1)); } else { _if_result_674 = (deleted_meta); } _if_result_674; });
|
||||
deleted_meta = ({ el_val_t _if_result_702 = 0; if ((is_match && !str_eq(node_id, EL_STR("")))) { (void)(engram_forget(node_id)); _if_result_702 = ((deleted_meta + 1)); } else { _if_result_702 = (deleted_meta); } _if_result_702; });
|
||||
i = (i + 1);
|
||||
}
|
||||
el_val_t msg_results = engram_search_json(el_str_concat(EL_STR("session:messages:"), session_id), 10);
|
||||
el_val_t m_total = ({ el_val_t _if_result_675 = 0; if (str_eq(msg_results, EL_STR(""))) { _if_result_675 = (0); } else { _if_result_675 = (json_array_len(msg_results)); } _if_result_675; });
|
||||
el_val_t m_total = ({ el_val_t _if_result_703 = 0; if (str_eq(msg_results, EL_STR(""))) { _if_result_703 = (0); } else { _if_result_703 = (json_array_len(msg_results)); } _if_result_703; });
|
||||
el_val_t deleted_msgs = 0;
|
||||
el_val_t j = 0;
|
||||
while (j < m_total) {
|
||||
@@ -30402,7 +30601,7 @@ el_val_t session_delete(el_val_t session_id) {
|
||||
el_val_t label = json_get(node, EL_STR("label"));
|
||||
el_val_t is_msgs = str_eq(label, el_str_concat(EL_STR("session:messages:"), session_id));
|
||||
el_val_t node_id = json_get(node, EL_STR("id"));
|
||||
deleted_msgs = ({ el_val_t _if_result_676 = 0; if ((is_msgs && !str_eq(node_id, EL_STR("")))) { (void)(engram_forget(node_id)); _if_result_676 = ((deleted_msgs + 1)); } else { _if_result_676 = (deleted_msgs); } _if_result_676; });
|
||||
deleted_msgs = ({ el_val_t _if_result_704 = 0; if ((is_msgs && !str_eq(node_id, EL_STR("")))) { (void)(engram_forget(node_id)); _if_result_704 = ((deleted_msgs + 1)); } else { _if_result_704 = (deleted_msgs); } _if_result_704; });
|
||||
j = (j + 1);
|
||||
}
|
||||
state_set(el_str_concat(EL_STR("session_hist_"), session_id), EL_STR(""));
|
||||
@@ -30425,7 +30624,7 @@ el_val_t session_update_patch(el_val_t session_id, el_val_t body) {
|
||||
return EL_STR("{\"error\":\"title or folder required in body\"}");
|
||||
}
|
||||
el_val_t results = engram_search_json(EL_STR("session:meta"), 50);
|
||||
el_val_t total = ({ el_val_t _if_result_677 = 0; if (str_eq(results, EL_STR(""))) { _if_result_677 = (0); } else { _if_result_677 = (json_array_len(results)); } _if_result_677; });
|
||||
el_val_t total = ({ el_val_t _if_result_705 = 0; if (str_eq(results, EL_STR(""))) { _if_result_705 = (0); } else { _if_result_705 = (json_array_len(results)); } _if_result_705; });
|
||||
el_val_t found = 0;
|
||||
el_val_t old_title = EL_STR("New conversation");
|
||||
el_val_t old_folder = EL_STR("");
|
||||
@@ -30438,23 +30637,23 @@ el_val_t session_update_patch(el_val_t session_id, el_val_t body) {
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t sid = json_get(content, EL_STR("id"));
|
||||
el_val_t is_match = ((str_eq(label, EL_STR("session:meta")) && str_eq(sid, session_id)) && !found);
|
||||
found = ({ el_val_t _if_result_678 = 0; if (is_match) { _if_result_678 = (1); } else { _if_result_678 = (found); } _if_result_678; });
|
||||
found = ({ el_val_t _if_result_706 = 0; if (is_match) { _if_result_706 = (1); } else { _if_result_706 = (found); } _if_result_706; });
|
||||
el_val_t title_raw = json_get(content, EL_STR("title"));
|
||||
old_title = ({ el_val_t _if_result_679 = 0; if ((is_match && !str_eq(title_raw, EL_STR("")))) { _if_result_679 = (title_raw); } else { _if_result_679 = (old_title); } _if_result_679; });
|
||||
old_title = ({ el_val_t _if_result_707 = 0; if ((is_match && !str_eq(title_raw, EL_STR("")))) { _if_result_707 = (title_raw); } else { _if_result_707 = (old_title); } _if_result_707; });
|
||||
el_val_t folder_raw = json_get(content, EL_STR("folder"));
|
||||
old_folder = ({ el_val_t _if_result_680 = 0; if (is_match) { _if_result_680 = (folder_raw); } else { _if_result_680 = (old_folder); } _if_result_680; });
|
||||
old_folder = ({ el_val_t _if_result_708 = 0; if (is_match) { _if_result_708 = (folder_raw); } else { _if_result_708 = (old_folder); } _if_result_708; });
|
||||
el_val_t created_raw = json_get(content, EL_STR("created_at"));
|
||||
old_created = ({ el_val_t _if_result_681 = 0; if ((is_match && !str_eq(created_raw, EL_STR("")))) { _if_result_681 = (created_raw); } else { _if_result_681 = (old_created); } _if_result_681; });
|
||||
old_created = ({ el_val_t _if_result_709 = 0; if ((is_match && !str_eq(created_raw, EL_STR("")))) { _if_result_709 = (created_raw); } else { _if_result_709 = (old_created); } _if_result_709; });
|
||||
el_val_t nid = json_get(node, EL_STR("id"));
|
||||
old_node_id = ({ el_val_t _if_result_682 = 0; if (is_match) { _if_result_682 = (nid); } else { _if_result_682 = (old_node_id); } _if_result_682; });
|
||||
old_node_id = ({ el_val_t _if_result_710 = 0; if (is_match) { _if_result_710 = (nid); } else { _if_result_710 = (old_node_id); } _if_result_710; });
|
||||
i = (i + 1);
|
||||
}
|
||||
if (!found) {
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"error\":\"session not found\",\"session_id\":\""), session_id), EL_STR("\"}"));
|
||||
}
|
||||
el_val_t req_title = json_get(body, EL_STR("title"));
|
||||
el_val_t eff_title = ({ el_val_t _if_result_683 = 0; if ((has_title && !str_eq(req_title, EL_STR("")))) { _if_result_683 = (req_title); } else { _if_result_683 = (old_title); } _if_result_683; });
|
||||
el_val_t eff_folder = ({ el_val_t _if_result_684 = 0; if (has_folder) { _if_result_684 = (json_get(body, EL_STR("folder"))); } else { _if_result_684 = (old_folder); } _if_result_684; });
|
||||
el_val_t eff_title = ({ el_val_t _if_result_711 = 0; if ((has_title && !str_eq(req_title, EL_STR("")))) { _if_result_711 = (req_title); } else { _if_result_711 = (old_title); } _if_result_711; });
|
||||
el_val_t eff_folder = ({ el_val_t _if_result_712 = 0; if (has_folder) { _if_result_712 = (json_get(body, EL_STR("folder"))); } else { _if_result_712 = (old_folder); } _if_result_712; });
|
||||
if (!str_eq(old_node_id, EL_STR(""))) {
|
||||
engram_forget(old_node_id);
|
||||
}
|
||||
@@ -30482,8 +30681,8 @@ el_val_t session_search_entry(el_val_t node) {
|
||||
el_val_t title = json_get(content, EL_STR("title"));
|
||||
el_val_t created_raw = json_get(content, EL_STR("created_at"));
|
||||
el_val_t updated_raw = json_get(content, EL_STR("updated_at"));
|
||||
el_val_t eff_created = ({ el_val_t _if_result_685 = 0; if (str_eq(created_raw, EL_STR(""))) { _if_result_685 = (EL_STR("0")); } else { _if_result_685 = (created_raw); } _if_result_685; });
|
||||
el_val_t eff_updated = ({ el_val_t _if_result_686 = 0; if (str_eq(updated_raw, EL_STR(""))) { _if_result_686 = (eff_created); } else { _if_result_686 = (updated_raw); } _if_result_686; });
|
||||
el_val_t eff_created = ({ el_val_t _if_result_713 = 0; if (str_eq(created_raw, EL_STR(""))) { _if_result_713 = (EL_STR("0")); } else { _if_result_713 = (created_raw); } _if_result_713; });
|
||||
el_val_t eff_updated = ({ el_val_t _if_result_714 = 0; if (str_eq(updated_raw, EL_STR(""))) { _if_result_714 = (eff_created); } else { _if_result_714 = (updated_raw); } _if_result_714; });
|
||||
el_val_t e_id = el_str_concat(el_str_concat(EL_STR("{\"id\":\""), json_safe(sess_id)), EL_STR("\""));
|
||||
el_val_t e_title = el_str_concat(el_str_concat(EL_STR(",\"title\":\""), json_safe(title)), EL_STR("\""));
|
||||
el_val_t e_ts = el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR(",\"created_at\":"), eff_created), EL_STR(",\"updated_at\":")), eff_updated), EL_STR("}"));
|
||||
@@ -30507,7 +30706,7 @@ el_val_t session_search(el_val_t query) {
|
||||
el_val_t i = 0;
|
||||
while (i < total) {
|
||||
el_val_t entry = session_search_entry(json_array_get(results, i));
|
||||
out = ({ el_val_t _if_result_687 = 0; if (!str_eq(entry, EL_STR(""))) { _if_result_687 = (({ el_val_t _if_result_688 = 0; if (str_eq(out, EL_STR(""))) { _if_result_688 = (entry); } else { _if_result_688 = (el_str_concat(el_str_concat(out, EL_STR(",")), entry)); } _if_result_688; })); } else { _if_result_687 = (out); } _if_result_687; });
|
||||
out = ({ el_val_t _if_result_715 = 0; if (!str_eq(entry, EL_STR(""))) { _if_result_715 = (({ el_val_t _if_result_716 = 0; if (str_eq(out, EL_STR(""))) { _if_result_716 = (entry); } else { _if_result_716 = (el_str_concat(el_str_concat(out, EL_STR(",")), entry)); } _if_result_716; })); } else { _if_result_715 = (out); } _if_result_715; });
|
||||
i = (i + 1);
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("["), out), EL_STR("]"));
|
||||
@@ -30543,7 +30742,7 @@ el_val_t session_hist_save(el_val_t session_id, el_val_t hist) {
|
||||
state_set(el_str_concat(EL_STR("session_hist_"), session_id), hist);
|
||||
state_set(el_str_concat(EL_STR("session_pending_first_msg_"), session_id), EL_STR(""));
|
||||
el_val_t old_results = engram_search_json(el_str_concat(EL_STR("session:messages:"), session_id), 3);
|
||||
el_val_t o_total = ({ el_val_t _if_result_689 = 0; if (str_eq(old_results, EL_STR(""))) { _if_result_689 = (0); } else { _if_result_689 = (json_array_len(old_results)); } _if_result_689; });
|
||||
el_val_t o_total = ({ el_val_t _if_result_717 = 0; if (str_eq(old_results, EL_STR(""))) { _if_result_717 = (0); } else { _if_result_717 = (json_array_len(old_results)); } _if_result_717; });
|
||||
el_val_t oi = 0;
|
||||
while (oi < o_total) {
|
||||
el_val_t node = json_array_get(old_results, oi);
|
||||
@@ -30561,35 +30760,35 @@ el_val_t session_hist_save(el_val_t session_id, el_val_t hist) {
|
||||
if (str_eq(already_written, EL_STR(""))) {
|
||||
el_val_t bell_count_key = el_str_concat(EL_STR("session_bell_count:"), session_id);
|
||||
el_val_t bell_count_raw = state_get(bell_count_key);
|
||||
el_val_t bell_count = ({ el_val_t _if_result_690 = 0; if (str_eq(bell_count_raw, EL_STR(""))) { _if_result_690 = (0); } else { _if_result_690 = (str_to_int(bell_count_raw)); } _if_result_690; });
|
||||
el_val_t bell_count = ({ el_val_t _if_result_718 = 0; if (str_eq(bell_count_raw, EL_STR(""))) { _if_result_718 = (0); } else { _if_result_718 = (str_to_int(bell_count_raw)); } _if_result_718; });
|
||||
if (bell_count > 0) {
|
||||
el_val_t bell_level_key = el_str_concat(EL_STR("session_bell_level:"), session_id);
|
||||
el_val_t bell_signal_key = el_str_concat(EL_STR("session_bell_signal:"), session_id);
|
||||
el_val_t dominant_level = state_get(bell_level_key);
|
||||
el_val_t last_signal = state_get(bell_signal_key);
|
||||
el_val_t eff_level = ({ el_val_t _if_result_691 = 0; if (str_eq(dominant_level, EL_STR(""))) { _if_result_691 = (EL_STR("soft")); } else { _if_result_691 = (dominant_level); } _if_result_691; });
|
||||
el_val_t eff_signal = ({ el_val_t _if_result_692 = 0; if (str_eq(last_signal, EL_STR(""))) { _if_result_692 = (EL_STR("(no signal captured)")); } else { _if_result_692 = (last_signal); } _if_result_692; });
|
||||
el_val_t eff_level = ({ el_val_t _if_result_719 = 0; if (str_eq(dominant_level, EL_STR(""))) { _if_result_719 = (EL_STR("soft")); } else { _if_result_719 = (dominant_level); } _if_result_719; });
|
||||
el_val_t eff_signal = ({ el_val_t _if_result_720 = 0; if (str_eq(last_signal, EL_STR(""))) { _if_result_720 = (EL_STR("(no signal captured)")); } else { _if_result_720 = (last_signal); } _if_result_720; });
|
||||
el_val_t ts_now = time_now();
|
||||
el_val_t summary_content = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("session:emotional-summary"), EL_STR(" | session:")), session_id), EL_STR(" | bell_count:")), int_to_str(bell_count)), EL_STR(" | dominant_level:")), eff_level), EL_STR(" | last_signal:")), eff_signal), EL_STR(" | ts:")), int_to_str(ts_now));
|
||||
el_val_t summary_tags = el_str_concat(el_str_concat(EL_STR("[\"session-emotional-summary\",\"affective\",\"bell:"), eff_level), EL_STR("\",\"BellEvent\"]"));
|
||||
el_val_t summary_sal = ({ el_val_t _if_result_693 = 0; if (str_eq(eff_level, EL_STR("hard"))) { _if_result_693 = (el_from_float(0.95)); } else { _if_result_693 = (el_from_float(0.85)); } _if_result_693; });
|
||||
el_val_t summary_sal = ({ el_val_t _if_result_721 = 0; if (str_eq(eff_level, EL_STR("hard"))) { _if_result_721 = (el_from_float(0.95)); } else { _if_result_721 = (el_from_float(0.85)); } _if_result_721; });
|
||||
el_val_t sum_discard = wt_node(summary_content, EL_STR("BellEvent"), EL_STR("session:emotional-summary"), summary_sal, summary_sal, el_from_float(1.0), EL_STR("Episodic"), summary_tags);
|
||||
state_set(summary_written_key, EL_STR("1"));
|
||||
}
|
||||
}
|
||||
el_val_t hist_arr_len = ({ el_val_t _if_result_694 = 0; if (str_eq(hist, EL_STR(""))) { _if_result_694 = (0); } else { _if_result_694 = (json_array_len(hist)); } _if_result_694; });
|
||||
el_val_t hist_arr_len = ({ el_val_t _if_result_722 = 0; if (str_eq(hist, EL_STR(""))) { _if_result_722 = (0); } else { _if_result_722 = (json_array_len(hist)); } _if_result_722; });
|
||||
if (hist_arr_len >= 2) {
|
||||
el_val_t last_entry = json_array_get(hist, (hist_arr_len - 1));
|
||||
el_val_t last_role = json_get(last_entry, EL_STR("role"));
|
||||
el_val_t last_content = json_get(last_entry, EL_STR("content"));
|
||||
el_val_t topic_snip = ({ el_val_t _if_result_695 = 0; if ((str_len(last_content) > 200)) { _if_result_695 = (str_slice(last_content, 0, 200)); } else { _if_result_695 = (last_content); } _if_result_695; });
|
||||
el_val_t topic_snip = ({ el_val_t _if_result_723 = 0; if ((str_len(last_content) > 200)) { _if_result_723 = (str_slice(last_content, 0, 200)); } else { _if_result_723 = (last_content); } _if_result_723; });
|
||||
el_val_t safe_topic = str_replace(topic_snip, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t ts_now = int_to_str(time_now());
|
||||
el_val_t topic_content = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("last-session-topic | ts:"), ts_now), EL_STR(" | session:")), session_id), EL_STR(" | topic:")), safe_topic);
|
||||
el_val_t topic_tags = EL_STR("[\"last-session-topic\",\"conv:history\",\"Conversation\",\"session:topic\"]");
|
||||
el_val_t topic_label = el_str_concat(EL_STR("last-session-topic:"), session_id);
|
||||
el_val_t old_topic = engram_search_json(el_str_concat(EL_STR("last-session-topic:"), session_id), 2);
|
||||
el_val_t ot_len = ({ el_val_t _if_result_696 = 0; if (str_eq(old_topic, EL_STR(""))) { _if_result_696 = (0); } else { _if_result_696 = (json_array_len(old_topic)); } _if_result_696; });
|
||||
el_val_t ot_len = ({ el_val_t _if_result_724 = 0; if (str_eq(old_topic, EL_STR(""))) { _if_result_724 = (0); } else { _if_result_724 = (json_array_len(old_topic)); } _if_result_724; });
|
||||
el_val_t oti = 0;
|
||||
while (oti < ot_len) {
|
||||
el_val_t ot_node = json_array_get(old_topic, oti);
|
||||
@@ -30606,7 +30805,7 @@ el_val_t session_hist_save(el_val_t session_id, el_val_t hist) {
|
||||
|
||||
el_val_t session_update_meta_timestamp(el_val_t session_id) {
|
||||
el_val_t results = engram_search_json(el_str_concat(EL_STR("session:meta "), session_id), 10);
|
||||
el_val_t total = ({ el_val_t _if_result_697 = 0; if (str_eq(results, EL_STR(""))) { _if_result_697 = (0); } else { _if_result_697 = (json_array_len(results)); } _if_result_697; });
|
||||
el_val_t total = ({ el_val_t _if_result_725 = 0; if (str_eq(results, EL_STR(""))) { _if_result_725 = (0); } else { _if_result_725 = (json_array_len(results)); } _if_result_725; });
|
||||
el_val_t found = 0;
|
||||
el_val_t old_title = EL_STR("New conversation");
|
||||
el_val_t old_folder = EL_STR("");
|
||||
@@ -30619,15 +30818,15 @@ el_val_t session_update_meta_timestamp(el_val_t session_id) {
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t sid = json_get(content, EL_STR("id"));
|
||||
el_val_t is_match = ((str_eq(label, EL_STR("session:meta")) && str_eq(sid, session_id)) && !found);
|
||||
found = ({ el_val_t _if_result_698 = 0; if (is_match) { _if_result_698 = (1); } else { _if_result_698 = (found); } _if_result_698; });
|
||||
found = ({ el_val_t _if_result_726 = 0; if (is_match) { _if_result_726 = (1); } else { _if_result_726 = (found); } _if_result_726; });
|
||||
el_val_t title_raw = json_get(content, EL_STR("title"));
|
||||
old_title = ({ el_val_t _if_result_699 = 0; if ((is_match && !str_eq(title_raw, EL_STR("")))) { _if_result_699 = (title_raw); } else { _if_result_699 = (old_title); } _if_result_699; });
|
||||
old_title = ({ el_val_t _if_result_727 = 0; if ((is_match && !str_eq(title_raw, EL_STR("")))) { _if_result_727 = (title_raw); } else { _if_result_727 = (old_title); } _if_result_727; });
|
||||
el_val_t folder_raw = json_get(content, EL_STR("folder"));
|
||||
old_folder = ({ el_val_t _if_result_700 = 0; if (is_match) { _if_result_700 = (folder_raw); } else { _if_result_700 = (old_folder); } _if_result_700; });
|
||||
old_folder = ({ el_val_t _if_result_728 = 0; if (is_match) { _if_result_728 = (folder_raw); } else { _if_result_728 = (old_folder); } _if_result_728; });
|
||||
el_val_t created_raw = json_get(content, EL_STR("created_at"));
|
||||
old_created = ({ el_val_t _if_result_701 = 0; if ((is_match && !str_eq(created_raw, EL_STR("")))) { _if_result_701 = (created_raw); } else { _if_result_701 = (old_created); } _if_result_701; });
|
||||
old_created = ({ el_val_t _if_result_729 = 0; if ((is_match && !str_eq(created_raw, EL_STR("")))) { _if_result_729 = (created_raw); } else { _if_result_729 = (old_created); } _if_result_729; });
|
||||
el_val_t nid = json_get(node, EL_STR("id"));
|
||||
old_node_id = ({ el_val_t _if_result_702 = 0; if (is_match) { _if_result_702 = (nid); } else { _if_result_702 = (old_node_id); } _if_result_702; });
|
||||
old_node_id = ({ el_val_t _if_result_730 = 0; if (is_match) { _if_result_730 = (nid); } else { _if_result_730 = (old_node_id); } _if_result_730; });
|
||||
i = (i + 1);
|
||||
}
|
||||
if (!found) {
|
||||
@@ -30647,7 +30846,7 @@ el_val_t session_update_meta_timestamp(el_val_t session_id) {
|
||||
|
||||
el_val_t session_auto_title(el_val_t session_id, el_val_t first_message) {
|
||||
el_val_t results = engram_search_json(el_str_concat(EL_STR("session:meta "), session_id), 10);
|
||||
el_val_t total = ({ el_val_t _if_result_703 = 0; if (str_eq(results, EL_STR(""))) { _if_result_703 = (0); } else { _if_result_703 = (json_array_len(results)); } _if_result_703; });
|
||||
el_val_t total = ({ el_val_t _if_result_731 = 0; if (str_eq(results, EL_STR(""))) { _if_result_731 = (0); } else { _if_result_731 = (json_array_len(results)); } _if_result_731; });
|
||||
el_val_t found = 0;
|
||||
el_val_t cur_title = EL_STR("");
|
||||
el_val_t old_folder = EL_STR("");
|
||||
@@ -30660,15 +30859,15 @@ el_val_t session_auto_title(el_val_t session_id, el_val_t first_message) {
|
||||
el_val_t content = json_get(node, EL_STR("content"));
|
||||
el_val_t sid = json_get(content, EL_STR("id"));
|
||||
el_val_t is_match = ((str_eq(label, EL_STR("session:meta")) && str_eq(sid, session_id)) && !found);
|
||||
found = ({ el_val_t _if_result_704 = 0; if (is_match) { _if_result_704 = (1); } else { _if_result_704 = (found); } _if_result_704; });
|
||||
found = ({ el_val_t _if_result_732 = 0; if (is_match) { _if_result_732 = (1); } else { _if_result_732 = (found); } _if_result_732; });
|
||||
el_val_t title_raw = json_get(content, EL_STR("title"));
|
||||
cur_title = ({ el_val_t _if_result_705 = 0; if (is_match) { _if_result_705 = (title_raw); } else { _if_result_705 = (cur_title); } _if_result_705; });
|
||||
cur_title = ({ el_val_t _if_result_733 = 0; if (is_match) { _if_result_733 = (title_raw); } else { _if_result_733 = (cur_title); } _if_result_733; });
|
||||
el_val_t folder_raw = json_get(content, EL_STR("folder"));
|
||||
old_folder = ({ el_val_t _if_result_706 = 0; if (is_match) { _if_result_706 = (folder_raw); } else { _if_result_706 = (old_folder); } _if_result_706; });
|
||||
old_folder = ({ el_val_t _if_result_734 = 0; if (is_match) { _if_result_734 = (folder_raw); } else { _if_result_734 = (old_folder); } _if_result_734; });
|
||||
el_val_t created_raw = json_get(content, EL_STR("created_at"));
|
||||
old_created = ({ el_val_t _if_result_707 = 0; if ((is_match && !str_eq(created_raw, EL_STR("")))) { _if_result_707 = (created_raw); } else { _if_result_707 = (old_created); } _if_result_707; });
|
||||
old_created = ({ el_val_t _if_result_735 = 0; if ((is_match && !str_eq(created_raw, EL_STR("")))) { _if_result_735 = (created_raw); } else { _if_result_735 = (old_created); } _if_result_735; });
|
||||
el_val_t nid = json_get(node, EL_STR("id"));
|
||||
old_node_id = ({ el_val_t _if_result_708 = 0; if (is_match) { _if_result_708 = (nid); } else { _if_result_708 = (old_node_id); } _if_result_708; });
|
||||
old_node_id = ({ el_val_t _if_result_736 = 0; if (is_match) { _if_result_736 = (nid); } else { _if_result_736 = (old_node_id); } _if_result_736; });
|
||||
i = (i + 1);
|
||||
}
|
||||
if (!found) {
|
||||
@@ -30702,13 +30901,13 @@ el_val_t handle_session_approve(el_val_t session_id, el_val_t body) {
|
||||
if (str_eq(action, EL_STR(""))) {
|
||||
return EL_STR("{\"error\":\"action is required (allow|deny|always)\"}");
|
||||
}
|
||||
el_val_t eff_action = ({ el_val_t _if_result_709 = 0; if (str_eq(action, EL_STR("always"))) { _if_result_709 = (EL_STR("allow")); } else { _if_result_709 = (action); } _if_result_709; });
|
||||
el_val_t eff_action = ({ el_val_t _if_result_737 = 0; if (str_eq(action, EL_STR("always"))) { _if_result_737 = (EL_STR("allow")); } else { _if_result_737 = (action); } _if_result_737; });
|
||||
el_val_t bridge_blob = state_get(el_str_concat(EL_STR("mcp_bridge:"), session_id));
|
||||
if (!str_eq(bridge_blob, EL_STR(""))) {
|
||||
state_set(EL_STR("agent_workspace_root"), state_get(el_str_concat(EL_STR("agent_workspace_root_"), session_id)));
|
||||
el_val_t always_key = el_str_concat(EL_STR("always_allow_"), session_id);
|
||||
el_val_t approve_tool_name = json_get(body, EL_STR("tool_name"));
|
||||
el_val_t discard_always = ({ el_val_t _if_result_710 = 0; if ((str_eq(action, EL_STR("always")) && !str_eq(approve_tool_name, EL_STR("")))) { el_val_t always_list = state_get(always_key); el_val_t new_always = ({ el_val_t _if_result_711 = 0; if (str_eq(always_list, EL_STR(""))) { _if_result_711 = (approve_tool_name); } else { _if_result_711 = (el_str_concat(el_str_concat(always_list, EL_STR(",")), approve_tool_name)); } _if_result_711; }); (void)(state_set(always_key, new_always)); _if_result_710 = (1); } else { _if_result_710 = (0); } _if_result_710; });
|
||||
el_val_t discard_always = ({ el_val_t _if_result_738 = 0; if ((str_eq(action, EL_STR("always")) && !str_eq(approve_tool_name, EL_STR("")))) { el_val_t always_list = state_get(always_key); el_val_t new_always = ({ el_val_t _if_result_739 = 0; if (str_eq(always_list, EL_STR(""))) { _if_result_739 = (approve_tool_name); } else { _if_result_739 = (el_str_concat(el_str_concat(always_list, EL_STR(",")), approve_tool_name)); } _if_result_739; }); (void)(state_set(always_key, new_always)); _if_result_738 = (1); } else { _if_result_738 = (0); } _if_result_738; });
|
||||
if (str_eq(approve_tool_name, EL_STR("")) && str_eq(eff_action, EL_STR("allow"))) {
|
||||
return EL_STR("{\"error\":\"tool_name is required for allow action\"}");
|
||||
}
|
||||
@@ -30716,8 +30915,8 @@ el_val_t handle_session_approve(el_val_t session_id, el_val_t body) {
|
||||
el_val_t use_client_content = (!str_eq(client_content, EL_STR("")) && !is_builtin_tool(approve_tool_name));
|
||||
el_val_t use_dispatch = (is_builtin_tool(approve_tool_name) && !use_client_content);
|
||||
el_val_t raw_input = json_get_raw(body, EL_STR("tool_input"));
|
||||
el_val_t eff_input = ({ el_val_t _if_result_712 = 0; if (str_eq(raw_input, EL_STR(""))) { _if_result_712 = (EL_STR("{}")); } else { _if_result_712 = (raw_input); } _if_result_712; });
|
||||
el_val_t content = ({ el_val_t _if_result_713 = 0; if (str_eq(eff_action, EL_STR("allow"))) { _if_result_713 = (({ el_val_t _if_result_714 = 0; if (use_client_content) { el_val_t trimmed = ({ el_val_t _if_result_715 = 0; if ((str_len(client_content) > 6000)) { _if_result_715 = (el_str_concat(str_slice(client_content, 0, 6000), EL_STR("...[truncated]"))); } else { _if_result_715 = (client_content); } _if_result_715; }); _if_result_714 = (trimmed); } else { _if_result_714 = (({ el_val_t _if_result_716 = 0; if (use_dispatch) { el_val_t raw = dispatch_tool(approve_tool_name, eff_input); _if_result_716 = (({ el_val_t _if_result_717 = 0; if ((str_len(raw) > 6000)) { _if_result_717 = (el_str_concat(str_slice(raw, 0, 6000), EL_STR("...[truncated]"))); } else { _if_result_717 = (raw); } _if_result_717; })); } else { _if_result_716 = (el_str_concat(el_str_concat(EL_STR("{\"error\":\"client content required for non-builtin tool: "), approve_tool_name), EL_STR("\"}"))); } _if_result_716; })); } _if_result_714; })); } else { _if_result_713 = (EL_STR("{\"error\":\"User denied this tool call\"}")); } _if_result_713; });
|
||||
el_val_t eff_input = ({ el_val_t _if_result_740 = 0; if (str_eq(raw_input, EL_STR(""))) { _if_result_740 = (EL_STR("{}")); } else { _if_result_740 = (raw_input); } _if_result_740; });
|
||||
el_val_t content = ({ el_val_t _if_result_741 = 0; if (str_eq(eff_action, EL_STR("allow"))) { _if_result_741 = (({ el_val_t _if_result_742 = 0; if (use_client_content) { el_val_t trimmed = ({ el_val_t _if_result_743 = 0; if ((str_len(client_content) > 6000)) { _if_result_743 = (el_str_concat(str_slice(client_content, 0, 6000), EL_STR("...[truncated]"))); } else { _if_result_743 = (client_content); } _if_result_743; }); _if_result_742 = (trimmed); } else { _if_result_742 = (({ el_val_t _if_result_744 = 0; if (use_dispatch) { el_val_t raw = dispatch_tool(approve_tool_name, eff_input); _if_result_744 = (({ el_val_t _if_result_745 = 0; if ((str_len(raw) > 6000)) { _if_result_745 = (el_str_concat(str_slice(raw, 0, 6000), EL_STR("...[truncated]"))); } else { _if_result_745 = (raw); } _if_result_745; })); } else { _if_result_744 = (el_str_concat(el_str_concat(EL_STR("{\"error\":\"client content required for non-builtin tool: "), approve_tool_name), EL_STR("\"}"))); } _if_result_744; })); } _if_result_742; })); } else { _if_result_741 = (EL_STR("{\"error\":\"User denied this tool call\"}")); } _if_result_741; });
|
||||
return agentic_resume(session_id, call_id, content);
|
||||
}
|
||||
el_val_t pending_raw = state_get(el_str_concat(EL_STR("pending_tool_"), session_id));
|
||||
@@ -30734,12 +30933,12 @@ el_val_t handle_session_approve(el_val_t session_id, el_val_t body) {
|
||||
el_val_t safe_sys = json_get(pending_raw, EL_STR("system"));
|
||||
el_val_t always_key = el_str_concat(EL_STR("always_allow_"), session_id);
|
||||
el_val_t always_list = state_get(always_key);
|
||||
el_val_t discard_always2 = ({ el_val_t _if_result_718 = 0; if (str_eq(action, EL_STR("always"))) { el_val_t new_always = ({ el_val_t _if_result_719 = 0; if (str_eq(always_list, EL_STR(""))) { _if_result_719 = (tool_name); } else { _if_result_719 = (el_str_concat(el_str_concat(always_list, EL_STR(",")), tool_name)); } _if_result_719; }); (void)(state_set(always_key, new_always)); _if_result_718 = (1); } else { _if_result_718 = (0); } _if_result_718; });
|
||||
el_val_t discard_always2 = ({ el_val_t _if_result_746 = 0; if (str_eq(action, EL_STR("always"))) { el_val_t new_always = ({ el_val_t _if_result_747 = 0; if (str_eq(always_list, EL_STR(""))) { _if_result_747 = (tool_name); } else { _if_result_747 = (el_str_concat(el_str_concat(always_list, EL_STR(",")), tool_name)); } _if_result_747; }); (void)(state_set(always_key, new_always)); _if_result_746 = (1); } else { _if_result_746 = (0); } _if_result_746; });
|
||||
state_set(el_str_concat(EL_STR("pending_tool_"), session_id), EL_STR(""));
|
||||
el_val_t tool_result = ({ el_val_t _if_result_720 = 0; if (str_eq(eff_action, EL_STR("allow"))) { el_val_t raw = dispatch_tool(tool_name, tool_input); _if_result_720 = (({ el_val_t _if_result_721 = 0; if ((str_len(raw) > 6000)) { _if_result_721 = (el_str_concat(str_slice(raw, 0, 6000), EL_STR("...[truncated]"))); } else { _if_result_721 = (raw); } _if_result_721; })); } else { _if_result_720 = (EL_STR("{\"error\":\"User denied this tool call\"}")); } _if_result_720; });
|
||||
el_val_t tool_result = ({ el_val_t _if_result_748 = 0; if (str_eq(eff_action, EL_STR("allow"))) { el_val_t raw = dispatch_tool(tool_name, tool_input); _if_result_748 = (({ el_val_t _if_result_749 = 0; if ((str_len(raw) > 6000)) { _if_result_749 = (el_str_concat(str_slice(raw, 0, 6000), EL_STR("...[truncated]"))); } else { _if_result_749 = (raw); } _if_result_749; })); } else { _if_result_748 = (EL_STR("{\"error\":\"User denied this tool call\"}")); } _if_result_748; });
|
||||
el_val_t legacy_messages = json_get_raw(pending_raw, EL_STR("messages_so_far"));
|
||||
el_val_t stored_variant = json_get(pending_raw, EL_STR("tools_variant"));
|
||||
el_val_t tools_json = ({ el_val_t _if_result_722 = 0; if (str_eq(stored_variant, EL_STR("web"))) { _if_result_722 = (agentic_tools_with_web()); } else { _if_result_722 = (({ el_val_t _if_result_723 = 0; if (str_eq(stored_variant, EL_STR("all"))) { _if_result_723 = (agentic_tools_all()); } else { _if_result_723 = (agentic_tools_literal()); } _if_result_723; })); } _if_result_722; });
|
||||
el_val_t tools_json = ({ el_val_t _if_result_750 = 0; if (str_eq(stored_variant, EL_STR("web"))) { _if_result_750 = (agentic_tools_with_web()); } else { _if_result_750 = (({ el_val_t _if_result_751 = 0; if (str_eq(stored_variant, EL_STR("all"))) { _if_result_751 = (agentic_tools_all()); } else { _if_result_751 = (agentic_tools_literal()); } _if_result_751; })); } _if_result_750; });
|
||||
el_val_t blob = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"model\":\""), json_safe(model)), EL_STR("\"")), EL_STR(",\"safe_sys\":\"")), json_safe(safe_sys)), EL_STR("\"")), EL_STR(",\"tools_json\":\"")), json_safe(tools_json)), EL_STR("\"")), EL_STR(",\"messages\":\"")), json_safe(legacy_messages)), EL_STR("\"")), EL_STR(",\"tools_log\":\"\"")), EL_STR(",\"tool_use_id\":\"")), json_safe(call_id)), EL_STR("\"}"));
|
||||
state_set(el_str_concat(EL_STR("mcp_bridge:"), session_id), blob);
|
||||
return agentic_resume(session_id, call_id, tool_result);
|
||||
@@ -30765,24 +30964,24 @@ el_val_t rate_limit_check(el_val_t ip, el_val_t path) {
|
||||
return EL_STR("");
|
||||
}
|
||||
el_val_t limit_str = state_get(EL_STR("soul_rate_limit"));
|
||||
el_val_t limit = ({ el_val_t _if_result_724 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_724 = (60); } else { _if_result_724 = (str_to_int(limit_str)); } _if_result_724; });
|
||||
el_val_t limit = ({ el_val_t _if_result_752 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_752 = (60); } else { _if_result_752 = (str_to_int(limit_str)); } _if_result_752; });
|
||||
el_val_t now = time_now();
|
||||
el_val_t window_key = el_str_concat(el_str_concat(EL_STR("rl:"), ip), EL_STR(":window"));
|
||||
el_val_t count_key = el_str_concat(el_str_concat(EL_STR("rl:"), ip), EL_STR(":count"));
|
||||
el_val_t win_str = state_get(window_key);
|
||||
el_val_t win_start = ({ el_val_t _if_result_725 = 0; if (str_eq(win_str, EL_STR(""))) { _if_result_725 = (now); } else { _if_result_725 = (str_to_int(win_str)); } _if_result_725; });
|
||||
el_val_t win_start = ({ el_val_t _if_result_753 = 0; if (str_eq(win_str, EL_STR(""))) { _if_result_753 = (now); } else { _if_result_753 = (str_to_int(win_str)); } _if_result_753; });
|
||||
el_val_t elapsed = (now - win_start);
|
||||
el_val_t in_window = (elapsed < 60);
|
||||
el_val_t prev_count_str = state_get(count_key);
|
||||
el_val_t prev_count = ({ el_val_t _if_result_726 = 0; if (str_eq(prev_count_str, EL_STR(""))) { _if_result_726 = (0); } else { _if_result_726 = (str_to_int(prev_count_str)); } _if_result_726; });
|
||||
el_val_t eff_count = ({ el_val_t _if_result_727 = 0; if (in_window) { _if_result_727 = (prev_count); } else { _if_result_727 = (0); } _if_result_727; });
|
||||
el_val_t eff_win = ({ el_val_t _if_result_728 = 0; if (in_window) { _if_result_728 = (win_start); } else { _if_result_728 = (now); } _if_result_728; });
|
||||
el_val_t prev_count = ({ el_val_t _if_result_754 = 0; if (str_eq(prev_count_str, EL_STR(""))) { _if_result_754 = (0); } else { _if_result_754 = (str_to_int(prev_count_str)); } _if_result_754; });
|
||||
el_val_t eff_count = ({ el_val_t _if_result_755 = 0; if (in_window) { _if_result_755 = (prev_count); } else { _if_result_755 = (0); } _if_result_755; });
|
||||
el_val_t eff_win = ({ el_val_t _if_result_756 = 0; if (in_window) { _if_result_756 = (win_start); } else { _if_result_756 = (now); } _if_result_756; });
|
||||
el_val_t new_count = (eff_count + 1);
|
||||
state_set(count_key, int_to_str(new_count));
|
||||
state_set(window_key, int_to_str(eff_win));
|
||||
if (new_count > limit) {
|
||||
el_val_t retry_after = (60 - (now - eff_win));
|
||||
el_val_t eff_retry = ({ el_val_t _if_result_729 = 0; if ((retry_after < 0)) { _if_result_729 = (0); } else { _if_result_729 = (retry_after); } _if_result_729; });
|
||||
el_val_t eff_retry = ({ el_val_t _if_result_757 = 0; if ((retry_after < 0)) { _if_result_757 = (0); } else { _if_result_757 = (retry_after); } _if_result_757; });
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"__status__\":429,\"error\":\"rate limit exceeded\",\"code\":\"rate_limited\",\"retry_after_secs\":"), int_to_str(eff_retry)), EL_STR("}"));
|
||||
}
|
||||
return EL_STR("");
|
||||
@@ -30811,18 +31010,18 @@ el_val_t err_405(el_val_t method, el_val_t path) {
|
||||
el_val_t route_health(void) {
|
||||
el_val_t cgi_id = state_get(EL_STR("soul_cgi_id"));
|
||||
el_val_t boot = state_get(EL_STR("soul_boot_count"));
|
||||
el_val_t boot_num = ({ el_val_t _if_result_730 = 0; if (str_eq(boot, EL_STR(""))) { _if_result_730 = (EL_STR("0")); } else { _if_result_730 = (boot); } _if_result_730; });
|
||||
el_val_t boot_num = ({ el_val_t _if_result_758 = 0; if (str_eq(boot, EL_STR(""))) { _if_result_758 = (EL_STR("0")); } else { _if_result_758 = (boot); } _if_result_758; });
|
||||
el_val_t node_ct = engram_node_count();
|
||||
el_val_t edge_ct = engram_edge_count();
|
||||
el_val_t pulse = state_get(EL_STR("soul.pulse"));
|
||||
el_val_t pulse_num = ({ el_val_t _if_result_731 = 0; if (str_eq(pulse, EL_STR(""))) { _if_result_731 = (EL_STR("0")); } else { _if_result_731 = (pulse); } _if_result_731; });
|
||||
el_val_t pulse_num = ({ el_val_t _if_result_759 = 0; if (str_eq(pulse, EL_STR(""))) { _if_result_759 = (EL_STR("0")); } else { _if_result_759 = (pulse); } _if_result_759; });
|
||||
el_val_t boot_ts_str = state_get(EL_STR("soul_boot_ts"));
|
||||
el_val_t uptime_secs = ({ el_val_t _if_result_732 = 0; if (str_eq(boot_ts_str, EL_STR(""))) { _if_result_732 = ((-1)); } else { _if_result_732 = ((time_now() - str_to_int(boot_ts_str))); } _if_result_732; });
|
||||
el_val_t uptime_secs = ({ el_val_t _if_result_760 = 0; if (str_eq(boot_ts_str, EL_STR(""))) { _if_result_760 = ((-1)); } else { _if_result_760 = ((time_now() - str_to_int(boot_ts_str))); } _if_result_760; });
|
||||
el_val_t model = state_get(EL_STR("soul_model"));
|
||||
el_val_t eff_model = ({ el_val_t _if_result_733 = 0; if (str_eq(model, EL_STR(""))) { _if_result_733 = (EL_STR("claude-sonnet-4-5")); } else { _if_result_733 = (model); } _if_result_733; });
|
||||
el_val_t eff_model = ({ el_val_t _if_result_761 = 0; if (str_eq(model, EL_STR(""))) { _if_result_761 = (EL_STR("claude-sonnet-4-5")); } else { _if_result_761 = (model); } _if_result_761; });
|
||||
el_val_t llm_probe = llm_call_system(eff_model, EL_STR("You are a health probe. Reply with the single word: ok"), EL_STR("ping"));
|
||||
el_val_t llm_ok = (((!str_eq(llm_probe, EL_STR("")) && !str_starts_with(llm_probe, EL_STR("{\"error\""))) && !str_starts_with(llm_probe, EL_STR("{\"type\":\"error\""))) && !str_contains(llm_probe, EL_STR("authentication_error")));
|
||||
el_val_t llm_status = ({ el_val_t _if_result_734 = 0; if (llm_ok) { _if_result_734 = (EL_STR("ok")); } else { _if_result_734 = (EL_STR("unreachable")); } _if_result_734; });
|
||||
el_val_t llm_status = ({ el_val_t _if_result_762 = 0; if (llm_ok) { _if_result_762 = (EL_STR("ok")); } else { _if_result_762 = (EL_STR("unreachable")); } _if_result_762; });
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"status\":\"alive\""), EL_STR(",\"cgi_id\":\"")), cgi_id), EL_STR("\"")), EL_STR(",\"boot\":")), boot_num), EL_STR(",\"uptime_secs\":")), int_to_str(uptime_secs)), EL_STR(",\"node_count\":")), int_to_str(node_ct)), EL_STR(",\"edge_count\":")), int_to_str(edge_ct)), EL_STR(",\"pulse\":")), pulse_num), EL_STR(",\"llm\":\"")), llm_status), EL_STR("\"")), EL_STR(",\"layers\":{\"l0\":\"core\",\"l1\":\"safety\",\"l2\":\"stewardship\",\"l3\":\"")), imprint_current()), EL_STR("\"}}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -30892,30 +31091,30 @@ el_val_t handle_dharma_recv(el_val_t body) {
|
||||
el_val_t from_id = json_get(body, EL_STR("from"));
|
||||
el_val_t event_type = json_get(content_raw, EL_STR("event_type"));
|
||||
el_val_t payload = json_get(content_raw, EL_STR("payload"));
|
||||
el_val_t eff_event = ({ el_val_t _if_result_735 = 0; if (str_eq(event_type, EL_STR(""))) { _if_result_735 = (EL_STR("chat")); } else { _if_result_735 = (event_type); } _if_result_735; });
|
||||
el_val_t eff_payload = ({ el_val_t _if_result_736 = 0; if (str_eq(payload, EL_STR(""))) { _if_result_736 = (content_raw); } else { _if_result_736 = (payload); } _if_result_736; });
|
||||
el_val_t eff_event = ({ el_val_t _if_result_763 = 0; if (str_eq(event_type, EL_STR(""))) { _if_result_763 = (EL_STR("chat")); } else { _if_result_763 = (event_type); } _if_result_763; });
|
||||
el_val_t eff_payload = ({ el_val_t _if_result_764 = 0; if (str_eq(payload, EL_STR(""))) { _if_result_764 = (content_raw); } else { _if_result_764 = (payload); } _if_result_764; });
|
||||
if (str_eq(eff_event, EL_STR("chat"))) {
|
||||
el_val_t msg = json_get(eff_payload, EL_STR("message"));
|
||||
el_val_t chat_body = ({ el_val_t _if_result_737 = 0; if (str_eq(msg, EL_STR(""))) { _if_result_737 = (el_str_concat(el_str_concat(EL_STR("{\"message\":\""), str_replace(str_replace(eff_payload, EL_STR("\\"), EL_STR("\\\\")), EL_STR("\""), EL_STR("\\\""))), EL_STR("\"}"))); } else { _if_result_737 = (eff_payload); } _if_result_737; });
|
||||
el_val_t chat_body = ({ el_val_t _if_result_765 = 0; if (str_eq(msg, EL_STR(""))) { _if_result_765 = (el_str_concat(el_str_concat(EL_STR("{\"message\":\""), str_replace(str_replace(eff_payload, EL_STR("\\"), EL_STR("\\\\")), EL_STR("\""), EL_STR("\\\""))), EL_STR("\"}"))); } else { _if_result_765 = (eff_payload); } _if_result_765; });
|
||||
el_val_t agentic_flag = json_get_bool(eff_payload, EL_STR("agentic"));
|
||||
el_val_t raw_msg = json_get(chat_body, EL_STR("message"));
|
||||
el_val_t req_mode = json_get(chat_body, EL_STR("mode"));
|
||||
el_val_t reply = ({ el_val_t _if_result_738 = 0; if (str_eq(req_mode, EL_STR("plan"))) { _if_result_738 = (handle_chat_plan(chat_body)); } else { _if_result_738 = (({ el_val_t _if_result_739 = 0; if (agentic_flag) { _if_result_739 = (handle_chat_agentic(chat_body)); } else { el_val_t screened_reply = layered_cycle(raw_msg, json_get(chat_body, EL_STR("session_id")), is_utility_request(chat_body, json_get(chat_body, EL_STR("session_id")))); _if_result_739 = (plain_chat_envelope(screened_reply, chat_default_model())); } _if_result_739; })); } _if_result_738; });
|
||||
el_val_t reply = ({ el_val_t _if_result_766 = 0; if (str_eq(req_mode, EL_STR("plan"))) { _if_result_766 = (handle_chat_plan(chat_body)); } else { _if_result_766 = (({ el_val_t _if_result_767 = 0; if (agentic_flag) { _if_result_767 = (handle_chat_agentic(chat_body)); } else { el_val_t screened_reply = layered_cycle(raw_msg, json_get(chat_body, EL_STR("session_id")), is_utility_request(chat_body, json_get(chat_body, EL_STR("session_id")))); _if_result_767 = (plain_chat_envelope(screened_reply, chat_default_model())); } _if_result_767; })); } _if_result_766; });
|
||||
auto_persist(chat_body, reply);
|
||||
return reply;
|
||||
}
|
||||
if (str_eq(eff_event, EL_STR("memory"))) {
|
||||
el_val_t query = json_get(eff_payload, EL_STR("query"));
|
||||
el_val_t limit_str = json_get(eff_payload, EL_STR("limit"));
|
||||
el_val_t limit = ({ el_val_t _if_result_740 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_740 = (20); } else { _if_result_740 = (str_to_int(limit_str)); } _if_result_740; });
|
||||
el_val_t q = ({ el_val_t _if_result_741 = 0; if (str_eq(query, EL_STR(""))) { _if_result_741 = (eff_payload); } else { _if_result_741 = (query); } _if_result_741; });
|
||||
el_val_t limit = ({ el_val_t _if_result_768 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_768 = (20); } else { _if_result_768 = (str_to_int(limit_str)); } _if_result_768; });
|
||||
el_val_t q = ({ el_val_t _if_result_769 = 0; if (str_eq(query, EL_STR(""))) { _if_result_769 = (eff_payload); } else { _if_result_769 = (query); } _if_result_769; });
|
||||
return engram_search_json(q, limit);
|
||||
}
|
||||
if (str_eq(eff_event, EL_STR("tool"))) {
|
||||
el_val_t path_field = json_get(eff_payload, EL_STR("path"));
|
||||
el_val_t method_field = json_get(eff_payload, EL_STR("method"));
|
||||
el_val_t tool_body = json_get(eff_payload, EL_STR("body"));
|
||||
el_val_t eff_method = ({ el_val_t _if_result_742 = 0; if (str_eq(method_field, EL_STR(""))) { _if_result_742 = (EL_STR("POST")); } else { _if_result_742 = (method_field); } _if_result_742; });
|
||||
el_val_t eff_method = ({ el_val_t _if_result_770 = 0; if (str_eq(method_field, EL_STR(""))) { _if_result_770 = (EL_STR("POST")); } else { _if_result_770 = (method_field); } _if_result_770; });
|
||||
return handle_tool(path_field, eff_method, tool_body);
|
||||
}
|
||||
if (str_eq(eff_event, EL_STR("see"))) {
|
||||
@@ -30950,7 +31149,7 @@ el_val_t connectd_get(el_val_t suffix) {
|
||||
}
|
||||
|
||||
el_val_t connectd_post(el_val_t suffix, el_val_t body) {
|
||||
el_val_t eff = ({ el_val_t _if_result_743 = 0; if (str_eq(body, EL_STR(""))) { _if_result_743 = (EL_STR("{}")); } else { _if_result_743 = (body); } _if_result_743; });
|
||||
el_val_t eff = ({ el_val_t _if_result_771 = 0; if (str_eq(body, EL_STR(""))) { _if_result_771 = (EL_STR("{}")); } else { _if_result_771 = (body); } _if_result_771; });
|
||||
el_val_t tmp = el_str_concat(el_str_concat(EL_STR("/tmp/neuron-connectors-req-"), int_to_str(time_now())), EL_STR(".json"));
|
||||
fs_write(tmp, eff);
|
||||
el_val_t out = exec_capture(el_str_concat(el_str_concat(el_str_concat(EL_STR("curl -s --max-time 20 -X POST http://127.0.0.1:7771"), suffix), EL_STR(" -H 'Content-Type: application/json' -d @")), tmp));
|
||||
@@ -31022,22 +31221,22 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
}
|
||||
if (str_eq(clean, EL_STR("/api/graph/edges"))) {
|
||||
el_val_t scratch_dir = env(EL_STR("TMPDIR"));
|
||||
el_val_t scratch_base = ({ el_val_t _if_result_744 = 0; if (str_eq(scratch_dir, EL_STR(""))) { _if_result_744 = (EL_STR("/tmp")); } else { _if_result_744 = (scratch_dir); } _if_result_744; });
|
||||
el_val_t scratch_base = ({ el_val_t _if_result_772 = 0; if (str_eq(scratch_dir, EL_STR(""))) { _if_result_772 = (EL_STR("/tmp")); } else { _if_result_772 = (scratch_dir); } _if_result_772; });
|
||||
el_val_t snap_path = el_str_concat(el_str_concat(el_str_concat(scratch_base, EL_STR("/soul-edges-export-")), state_get(EL_STR("soul_cgi_id"))), EL_STR(".json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
el_val_t edges_raw = json_get_raw(snap, EL_STR("edges"));
|
||||
return ({ el_val_t _if_result_745 = 0; if (str_eq(edges_raw, EL_STR(""))) { _if_result_745 = (EL_STR("[]")); } else { _if_result_745 = (edges_raw); } _if_result_745; });
|
||||
return ({ el_val_t _if_result_773 = 0; if (str_eq(edges_raw, EL_STR(""))) { _if_result_773 = (EL_STR("[]")); } else { _if_result_773 = (edges_raw); } _if_result_773; });
|
||||
}
|
||||
if (str_eq(clean, EL_STR("/api/chat"))) {
|
||||
el_val_t raw_msg = json_get(body, EL_STR("message"));
|
||||
el_val_t eff_msg = ({ el_val_t _if_result_746 = 0; if (str_eq(raw_msg, EL_STR(""))) { _if_result_746 = (body); } else { _if_result_746 = (raw_msg); } _if_result_746; });
|
||||
el_val_t eff_msg = ({ el_val_t _if_result_774 = 0; if (str_eq(raw_msg, EL_STR(""))) { _if_result_774 = (body); } else { _if_result_774 = (raw_msg); } _if_result_774; });
|
||||
if (str_eq(eff_msg, EL_STR(""))) {
|
||||
return EL_STR("{\"error\":\"message is required\",\"code\":\"missing_param\"}");
|
||||
}
|
||||
el_val_t agentic_flag = json_get_bool(body, EL_STR("agentic"));
|
||||
el_val_t req_mode = json_get(body, EL_STR("mode"));
|
||||
el_val_t reply = ({ el_val_t _if_result_747 = 0; if (str_eq(req_mode, EL_STR("plan"))) { _if_result_747 = (handle_chat_plan(body)); } else { _if_result_747 = (({ el_val_t _if_result_748 = 0; if (agentic_flag) { _if_result_748 = (handle_chat_agentic(body)); } else { el_val_t screened_reply = layered_cycle(eff_msg, json_get(body, EL_STR("session_id")), is_utility_request(body, json_get(body, EL_STR("session_id")))); _if_result_748 = (plain_chat_envelope(screened_reply, chat_default_model())); } _if_result_748; })); } _if_result_747; });
|
||||
el_val_t reply = ({ el_val_t _if_result_775 = 0; if (str_eq(req_mode, EL_STR("plan"))) { _if_result_775 = (handle_chat_plan(body)); } else { _if_result_775 = (({ el_val_t _if_result_776 = 0; if (agentic_flag) { _if_result_776 = (handle_chat_agentic(body)); } else { el_val_t screened_reply = layered_cycle(eff_msg, json_get(body, EL_STR("session_id")), is_utility_request(body, json_get(body, EL_STR("session_id")))); _if_result_776 = (plain_chat_envelope(screened_reply, chat_default_model())); } _if_result_776; })); } _if_result_775; });
|
||||
auto_persist(body, reply);
|
||||
return reply;
|
||||
}
|
||||
@@ -31104,6 +31303,9 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_starts_with(clean, EL_STR("/api/neuron/graph"))) {
|
||||
return handle_api_inspect_graph(method, path, body);
|
||||
}
|
||||
if (str_starts_with(clean, EL_STR("/api/neuron/audit/structural"))) {
|
||||
return handle_api_structural_audit(method, path, body);
|
||||
}
|
||||
if (str_starts_with(clean, EL_STR("/api/neuron/list/"))) {
|
||||
el_val_t node_type = str_slice(clean, 17, str_len(clean));
|
||||
return handle_api_list_typed(node_type, path, body);
|
||||
@@ -31118,7 +31320,7 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t rp_id = str_slice(clean, 18, str_len(clean));
|
||||
if (!str_eq(rp_id, EL_STR(""))) {
|
||||
el_val_t rp_raw = state_get(el_str_concat(EL_STR("run_progress_"), rp_id));
|
||||
el_val_t rp_arr = ({ el_val_t _if_result_749 = 0; if (str_eq(rp_raw, EL_STR(""))) { _if_result_749 = (EL_STR("[]")); } else { _if_result_749 = (el_str_concat(el_str_concat(EL_STR("["), rp_raw), EL_STR("]"))); } _if_result_749; });
|
||||
el_val_t rp_arr = ({ el_val_t _if_result_777 = 0; if (str_eq(rp_raw, EL_STR(""))) { _if_result_777 = (EL_STR("[]")); } else { _if_result_777 = (el_str_concat(el_str_concat(EL_STR("["), rp_raw), EL_STR("]"))); } _if_result_777; });
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"progress\":"), rp_arr), EL_STR("}"));
|
||||
}
|
||||
}
|
||||
@@ -31128,7 +31330,7 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_starts_with(clean, EL_STR("/api/sessions/"))) {
|
||||
el_val_t gs_after = str_slice(clean, 14, str_len(clean));
|
||||
el_val_t gs_slash = str_index_of(gs_after, EL_STR("/"));
|
||||
el_val_t gs_id = ({ el_val_t _if_result_750 = 0; if ((gs_slash < 0)) { _if_result_750 = (gs_after); } else { _if_result_750 = (str_slice(gs_after, 0, gs_slash)); } _if_result_750; });
|
||||
el_val_t gs_id = ({ el_val_t _if_result_778 = 0; if ((gs_slash < 0)) { _if_result_778 = (gs_after); } else { _if_result_778 = (str_slice(gs_after, 0, gs_slash)); } _if_result_778; });
|
||||
if (!str_eq(gs_id, EL_STR(""))) {
|
||||
return session_get(gs_id);
|
||||
}
|
||||
@@ -31142,14 +31344,14 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_starts_with(clean, EL_STR("/api/sessions/")) && str_ends_with(clean, EL_STR("/tool_result"))) {
|
||||
el_val_t after = str_slice(clean, 14, str_len(clean));
|
||||
el_val_t slash = str_index_of(after, EL_STR("/"));
|
||||
el_val_t session_id = ({ el_val_t _if_result_751 = 0; if ((slash < 0)) { _if_result_751 = (after); } else { _if_result_751 = (str_slice(after, 0, slash)); } _if_result_751; });
|
||||
el_val_t session_id = ({ el_val_t _if_result_779 = 0; if ((slash < 0)) { _if_result_779 = (after); } else { _if_result_779 = (str_slice(after, 0, slash)); } _if_result_779; });
|
||||
return handle_tool_result(session_id, body);
|
||||
}
|
||||
if (str_starts_with(clean, EL_STR("/api/sessions/"))) {
|
||||
el_val_t sess_after = str_slice(clean, 14, str_len(clean));
|
||||
el_val_t sess_slash = str_index_of(sess_after, EL_STR("/"));
|
||||
el_val_t sess_id = ({ el_val_t _if_result_752 = 0; if ((sess_slash < 0)) { _if_result_752 = (sess_after); } else { _if_result_752 = (str_slice(sess_after, 0, sess_slash)); } _if_result_752; });
|
||||
el_val_t sess_sub = ({ el_val_t _if_result_753 = 0; if ((sess_slash < 0)) { _if_result_753 = (EL_STR("")); } else { _if_result_753 = (str_slice(sess_after, (sess_slash + 1), str_len(sess_after))); } _if_result_753; });
|
||||
el_val_t sess_id = ({ el_val_t _if_result_780 = 0; if ((sess_slash < 0)) { _if_result_780 = (sess_after); } else { _if_result_780 = (str_slice(sess_after, 0, sess_slash)); } _if_result_780; });
|
||||
el_val_t sess_sub = ({ el_val_t _if_result_781 = 0; if ((sess_slash < 0)) { _if_result_781 = (EL_STR("")); } else { _if_result_781 = (str_slice(sess_after, (sess_slash + 1), str_len(sess_after))); } _if_result_781; });
|
||||
if (!str_eq(sess_id, EL_STR("")) && str_eq(sess_sub, EL_STR("approve"))) {
|
||||
return handle_session_approve(sess_id, body);
|
||||
}
|
||||
@@ -31173,7 +31375,7 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
}
|
||||
el_val_t agentic_flag = json_get_bool(body, EL_STR("agentic"));
|
||||
el_val_t req_mode = json_get(body, EL_STR("mode"));
|
||||
el_val_t reply = ({ el_val_t _if_result_754 = 0; if (str_eq(req_mode, EL_STR("plan"))) { _if_result_754 = (handle_chat_plan(body)); } else { _if_result_754 = (({ el_val_t _if_result_755 = 0; if (agentic_flag) { _if_result_755 = (handle_chat_agentic(body)); } else { el_val_t screened_reply = layered_cycle(raw_msg, json_get(body, EL_STR("session_id")), is_utility_request(body, json_get(body, EL_STR("session_id")))); _if_result_755 = (plain_chat_envelope(screened_reply, chat_default_model())); } _if_result_755; })); } _if_result_754; });
|
||||
el_val_t reply = ({ el_val_t _if_result_782 = 0; if (str_eq(req_mode, EL_STR("plan"))) { _if_result_782 = (handle_chat_plan(body)); } else { _if_result_782 = (({ el_val_t _if_result_783 = 0; if (agentic_flag) { _if_result_783 = (handle_chat_agentic(body)); } else { el_val_t screened_reply = layered_cycle(raw_msg, json_get(body, EL_STR("session_id")), is_utility_request(body, json_get(body, EL_STR("session_id")))); _if_result_783 = (plain_chat_envelope(screened_reply, chat_default_model())); } _if_result_783; })); } _if_result_782; });
|
||||
auto_persist(body, reply);
|
||||
return reply;
|
||||
}
|
||||
@@ -31252,6 +31454,9 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(clean, EL_STR("/api/neuron/graph/link"))) {
|
||||
return handle_api_link_entities(body);
|
||||
}
|
||||
if (str_eq(clean, EL_STR("/api/neuron/audit/structural"))) {
|
||||
return handle_api_structural_audit(method, path, body);
|
||||
}
|
||||
if (str_eq(clean, EL_STR("/api/neuron/memory"))) {
|
||||
return handle_api_remember(body);
|
||||
}
|
||||
@@ -31297,7 +31502,7 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_starts_with(clean, EL_STR("/api/sessions/"))) {
|
||||
el_val_t del_after = str_slice(clean, 14, str_len(clean));
|
||||
el_val_t del_slash = str_index_of(del_after, EL_STR("/"));
|
||||
el_val_t del_id = ({ el_val_t _if_result_756 = 0; if ((del_slash < 0)) { _if_result_756 = (del_after); } else { _if_result_756 = (str_slice(del_after, 0, del_slash)); } _if_result_756; });
|
||||
el_val_t del_id = ({ el_val_t _if_result_784 = 0; if ((del_slash < 0)) { _if_result_784 = (del_after); } else { _if_result_784 = (str_slice(del_after, 0, del_slash)); } _if_result_784; });
|
||||
if (!str_eq(del_id, EL_STR(""))) {
|
||||
return session_delete(del_id);
|
||||
}
|
||||
@@ -31308,7 +31513,7 @@ el_val_t route_dispatch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_starts_with(clean, EL_STR("/api/sessions/"))) {
|
||||
el_val_t patch_after = str_slice(clean, 14, str_len(clean));
|
||||
el_val_t patch_slash = str_index_of(patch_after, EL_STR("/"));
|
||||
el_val_t patch_id = ({ el_val_t _if_result_757 = 0; if ((patch_slash < 0)) { _if_result_757 = (patch_after); } else { _if_result_757 = (str_slice(patch_after, 0, patch_slash)); } _if_result_757; });
|
||||
el_val_t patch_id = ({ el_val_t _if_result_785 = 0; if ((patch_slash < 0)) { _if_result_785 = (patch_after); } else { _if_result_785 = (str_slice(patch_after, 0, patch_slash)); } _if_result_785; });
|
||||
if (!str_eq(patch_id, EL_STR(""))) {
|
||||
return session_update_patch(patch_id, body);
|
||||
}
|
||||
@@ -31434,8 +31639,8 @@ el_val_t aff_try_slot(el_val_t slot_json, el_val_t aff_7d_ts, el_val_t acc_key)
|
||||
}
|
||||
}
|
||||
el_val_t bn_ts_raw = state_get(EL_STR("_ats_ts_raw"));
|
||||
el_val_t bn_ts = ({ el_val_t _if_result_758 = 0; if (str_eq(bn_ts_raw, EL_STR(""))) { _if_result_758 = (0); } else { _if_result_758 = (str_to_int(bn_ts_raw)); } _if_result_758; });
|
||||
el_val_t snip = ({ el_val_t _if_result_759 = 0; if ((str_len(bn_c) > 200)) { _if_result_759 = (str_slice(bn_c, 0, 200)); } else { _if_result_759 = (bn_c); } _if_result_759; });
|
||||
el_val_t bn_ts = ({ el_val_t _if_result_786 = 0; if (str_eq(bn_ts_raw, EL_STR(""))) { _if_result_786 = (0); } else { _if_result_786 = (str_to_int(bn_ts_raw)); } _if_result_786; });
|
||||
el_val_t snip = ({ el_val_t _if_result_787 = 0; if ((str_len(bn_c) > 200)) { _if_result_787 = (str_slice(bn_c, 0, 200)); } else { _if_result_787 = (bn_c); } _if_result_787; });
|
||||
if ((bn_ts >= aff_7d_ts) && !str_eq(snip, EL_STR(""))) {
|
||||
el_val_t cur_acc = state_get(acc_key);
|
||||
if (str_eq(cur_acc, EL_STR(""))) {
|
||||
@@ -31456,21 +31661,21 @@ el_val_t load_identity_context(void) {
|
||||
el_val_t intel_ok = (!str_eq(node_intel, EL_STR("")) && !str_eq(node_intel, EL_STR("null")));
|
||||
el_val_t values_ok = (!str_eq(node_values, EL_STR("")) && !str_eq(node_values, EL_STR("null")));
|
||||
el_val_t mem_ok = (!str_eq(node_mem_phil, EL_STR("")) && !str_eq(node_mem_phil, EL_STR("null")));
|
||||
el_val_t intel_content = ({ el_val_t _if_result_760 = 0; if (intel_ok) { _if_result_760 = (json_get(node_intel, EL_STR("content"))); } else { _if_result_760 = (EL_STR("")); } _if_result_760; });
|
||||
el_val_t values_content = ({ el_val_t _if_result_761 = 0; if (values_ok) { _if_result_761 = (json_get(node_values, EL_STR("content"))); } else { _if_result_761 = (EL_STR("")); } _if_result_761; });
|
||||
el_val_t mem_content = ({ el_val_t _if_result_762 = 0; if (mem_ok) { _if_result_762 = (json_get(node_mem_phil, EL_STR("content"))); } else { _if_result_762 = (EL_STR("")); } _if_result_762; });
|
||||
el_val_t intel_short = ({ el_val_t _if_result_763 = 0; if ((str_len(intel_content) > 2000)) { _if_result_763 = (str_slice(intel_content, 0, 2000)); } else { _if_result_763 = (intel_content); } _if_result_763; });
|
||||
el_val_t values_short = ({ el_val_t _if_result_764 = 0; if ((str_len(values_content) > 2000)) { _if_result_764 = (str_slice(values_content, 0, 2000)); } else { _if_result_764 = (values_content); } _if_result_764; });
|
||||
el_val_t mem_short = ({ el_val_t _if_result_765 = 0; if ((str_len(mem_content) > 2000)) { _if_result_765 = (str_slice(mem_content, 0, 2000)); } else { _if_result_765 = (mem_content); } _if_result_765; });
|
||||
el_val_t intel_content = ({ el_val_t _if_result_788 = 0; if (intel_ok) { _if_result_788 = (json_get(node_intel, EL_STR("content"))); } else { _if_result_788 = (EL_STR("")); } _if_result_788; });
|
||||
el_val_t values_content = ({ el_val_t _if_result_789 = 0; if (values_ok) { _if_result_789 = (json_get(node_values, EL_STR("content"))); } else { _if_result_789 = (EL_STR("")); } _if_result_789; });
|
||||
el_val_t mem_content = ({ el_val_t _if_result_790 = 0; if (mem_ok) { _if_result_790 = (json_get(node_mem_phil, EL_STR("content"))); } else { _if_result_790 = (EL_STR("")); } _if_result_790; });
|
||||
el_val_t intel_short = ({ el_val_t _if_result_791 = 0; if ((str_len(intel_content) > 2000)) { _if_result_791 = (str_slice(intel_content, 0, 2000)); } else { _if_result_791 = (intel_content); } _if_result_791; });
|
||||
el_val_t values_short = ({ el_val_t _if_result_792 = 0; if ((str_len(values_content) > 2000)) { _if_result_792 = (str_slice(values_content, 0, 2000)); } else { _if_result_792 = (values_content); } _if_result_792; });
|
||||
el_val_t mem_short = ({ el_val_t _if_result_793 = 0; if ((str_len(mem_content) > 2000)) { _if_result_793 = (str_slice(mem_content, 0, 2000)); } else { _if_result_793 = (mem_content); } _if_result_793; });
|
||||
el_val_t parts_count = 0;
|
||||
parts_count = ({ el_val_t _if_result_766 = 0; if (intel_ok) { _if_result_766 = ((parts_count + 1)); } else { _if_result_766 = (parts_count); } _if_result_766; });
|
||||
parts_count = ({ el_val_t _if_result_767 = 0; if (values_ok) { _if_result_767 = ((parts_count + 1)); } else { _if_result_767 = (parts_count); } _if_result_767; });
|
||||
parts_count = ({ el_val_t _if_result_768 = 0; if (mem_ok) { _if_result_768 = ((parts_count + 1)); } else { _if_result_768 = (parts_count); } _if_result_768; });
|
||||
parts_count = ({ el_val_t _if_result_794 = 0; if (intel_ok) { _if_result_794 = ((parts_count + 1)); } else { _if_result_794 = (parts_count); } _if_result_794; });
|
||||
parts_count = ({ el_val_t _if_result_795 = 0; if (values_ok) { _if_result_795 = ((parts_count + 1)); } else { _if_result_795 = (parts_count); } _if_result_795; });
|
||||
parts_count = ({ el_val_t _if_result_796 = 0; if (mem_ok) { _if_result_796 = ((parts_count + 1)); } else { _if_result_796 = (parts_count); } _if_result_796; });
|
||||
if (parts_count > 0) {
|
||||
el_val_t ctx = EL_STR("");
|
||||
ctx = ({ el_val_t _if_result_769 = 0; if (intel_ok) { _if_result_769 = (el_str_concat(el_str_concat(el_str_concat(ctx, EL_STR("[INTELLECTUAL-DNA]\n")), intel_short), EL_STR("\n\n"))); } else { _if_result_769 = (ctx); } _if_result_769; });
|
||||
ctx = ({ el_val_t _if_result_770 = 0; if (values_ok) { _if_result_770 = (el_str_concat(el_str_concat(el_str_concat(ctx, EL_STR("[VALUES]\n")), values_short), EL_STR("\n\n"))); } else { _if_result_770 = (ctx); } _if_result_770; });
|
||||
ctx = ({ el_val_t _if_result_771 = 0; if (mem_ok) { _if_result_771 = (el_str_concat(el_str_concat(ctx, EL_STR("[MEMORY-PHILOSOPHY]\n")), mem_short)); } else { _if_result_771 = (ctx); } _if_result_771; });
|
||||
ctx = ({ el_val_t _if_result_797 = 0; if (intel_ok) { _if_result_797 = (el_str_concat(el_str_concat(el_str_concat(ctx, EL_STR("[INTELLECTUAL-DNA]\n")), intel_short), EL_STR("\n\n"))); } else { _if_result_797 = (ctx); } _if_result_797; });
|
||||
ctx = ({ el_val_t _if_result_798 = 0; if (values_ok) { _if_result_798 = (el_str_concat(el_str_concat(el_str_concat(ctx, EL_STR("[VALUES]\n")), values_short), EL_STR("\n\n"))); } else { _if_result_798 = (ctx); } _if_result_798; });
|
||||
ctx = ({ el_val_t _if_result_799 = 0; if (mem_ok) { _if_result_799 = (el_str_concat(el_str_concat(ctx, EL_STR("[MEMORY-PHILOSOPHY]\n")), mem_short)); } else { _if_result_799 = (ctx); } _if_result_799; });
|
||||
state_set(EL_STR("soul_identity_context"), ctx);
|
||||
println(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("[soul] identity context loaded ("), int_to_str(str_len(ctx))), EL_STR(" chars, ")), int_to_str(parts_count)), EL_STR(" nodes)")));
|
||||
}
|
||||
@@ -31493,10 +31698,10 @@ el_val_t load_identity_context(void) {
|
||||
el_val_t bell_raw = engram_search_json(EL_STR("bell:soft bell:hard BellEvent affective"), 3);
|
||||
el_val_t bell_aff_ok = (!str_eq(bell_raw, EL_STR("")) && !str_eq(bell_raw, EL_STR("[]")));
|
||||
el_val_t aff_ctx = EL_STR("");
|
||||
aff_ctx = ({ el_val_t _if_result_772 = 0; if (bell_aff_ok) { (void)(state_set(EL_STR("_bell_acc"), EL_STR(""))); (void)(aff_try_slot(json_array_get(bell_raw, 0), aff_7d, EL_STR("_bell_acc"))); (void)(aff_try_slot(json_array_get(bell_raw, 1), aff_7d, EL_STR("_bell_acc"))); (void)(aff_try_slot(json_array_get(bell_raw, 2), aff_7d, EL_STR("_bell_acc"))); _if_result_772 = (state_get(EL_STR("_bell_acc"))); } else { _if_result_772 = (EL_STR("")); } _if_result_772; });
|
||||
aff_ctx = ({ el_val_t _if_result_800 = 0; if (bell_aff_ok) { (void)(state_set(EL_STR("_bell_acc"), EL_STR(""))); (void)(aff_try_slot(json_array_get(bell_raw, 0), aff_7d, EL_STR("_bell_acc"))); (void)(aff_try_slot(json_array_get(bell_raw, 1), aff_7d, EL_STR("_bell_acc"))); (void)(aff_try_slot(json_array_get(bell_raw, 2), aff_7d, EL_STR("_bell_acc"))); _if_result_800 = (state_get(EL_STR("_bell_acc"))); } else { _if_result_800 = (EL_STR("")); } _if_result_800; });
|
||||
el_val_t pos_raw = engram_search_json(EL_STR("PositiveEvent joy:high joy:low affective"), 3);
|
||||
el_val_t pos_aff_ok = (!str_eq(pos_raw, EL_STR("")) && !str_eq(pos_raw, EL_STR("[]")));
|
||||
aff_ctx = ({ el_val_t _if_result_773 = 0; if (pos_aff_ok) { (void)(state_set(EL_STR("_pos_acc"), aff_ctx)); (void)(aff_try_slot(json_array_get(pos_raw, 0), aff_7d, EL_STR("_pos_acc"))); (void)(aff_try_slot(json_array_get(pos_raw, 1), aff_7d, EL_STR("_pos_acc"))); (void)(aff_try_slot(json_array_get(pos_raw, 2), aff_7d, EL_STR("_pos_acc"))); _if_result_773 = (state_get(EL_STR("_pos_acc"))); } else { _if_result_773 = (aff_ctx); } _if_result_773; });
|
||||
aff_ctx = ({ el_val_t _if_result_801 = 0; if (pos_aff_ok) { (void)(state_set(EL_STR("_pos_acc"), aff_ctx)); (void)(aff_try_slot(json_array_get(pos_raw, 0), aff_7d, EL_STR("_pos_acc"))); (void)(aff_try_slot(json_array_get(pos_raw, 1), aff_7d, EL_STR("_pos_acc"))); (void)(aff_try_slot(json_array_get(pos_raw, 2), aff_7d, EL_STR("_pos_acc"))); _if_result_801 = (state_get(EL_STR("_pos_acc"))); } else { _if_result_801 = (aff_ctx); } _if_result_801; });
|
||||
if (!str_eq(aff_ctx, EL_STR(""))) {
|
||||
state_set(EL_STR("soul_affective_context"), aff_ctx);
|
||||
println(el_str_concat(el_str_concat(EL_STR("[soul] affective context loaded ("), int_to_str(str_len(aff_ctx))), EL_STR(" chars)")));
|
||||
@@ -31542,19 +31747,19 @@ el_val_t seed_persona_from_env(void) {
|
||||
|
||||
el_val_t emit_session_start_event(void) {
|
||||
el_val_t boot = state_get(EL_STR("soul_boot_count"));
|
||||
el_val_t boot_num = ({ el_val_t _if_result_774 = 0; if (str_eq(boot, EL_STR(""))) { _if_result_774 = (EL_STR("0")); } else { _if_result_774 = (boot); } _if_result_774; });
|
||||
el_val_t boot_num = ({ el_val_t _if_result_802 = 0; if (str_eq(boot, EL_STR(""))) { _if_result_802 = (EL_STR("0")); } else { _if_result_802 = (boot); } _if_result_802; });
|
||||
el_val_t node_ct = engram_node_count();
|
||||
el_val_t edge_ct = engram_edge_count();
|
||||
el_val_t id_ctx = state_get(EL_STR("soul_identity_context"));
|
||||
el_val_t has_identity = ({ el_val_t _if_result_775 = 0; if (str_eq(id_ctx, EL_STR(""))) { _if_result_775 = (EL_STR("false")); } else { _if_result_775 = (EL_STR("true")); } _if_result_775; });
|
||||
el_val_t has_identity = ({ el_val_t _if_result_803 = 0; if (str_eq(id_ctx, EL_STR(""))) { _if_result_803 = (EL_STR("false")); } else { _if_result_803 = (EL_STR("true")); } _if_result_803; });
|
||||
el_val_t cgi_from_state = state_get(EL_STR("soul_cgi_id"));
|
||||
el_val_t cgi_from_env = env(EL_STR("SOUL_CGI_ID"));
|
||||
el_val_t eff_cgi = ({ el_val_t _if_result_776 = 0; if (!str_eq(cgi_from_state, EL_STR(""))) { _if_result_776 = (cgi_from_state); } else { _if_result_776 = (({ el_val_t _if_result_777 = 0; if (!str_eq(cgi_from_env, EL_STR(""))) { _if_result_777 = (cgi_from_env); } else { _if_result_777 = (EL_STR("ntn-genesis")); } _if_result_777; })); } _if_result_776; });
|
||||
el_val_t eff_cgi = ({ el_val_t _if_result_804 = 0; if (!str_eq(cgi_from_state, EL_STR(""))) { _if_result_804 = (cgi_from_state); } else { _if_result_804 = (({ el_val_t _if_result_805 = 0; if (!str_eq(cgi_from_env, EL_STR(""))) { _if_result_805 = (cgi_from_env); } else { _if_result_805 = (EL_STR("ntn-genesis")); } _if_result_805; })); } _if_result_804; });
|
||||
el_val_t ts = time_now();
|
||||
el_val_t prev_sum_node = engram_get_node_by_label(EL_STR("session:summary"));
|
||||
el_val_t prev_sum_ok = (!str_eq(prev_sum_node, EL_STR("")) && !str_eq(prev_sum_node, EL_STR("null")));
|
||||
el_val_t prev_sum_content = ({ el_val_t _if_result_778 = 0; if (prev_sum_ok) { _if_result_778 = (json_get(prev_sum_node, EL_STR("content"))); } else { el_val_t sum_search = engram_search_json(EL_STR("SessionSummary session:summary previous-session"), 2); el_val_t sum_srch_ok = (!str_eq(sum_search, EL_STR("")) && !str_eq(sum_search, EL_STR("[]"))); _if_result_778 = (({ el_val_t _if_result_779 = 0; if (sum_srch_ok) { el_val_t sn = json_array_get(sum_search, 0); el_val_t stype = json_get(sn, EL_STR("node_type")); el_val_t scontent = json_get(sn, EL_STR("content")); _if_result_779 = (({ el_val_t _if_result_780 = 0; if ((str_eq(stype, EL_STR("SessionSummary")) && !str_eq(scontent, EL_STR("")))) { _if_result_780 = (scontent); } else { _if_result_780 = (EL_STR("")); } _if_result_780; })); } else { _if_result_779 = (EL_STR("")); } _if_result_779; })); } _if_result_778; });
|
||||
el_val_t has_prev_sum = ({ el_val_t _if_result_781 = 0; if (str_eq(prev_sum_content, EL_STR(""))) { _if_result_781 = (EL_STR("false")); } else { _if_result_781 = (EL_STR("true")); } _if_result_781; });
|
||||
el_val_t prev_sum_content = ({ el_val_t _if_result_806 = 0; if (prev_sum_ok) { _if_result_806 = (json_get(prev_sum_node, EL_STR("content"))); } else { el_val_t sum_search = engram_search_json(EL_STR("SessionSummary session:summary previous-session"), 2); el_val_t sum_srch_ok = (!str_eq(sum_search, EL_STR("")) && !str_eq(sum_search, EL_STR("[]"))); _if_result_806 = (({ el_val_t _if_result_807 = 0; if (sum_srch_ok) { el_val_t sn = json_array_get(sum_search, 0); el_val_t stype = json_get(sn, EL_STR("node_type")); el_val_t scontent = json_get(sn, EL_STR("content")); _if_result_807 = (({ el_val_t _if_result_808 = 0; if ((str_eq(stype, EL_STR("SessionSummary")) && !str_eq(scontent, EL_STR("")))) { _if_result_808 = (scontent); } else { _if_result_808 = (EL_STR("")); } _if_result_808; })); } else { _if_result_807 = (EL_STR("")); } _if_result_807; })); } _if_result_806; });
|
||||
el_val_t has_prev_sum = ({ el_val_t _if_result_809 = 0; if (str_eq(prev_sum_content, EL_STR(""))) { _if_result_809 = (EL_STR("false")); } else { _if_result_809 = (EL_STR("true")); } _if_result_809; });
|
||||
if (!str_eq(prev_sum_content, EL_STR(""))) {
|
||||
state_set(EL_STR("soul_prev_session_summary"), prev_sum_content);
|
||||
println(el_str_concat(el_str_concat(EL_STR("[soul] previous session summary loaded ("), int_to_str(str_len(prev_sum_content))), EL_STR(" chars)")));
|
||||
@@ -31601,23 +31806,23 @@ el_val_t layered_cycle(el_val_t raw_input, el_val_t session_id, el_val_t utility
|
||||
el_val_t continuity = steward_session_check(screened, session_id);
|
||||
el_val_t cont_status = json_get(continuity, EL_STR("status"));
|
||||
el_val_t cont_action = json_get(continuity, EL_STR("action"));
|
||||
el_val_t cont_key = ({ el_val_t _if_result_782 = 0; if (str_eq(session_id, EL_STR(""))) { _if_result_782 = (EL_STR("session_continuity")); } else { _if_result_782 = (el_str_concat(EL_STR("session_continuity:"), session_id)); } _if_result_782; });
|
||||
el_val_t cont_key = ({ el_val_t _if_result_810 = 0; if (str_eq(session_id, EL_STR(""))) { _if_result_810 = (EL_STR("session_continuity")); } else { _if_result_810 = (el_str_concat(EL_STR("session_continuity:"), session_id)); } _if_result_810; });
|
||||
state_set(cont_key, cont_status);
|
||||
el_val_t guided = ({ el_val_t _if_result_783 = 0; if (str_eq(cont_action, EL_STR("identity_check"))) { _if_result_783 = (el_str_concat(screened, EL_STR(" [steward:identity_check]"))); } else { _if_result_783 = (({ el_val_t _if_result_784 = 0; if (str_eq(cont_action, EL_STR("soft_check"))) { _if_result_784 = (el_str_concat(screened, EL_STR(" [steward:continuity_concern]"))); } else { _if_result_784 = (screened); } _if_result_784; })); } _if_result_783; });
|
||||
el_val_t guided = ({ el_val_t _if_result_811 = 0; if (str_eq(cont_action, EL_STR("identity_check"))) { _if_result_811 = (el_str_concat(screened, EL_STR(" [steward:identity_check]"))); } else { _if_result_811 = (({ el_val_t _if_result_812 = 0; if (str_eq(cont_action, EL_STR("soft_check"))) { _if_result_812 = (el_str_concat(screened, EL_STR(" [steward:continuity_concern]"))); } else { _if_result_812 = (screened); } _if_result_812; })); } _if_result_811; });
|
||||
el_val_t imprint_id = imprint_current();
|
||||
el_val_t steward_result = steward_align(guided, imprint_id);
|
||||
el_val_t steward_action = json_get(steward_result, EL_STR("action"));
|
||||
el_val_t aligned = ({ el_val_t _if_result_785 = 0; if (str_eq(steward_action, EL_STR("pass"))) { _if_result_785 = (json_get(steward_result, EL_STR("content"))); } else { _if_result_785 = (json_get(steward_result, EL_STR("redirect_to"))); } _if_result_785; });
|
||||
el_val_t aligned = ({ el_val_t _if_result_813 = 0; if (str_eq(steward_action, EL_STR("pass"))) { _if_result_813 = (json_get(steward_result, EL_STR("content"))); } else { _if_result_813 = (json_get(steward_result, EL_STR("redirect_to"))); } _if_result_813; });
|
||||
el_val_t lc_aff_cutoff = (time_now() - 259200);
|
||||
el_val_t lc_bell_nodes = engram_search_json(EL_STR("bell:soft bell:hard BellEvent affective"), 2);
|
||||
el_val_t lc_has_bell = (!str_eq(lc_bell_nodes, EL_STR("")) && !str_eq(lc_bell_nodes, EL_STR("[]")));
|
||||
el_val_t lc_bell_note = ({ el_val_t _if_result_786 = 0; if (lc_has_bell) { el_val_t lb0 = json_array_get(lc_bell_nodes, 0); el_val_t lb_ts = affective_node_ts(lb0); _if_result_786 = (({ el_val_t _if_result_787 = 0; if ((lb_ts > lc_aff_cutoff)) { _if_result_787 = (EL_STR("[AFFECTIVE NOTE: User was in distress in a recent session.]")); } else { _if_result_787 = (EL_STR("")); } _if_result_787; })); } else { _if_result_786 = (EL_STR("")); } _if_result_786; });
|
||||
el_val_t lc_bell_note = ({ el_val_t _if_result_814 = 0; if (lc_has_bell) { el_val_t lb0 = json_array_get(lc_bell_nodes, 0); el_val_t lb_ts = affective_node_ts(lb0); _if_result_814 = (({ el_val_t _if_result_815 = 0; if ((lb_ts > lc_aff_cutoff)) { _if_result_815 = (EL_STR("[AFFECTIVE NOTE: User was in distress in a recent session.]")); } else { _if_result_815 = (EL_STR("")); } _if_result_815; })); } else { _if_result_814 = (EL_STR("")); } _if_result_814; });
|
||||
el_val_t lc_pos_nodes = engram_search_json(EL_STR("PositiveEvent joy:high joy:low affective"), 2);
|
||||
el_val_t lc_has_pos = (!str_eq(lc_pos_nodes, EL_STR("")) && !str_eq(lc_pos_nodes, EL_STR("[]")));
|
||||
el_val_t lc_pos_note = ({ el_val_t _if_result_788 = 0; if ((lc_has_pos && str_eq(lc_bell_note, EL_STR("")))) { el_val_t lp0 = json_array_get(lc_pos_nodes, 0); el_val_t lp_ts = affective_node_ts(lp0); _if_result_788 = (({ el_val_t _if_result_789 = 0; if ((lp_ts > lc_aff_cutoff)) { _if_result_789 = (EL_STR("[AFFECTIVE NOTE: User shared positive news in a recent session.]")); } else { _if_result_789 = (EL_STR("")); } _if_result_789; })); } else { _if_result_788 = (EL_STR("")); } _if_result_788; });
|
||||
el_val_t lc_affective_note = ({ el_val_t _if_result_790 = 0; if (!str_eq(lc_bell_note, EL_STR(""))) { _if_result_790 = (lc_bell_note); } else { _if_result_790 = (lc_pos_note); } _if_result_790; });
|
||||
el_val_t lc_pos_note = ({ el_val_t _if_result_816 = 0; if ((lc_has_pos && str_eq(lc_bell_note, EL_STR("")))) { el_val_t lp0 = json_array_get(lc_pos_nodes, 0); el_val_t lp_ts = affective_node_ts(lp0); _if_result_816 = (({ el_val_t _if_result_817 = 0; if ((lp_ts > lc_aff_cutoff)) { _if_result_817 = (EL_STR("[AFFECTIVE NOTE: User shared positive news in a recent session.]")); } else { _if_result_817 = (EL_STR("")); } _if_result_817; })); } else { _if_result_816 = (EL_STR("")); } _if_result_816; });
|
||||
el_val_t lc_affective_note = ({ el_val_t _if_result_818 = 0; if (!str_eq(lc_bell_note, EL_STR(""))) { _if_result_818 = (lc_bell_note); } else { _if_result_818 = (lc_pos_note); } _if_result_818; });
|
||||
el_val_t augmented_addendum = safety_augment_system(EL_STR(""), raw_input);
|
||||
augmented_addendum = ({ el_val_t _if_result_791 = 0; if (str_eq(lc_affective_note, EL_STR(""))) { _if_result_791 = (augmented_addendum); } else { _if_result_791 = (({ el_val_t _if_result_792 = 0; if (str_eq(augmented_addendum, EL_STR(""))) { _if_result_792 = (lc_affective_note); } else { _if_result_792 = (el_str_concat(el_str_concat(lc_affective_note, EL_STR("\n")), augmented_addendum)); } _if_result_792; })); } _if_result_791; });
|
||||
augmented_addendum = ({ el_val_t _if_result_819 = 0; if (str_eq(lc_affective_note, EL_STR(""))) { _if_result_819 = (augmented_addendum); } else { _if_result_819 = (({ el_val_t _if_result_820 = 0; if (str_eq(augmented_addendum, EL_STR(""))) { _if_result_820 = (lc_affective_note); } else { _if_result_820 = (el_str_concat(el_str_concat(lc_affective_note, EL_STR("\n")), augmented_addendum)); } _if_result_820; })); } _if_result_819; });
|
||||
state_set(EL_STR("layered_cycle_safety_system_addendum"), augmented_addendum);
|
||||
el_val_t prompt = imprint_respond(aligned, imprint_id);
|
||||
el_val_t output = layered_generate(prompt, imprint_id, session_id);
|
||||
@@ -31633,17 +31838,17 @@ el_val_t layered_cycle(el_val_t raw_input, el_val_t session_id, el_val_t utility
|
||||
int main(int _argc, char** _argv) {
|
||||
el_runtime_init_args(_argc, _argv);
|
||||
soul_cgi_id_raw = env(EL_STR("SOUL_CGI_ID"));
|
||||
soul_cgi_id = ({ el_val_t _if_result_793 = 0; if (str_eq(soul_cgi_id_raw, EL_STR(""))) { _if_result_793 = (EL_STR("ntn-genesis")); } else { _if_result_793 = (soul_cgi_id_raw); } _if_result_793; });
|
||||
soul_cgi_id = ({ el_val_t _if_result_821 = 0; if (str_eq(soul_cgi_id_raw, EL_STR(""))) { _if_result_821 = (EL_STR("ntn-genesis")); } else { _if_result_821 = (soul_cgi_id_raw); } _if_result_821; });
|
||||
port_raw = env(EL_STR("NEURON_PORT"));
|
||||
port = ({ el_val_t _if_result_794 = 0; if (str_eq(port_raw, EL_STR(""))) { _if_result_794 = (7770); } else { _if_result_794 = (str_to_int(port_raw)); } _if_result_794; });
|
||||
port = ({ el_val_t _if_result_822 = 0; if (str_eq(port_raw, EL_STR(""))) { _if_result_822 = (7770); } else { _if_result_822 = (str_to_int(port_raw)); } _if_result_822; });
|
||||
engram_url_raw = env(EL_STR("ENGRAM_URL"));
|
||||
engram_api_key_raw = env(EL_STR("ENGRAM_API_KEY"));
|
||||
snapshot_raw = env(EL_STR("SOUL_ENGRAM_PATH"));
|
||||
snapshot = ({ el_val_t _if_result_795 = 0; if (str_eq(snapshot_raw, EL_STR(""))) { _if_result_795 = (el_str_concat(env(EL_STR("HOME")), EL_STR("/.neuron/engram/snapshot.json"))); } else { _if_result_795 = (snapshot_raw); } _if_result_795; });
|
||||
snapshot = ({ el_val_t _if_result_823 = 0; if (str_eq(snapshot_raw, EL_STR(""))) { _if_result_823 = (el_str_concat(env(EL_STR("HOME")), EL_STR("/.neuron/engram/snapshot.json"))); } else { _if_result_823 = (snapshot_raw); } _if_result_823; });
|
||||
axon_raw = env(EL_STR("NEURON_API_URL"));
|
||||
axon_base = ({ el_val_t _if_result_796 = 0; if (str_eq(axon_raw, EL_STR(""))) { _if_result_796 = (EL_STR("http://localhost:7771")); } else { _if_result_796 = (axon_raw); } _if_result_796; });
|
||||
axon_base = ({ el_val_t _if_result_824 = 0; if (str_eq(axon_raw, EL_STR(""))) { _if_result_824 = (EL_STR("http://localhost:7771")); } else { _if_result_824 = (axon_raw); } _if_result_824; });
|
||||
studio_dir_raw = env(EL_STR("SOUL_STUDIO_DIR"));
|
||||
studio_dir = ({ el_val_t _if_result_797 = 0; if (str_eq(studio_dir_raw, EL_STR(""))) { _if_result_797 = (el_str_concat(env(EL_STR("HOME")), EL_STR("/Development/neuron-technologies/products/cgi-studio/el-daemon"))); } else { _if_result_797 = (studio_dir_raw); } _if_result_797; });
|
||||
studio_dir = ({ el_val_t _if_result_825 = 0; if (str_eq(studio_dir_raw, EL_STR(""))) { _if_result_825 = (el_str_concat(env(EL_STR("HOME")), EL_STR("/Development/neuron-technologies/products/cgi-studio/el-daemon"))); } else { _if_result_825 = (studio_dir_raw); } _if_result_825; });
|
||||
println(el_str_concat(el_str_concat(el_str_concat(EL_STR("[soul] boot - cgi="), soul_cgi_id), EL_STR(" port=")), int_to_str(port)));
|
||||
using_http_engram = !str_eq(engram_url_raw, EL_STR(""));
|
||||
engram_load(snapshot);
|
||||
@@ -31653,8 +31858,8 @@ int main(int _argc, char** _argv) {
|
||||
println(el_str_concat(el_str_concat(EL_STR("[soul] engram -> HTTP "), engram_url_raw), EL_STR(" (no local snapshot, first boot)")));
|
||||
el_val_t nodes_json = http_get(el_str_concat(engram_url_raw, EL_STR("/api/nodes?limit=10000")));
|
||||
el_val_t edges_json = http_get(el_str_concat(engram_url_raw, EL_STR("/api/edges")));
|
||||
el_val_t nodes_part = ({ el_val_t _if_result_798 = 0; if (str_eq(nodes_json, EL_STR(""))) { _if_result_798 = (EL_STR("[]")); } else { _if_result_798 = (nodes_json); } _if_result_798; });
|
||||
el_val_t edges_part = ({ el_val_t _if_result_799 = 0; if (str_eq(edges_json, EL_STR(""))) { _if_result_799 = (EL_STR("[]")); } else { _if_result_799 = (edges_json); } _if_result_799; });
|
||||
el_val_t nodes_part = ({ el_val_t _if_result_826 = 0; if (str_eq(nodes_json, EL_STR(""))) { _if_result_826 = (EL_STR("[]")); } else { _if_result_826 = (nodes_json); } _if_result_826; });
|
||||
el_val_t edges_part = ({ el_val_t _if_result_827 = 0; if (str_eq(edges_json, EL_STR(""))) { _if_result_827 = (EL_STR("[]")); } else { _if_result_827 = (edges_json); } _if_result_827; });
|
||||
el_val_t snapshot_data = el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"nodes\":"), nodes_part), EL_STR(",\"edges\":")), edges_part), EL_STR("}"));
|
||||
el_val_t tmp_path = el_str_concat(el_str_concat(EL_STR("/tmp/soul-engram-"), soul_cgi_id), EL_STR(".json"));
|
||||
fs_write(tmp_path, snapshot_data);
|
||||
@@ -31678,7 +31883,7 @@ int main(int _argc, char** _argv) {
|
||||
state_set(EL_STR("soul_engram_api_key"), engram_api_key_raw);
|
||||
state_set(EL_STR("soul.running"), EL_STR("true"));
|
||||
is_genesis = str_eq(soul_cgi_id, EL_STR("ntn-genesis"));
|
||||
guard_disk = ({ el_val_t _if_result_800 = 0; if (str_eq(engram_url_raw, EL_STR(""))) { _if_result_800 = (fs_read(snapshot)); } else { _if_result_800 = (EL_STR("")); } _if_result_800; });
|
||||
guard_disk = ({ el_val_t _if_result_828 = 0; if (str_eq(engram_url_raw, EL_STR(""))) { _if_result_828 = (fs_read(snapshot)); } else { _if_result_828 = (EL_STR("")); } _if_result_828; });
|
||||
guard_disk_len = str_len(guard_disk);
|
||||
safe_to_seed = (!using_http_engram && !((guard_disk_len > 200000) && ((engram_node_count() * 16000) < guard_disk_len)));
|
||||
if (is_genesis && !safe_to_seed) {
|
||||
|
||||
+11
-2
@@ -110,7 +110,7 @@ tool("beginSession", "Initialize session: surface recent high-importance memorie
|
||||
"," + tool("linkCausal", "Create a causal edge (cause -> effect).") +
|
||||
"," + tool("restructureCausalGraph", "Re-balance the causal subgraph after new evidence.") +
|
||||
"," + tool("rebuildGraph", "Rebuild graph indices from the on-disk snapshot.") +
|
||||
"," + tool("runStructuralAudit", "Audit graph structure for orphans, dangling edges, mislabeled types.") +
|
||||
"," + tool("runStructuralAudit", "Stage 1 structural audit: owner-vs-runtime divergence, orphans and dangling edges, typed-edge distribution, self-model connectivity. Returns an annotated characterization, not a score.") +
|
||||
// ── Backlog + work ──────────────────────────────────────────────────────────
|
||||
"," + tool("planWork", "Create a backlog item.") +
|
||||
"," + tool("reviewBacklog", "Browse work items.") +
|
||||
@@ -680,7 +680,16 @@ fn dispatch_tool_call(tool_name: String, args: String) -> String {
|
||||
return mcp_json_result(resp)
|
||||
}
|
||||
if str_eq(tool_name, "runStructuralAudit") {
|
||||
let resp: String = http_get(neuron_url() + "/session/begin")
|
||||
// Was: GET /session/begin — an unrelated session digest returned under an
|
||||
// audit tool name, i.e. the tool advertised a check that did not exist.
|
||||
// Now points at the real Stage 1 route (neuron-api.el
|
||||
// handle_api_structural_audit). Sample caps ride the query string; the
|
||||
// defaults keep a manual audit to a couple of seconds.
|
||||
let e_s: Int = json_get_int(args, "edge_sample")
|
||||
let n_s: Int = json_get_int(args, "node_sample")
|
||||
let qs: String = "?edge_sample=" + int_to_str(if e_s > 0 { e_s } else { 3000 })
|
||||
+ "&node_sample=" + int_to_str(if n_s > 0 { n_s } else { 300 })
|
||||
let resp: String = http_get(neuron_url() + "/audit/structural" + qs)
|
||||
return mcp_json_result(resp)
|
||||
}
|
||||
|
||||
|
||||
+460
-6
@@ -459,12 +459,7 @@ fn handle_api_recall(method: String, path: String, body: String) -> String {
|
||||
if str_eq(eff_q, "") {
|
||||
return api_or_empty(engram_scan_nodes_json(limit, 0))
|
||||
}
|
||||
// engram_recall_json, not engram_search_json: this route IS the retrieval
|
||||
// surface (claim 24's "embedding search queries"), so it gets the semantic
|
||||
// and associative legs. engram_search_json stays lexical because ~40
|
||||
// internal call sites pass a KEY and seven of them delete every record
|
||||
// that comes back — see the boundary note above eg_search_json_impl.
|
||||
let results: String = engram_recall_json(eff_q, limit)
|
||||
let results: String = engram_search_json(eff_q, limit)
|
||||
return api_or_empty(results)
|
||||
}
|
||||
|
||||
@@ -894,3 +889,462 @@ fn handle_api_consolidate(body: String) -> String {
|
||||
}
|
||||
return "{\"ok\":true,\"snapshot\":\"" + snap + "\"}"
|
||||
}
|
||||
|
||||
// ── Stage 1: structural audit ─────────────────────────────────────────────────
|
||||
//
|
||||
// WHAT THIS IMPLEMENTS
|
||||
// The CGI provisional, 05-detailed-description.md, "Stage 1: Structural audit
|
||||
// 430". Verbatim, the audit module evaluates: the density and typed
|
||||
// distribution of causal edges; the consistency between value nodes and
|
||||
// execution-record neighborhoods; the richness and connectivity of the
|
||||
// self-model; and the authenticity of open-question nodes in the wonder
|
||||
// manifest. It "produces a coherence assessment 432 — NOT A BINARY SCORE but
|
||||
// an annotated characterization of the graph's structural properties".
|
||||
//
|
||||
// That last clause is the whole shape of this handler. Every finding carries
|
||||
// its own numbers AND a plain-language note saying what the numbers mean and
|
||||
// how they were obtained. There is no pass/fail, no percentage-of-health, no
|
||||
// composite score, and `"score":null` is emitted explicitly so a downstream
|
||||
// reader cannot mistake its absence for an omission.
|
||||
//
|
||||
// WHY IT EXISTS NOW, AND WHY THE FIRST FINDING IS THE ONE IT IS
|
||||
// `runStructuralAudit` has been an advertised MCP tool with nothing behind it:
|
||||
// the dispatcher GET'd /session/begin and returned that blob (mcp-wrapper/src/
|
||||
// main.el). Meanwhile the failure the audit would have caught ran silently for
|
||||
// about three weeks — the soul reported 103,089 nodes while the engram, which
|
||||
// OWNS persistence, held ~79,900; a crash discarded the difference. Every boot
|
||||
// reported green throughout, because nothing in the system ever compared the
|
||||
// two sides. So finding 1 is owner-versus-runtime divergence: it is the check
|
||||
// whose absence cost real memory, and it is cheap and exact.
|
||||
//
|
||||
// WHAT IS DELIBERATELY NOT HERE (stage 1b, see the `deferred` array in the
|
||||
// response): value/execution-record consistency and wonder-manifest
|
||||
// authenticity. Both need node types that barely exist in this graph today —
|
||||
// the response MEASURES those populations and reports the counts as the reason,
|
||||
// rather than asserting a deferral without evidence.
|
||||
//
|
||||
// MEASUREMENT HONESTY: EXACT WHERE CHEAP, SAMPLED WHERE NOT, ALWAYS LABELLED
|
||||
// Counts, edge typing and self-model connectivity are exact. Orphan rate and
|
||||
// dangling-edge rate are SAMPLED, because the engram runtime has no node-id
|
||||
// index — `engram_find_node_index` is a linear scan over every node, so an
|
||||
// exhaustive dangling check is O(nodes x edges) (~2.2e9 string compares at
|
||||
// today's scale, tens of seconds inside one request). The samples are UNIFORM
|
||||
// across the whole population, not head-of-list, and every sampled figure is
|
||||
// emitted with its own `sampled` / `population` fields plus an extrapolation
|
||||
// labelled as such. Raise `?edge_sample=` / `?node_sample=` to the population
|
||||
// size to run either check exhaustively and pay the time. The real fix is an
|
||||
// id index in the runtime; that is the engram repo's, not this handler's.
|
||||
|
||||
// audit_pct1 — one-decimal percentage as a bare JSON number, sign-safe.
|
||||
// Integer math only: EL has no fixed-precision formatter, and float_to_str
|
||||
// would put an unbounded mantissa in the response.
|
||||
fn audit_pct1(num: Int, den: Int) -> String {
|
||||
if den <= 0 { return "null" }
|
||||
let neg: Bool = num < 0
|
||||
let a: Int = if neg { 0 - num } else { num }
|
||||
let tenths: Int = (a * 1000) / den
|
||||
let whole: Int = tenths / 10
|
||||
let frac: Int = tenths - (whole * 10)
|
||||
let sign: String = if neg { "-" } else { "" }
|
||||
return sign + int_to_str(whole) + "." + int_to_str(frac)
|
||||
}
|
||||
|
||||
// audit_finding — the one envelope every finding uses: name, the measurements,
|
||||
// and the annotation. Keeping it in one place is what stops the characterization
|
||||
// from degenerating into a bag of numbers with no reading attached.
|
||||
fn audit_finding(name: String, measured: String, note: String) -> String {
|
||||
return "{\"finding\":\"" + name + "\""
|
||||
+ ",\"measured\":{" + measured + "}"
|
||||
+ ",\"note\":\"" + api_json_escape(note) + "\"}"
|
||||
}
|
||||
|
||||
// audit_str_at — read the quoted string value starting at byte `start`.
|
||||
// Slices a bounded window rather than the tail of the (multi-MB) edges array, so
|
||||
// this is O(window) per call instead of O(remaining input).
|
||||
fn audit_str_at(s: String, start: Int, maxlen: Int) -> String {
|
||||
let n: Int = str_len(s)
|
||||
if start < 0 || start >= n { return "" }
|
||||
let end_guess: Int = start + maxlen
|
||||
let stop: Int = if end_guess > n { n } else { end_guess }
|
||||
let win: String = str_slice(s, start, stop)
|
||||
let q: Int = str_index_of(win, "\"")
|
||||
if q < 0 { return "" }
|
||||
return str_slice(win, 0, q)
|
||||
}
|
||||
|
||||
// audit_rel_count — exact count of edges carrying `rel`, by scanning the emitted
|
||||
// edge array for the literal `"relation":"<rel>"`. engram_emit_edge_json writes
|
||||
// metadata ESCAPED as a string, so no nested object can contain that literal and
|
||||
// the count cannot be inflated by edge payloads.
|
||||
fn audit_rel_count(edges: String, rel: String) -> Int {
|
||||
return str_count(edges, "\"relation\":\"" + rel + "\"")
|
||||
}
|
||||
|
||||
// audit_owner_stats — ask the persistence OWNER for its own counts.
|
||||
// Returns "" when there is no HTTP owner configured or the owner is unreachable;
|
||||
// both are reported as findings, never as a failure of the audit.
|
||||
fn audit_owner_stats(url: String) -> String {
|
||||
if str_eq(url, "") { return "" }
|
||||
return http_get(url + "/api/stats")
|
||||
}
|
||||
|
||||
// audit_divergence — FINDING 1. Runtime (this soul's in-process graph) versus
|
||||
// the persistence owner's own count. Trend is measured against the previous
|
||||
// audit recorded in soul state, so a second call answers "is the gap growing?"
|
||||
// rather than just restating it.
|
||||
fn audit_divergence() -> String {
|
||||
let rt_nodes: Int = engram_node_count()
|
||||
let rt_edges: Int = engram_edge_count()
|
||||
let url: String = wt_engram_url()
|
||||
|
||||
if str_eq(url, "") {
|
||||
return audit_finding("owner_runtime_divergence",
|
||||
"\"runtime_nodes\":" + int_to_str(rt_nodes)
|
||||
+ ",\"runtime_edges\":" + int_to_str(rt_edges)
|
||||
+ ",\"owner\":\"none\",\"owner_reachable\":false",
|
||||
"No HTTP persistence owner is configured, so this soul IS the owner "
|
||||
+ "(file mode) and divergence is not defined. This check only has "
|
||||
+ "meaning when ENGRAM_URL points at a separate engram that owns the "
|
||||
+ "canonical store.")
|
||||
}
|
||||
|
||||
let stats: String = audit_owner_stats(url)
|
||||
// REACHABILITY IS PROVED BY THE PAYLOAD, NOT BY A NON-EMPTY REPLY.
|
||||
// http_get does not return "" on a connection failure — it returns a JSON
|
||||
// error object ({"error":"Failed to connect to ... Couldn't connect to
|
||||
// server"}). Testing only for "" made a DEAD owner read as reachable with
|
||||
// node_count 0, i.e. the audit would have reported a 100% divergence and
|
||||
// named it as data loss. That false positive is worse than no check at all:
|
||||
// it is precisely the kind of confident wrong answer this route exists to
|
||||
// stop. Require the field the contract promises.
|
||||
let owner_nc_raw: String = json_get_raw(stats, "node_count")
|
||||
if str_eq(stats, "") || str_eq(owner_nc_raw, "") {
|
||||
return audit_finding("owner_runtime_divergence",
|
||||
"\"runtime_nodes\":" + int_to_str(rt_nodes)
|
||||
+ ",\"runtime_edges\":" + int_to_str(rt_edges)
|
||||
+ ",\"owner\":\"" + api_json_escape(url) + "\",\"owner_reachable\":false"
|
||||
+ ",\"owner_reply\":\"" + api_json_escape(api_utf8_trunc(stats, 200)) + "\"",
|
||||
"The persistence owner at " + url + " did not return a node_count "
|
||||
+ "from GET /api/stats. Divergence is UNKNOWN, NOT ZERO — an owner "
|
||||
+ "that cannot be read is exactly the condition under which the "
|
||||
+ "runtime's own count means least, and reporting 0 for the owner "
|
||||
+ "would manufacture a total-loss reading out of a network error. "
|
||||
+ "Reported as a finding rather than raised as an error so the rest "
|
||||
+ "of the audit still returns; the owner's raw reply is in "
|
||||
+ "owner_reply.")
|
||||
}
|
||||
|
||||
let ow_nodes: Int = json_get_int(stats, "node_count")
|
||||
let ow_edges: Int = json_get_int(stats, "edge_count")
|
||||
let d_nodes: Int = rt_nodes - ow_nodes
|
||||
let d_edges: Int = rt_edges - ow_edges
|
||||
|
||||
// Trend against the previous audit in this soul's state.
|
||||
let prev_raw: String = state_get("audit_prev_node_delta")
|
||||
let prev: Int = str_to_int(prev_raw)
|
||||
let abs_now: Int = if d_nodes < 0 { 0 - d_nodes } else { d_nodes }
|
||||
let abs_prev: Int = if prev < 0 { 0 - prev } else { prev }
|
||||
let trend: String = if str_eq(prev_raw, "") {
|
||||
"no_prior_audit"
|
||||
} else {
|
||||
if abs_now > abs_prev { "growing" } else {
|
||||
if abs_now < abs_prev { "shrinking" } else { "flat" }
|
||||
}
|
||||
}
|
||||
state_set("audit_prev_node_delta", int_to_str(d_nodes))
|
||||
state_set("audit_prev_ts", int_to_str(time_now()))
|
||||
|
||||
let note_head: String = if d_nodes == 0 {
|
||||
"Runtime and owner agree on node count."
|
||||
} else {
|
||||
"Runtime holds " + int_to_str(d_nodes) + " nodes (" + audit_pct1(d_nodes, rt_nodes)
|
||||
+ "% of its own graph) that the persistence owner does not report. Nodes "
|
||||
+ "that exist only in runtime memory do not survive a restart."
|
||||
}
|
||||
return audit_finding("owner_runtime_divergence",
|
||||
"\"runtime_nodes\":" + int_to_str(rt_nodes)
|
||||
+ ",\"runtime_edges\":" + int_to_str(rt_edges)
|
||||
+ ",\"owner\":\"" + api_json_escape(url) + "\",\"owner_reachable\":true"
|
||||
+ ",\"owner_nodes\":" + int_to_str(ow_nodes)
|
||||
+ ",\"owner_edges\":" + int_to_str(ow_edges)
|
||||
+ ",\"node_delta\":" + int_to_str(d_nodes)
|
||||
+ ",\"edge_delta\":" + int_to_str(d_edges)
|
||||
+ ",\"node_delta_pct_of_runtime\":" + audit_pct1(d_nodes, rt_nodes)
|
||||
+ ",\"trend_vs_previous_audit\":\"" + trend + "\""
|
||||
+ ",\"previous_node_delta\":" + (if str_eq(prev_raw, "") { "null" } else { int_to_str(prev) }),
|
||||
note_head + " Trend against the previous audit recorded in this soul's "
|
||||
+ "state: " + trend + ". This is the comparison whose absence let a "
|
||||
+ "~24,000-node loss run for weeks with every boot reporting green.")
|
||||
}
|
||||
|
||||
// audit_edge_typing — FINDING 2. Density plus the typed distribution the patent
|
||||
// asks for, against the claim-10 relation vocabulary. Exact: str_count over the
|
||||
// emitted edge array, one linear pass per relation.
|
||||
fn audit_edge_typing(edges: String, total_edges: Int, node_total: Int) -> String {
|
||||
let c_sup: Int = audit_rel_count(edges, "Supersedes")
|
||||
let c_cau: Int = audit_rel_count(edges, "Causes")
|
||||
let c_con: Int = audit_rel_count(edges, "Contains")
|
||||
let c_ref: Int = audit_rel_count(edges, "References")
|
||||
let c_ctr: Int = audit_rel_count(edges, "Contradicts")
|
||||
let c_exe: Int = audit_rel_count(edges, "Exemplifies")
|
||||
let c_act: Int = audit_rel_count(edges, "Activates")
|
||||
let c_tmp: Int = audit_rel_count(edges, "TemporallyPrecedes")
|
||||
let typed: Int = c_sup + c_cau + c_con + c_ref + c_ctr + c_exe + c_act + c_tmp
|
||||
|
||||
// Lowercase near-misses: the same eight concepts written by the ad-hoc write
|
||||
// paths (linkEntities defaults to "associates", linkCausal to "causes").
|
||||
// Counted separately because "the vocabulary is unused" and "the vocabulary
|
||||
// is used in the wrong case" are different defects with different fixes.
|
||||
let l_sup: Int = audit_rel_count(edges, "supersedes")
|
||||
let l_cau: Int = audit_rel_count(edges, "causes")
|
||||
let l_con: Int = audit_rel_count(edges, "contains")
|
||||
let l_ref: Int = audit_rel_count(edges, "references")
|
||||
let l_ctr: Int = audit_rel_count(edges, "contradicts")
|
||||
let l_exe: Int = audit_rel_count(edges, "exemplifies")
|
||||
let l_act: Int = audit_rel_count(edges, "activates")
|
||||
let l_tmp: Int = audit_rel_count(edges, "temporallyPrecedes")
|
||||
let near: Int = l_sup + l_cau + l_con + l_ref + l_ctr + l_exe + l_act + l_tmp
|
||||
|
||||
let untyped: Int = total_edges - typed
|
||||
return audit_finding("typed_edge_distribution",
|
||||
"\"total_edges\":" + int_to_str(total_edges)
|
||||
+ ",\"total_nodes\":" + int_to_str(node_total)
|
||||
// Density per 100 nodes, not per node: EL has no fixed-precision float
|
||||
// formatter, and "0.3 edges per node" rounded to an integer is a lie.
|
||||
+ ",\"edges_per_100_nodes\":" + audit_pct1(total_edges, node_total)
|
||||
+ ",\"claim10_typed\":" + int_to_str(typed)
|
||||
+ ",\"claim10_typed_pct\":" + audit_pct1(typed, total_edges)
|
||||
+ ",\"outside_claim10_vocabulary\":" + int_to_str(untyped)
|
||||
+ ",\"lowercase_near_miss\":" + int_to_str(near)
|
||||
+ ",\"by_relation\":{"
|
||||
+ "\"Supersedes\":" + int_to_str(c_sup)
|
||||
+ ",\"Causes\":" + int_to_str(c_cau)
|
||||
+ ",\"Contains\":" + int_to_str(c_con)
|
||||
+ ",\"References\":" + int_to_str(c_ref)
|
||||
+ ",\"Contradicts\":" + int_to_str(c_ctr)
|
||||
+ ",\"Exemplifies\":" + int_to_str(c_exe)
|
||||
+ ",\"Activates\":" + int_to_str(c_act)
|
||||
+ ",\"TemporallyPrecedes\":" + int_to_str(c_tmp) + "}",
|
||||
"Only " + int_to_str(typed) + " of " + int_to_str(total_edges)
|
||||
+ " edges use the claim-10 causal vocabulary; the remainder are ad-hoc "
|
||||
+ "relation strings, which is why the graph's causal claims cannot yet "
|
||||
+ "be checked for internal consistency — an untyped edge asserts "
|
||||
+ "association, not causation. " + int_to_str(near) + " edges use a "
|
||||
+ "lowercase spelling of a claim-10 relation: those are near-misses the "
|
||||
+ "write paths could be corrected to emit, not genuinely foreign types.")
|
||||
}
|
||||
|
||||
// audit_orphans_dangling — FINDING 3. Both figures are SAMPLED; see the header
|
||||
// for why exhaustive is O(nodes x edges) on this runtime.
|
||||
//
|
||||
// An "orphan" here is a node with zero RESOLVABLE edges: engram_neighbors_json
|
||||
// drops any edge whose other endpoint does not resolve to a node, so a node
|
||||
// whose only edges are dangling reads as an orphan. That is the right reading —
|
||||
// such a node is unreachable by traversal — but it is stated rather than hidden.
|
||||
fn audit_orphans_dangling(edges: String, total_edges: Int, node_total: Int,
|
||||
edge_cap: Int, node_cap: Int) -> String {
|
||||
// ── orphan sample: uniform stride over the node store ──
|
||||
let n_take: Int = if node_total < node_cap { node_total } else { node_cap }
|
||||
let n_stride: Int = if n_take > 0 { node_total / n_take } else { 1 }
|
||||
let n_stride = if n_stride < 1 { 1 } else { n_stride }
|
||||
let orphans: Int = 0
|
||||
let n_checked: Int = 0
|
||||
let j: Int = 0
|
||||
while j < n_take {
|
||||
let one: String = engram_scan_nodes_json(1, j * n_stride)
|
||||
let nid: String = json_get(json_array_get(one, 0), "id")
|
||||
if !str_eq(nid, "") {
|
||||
let nbrs: String = engram_neighbors_json(nid, 1, "both")
|
||||
let deg: Int = json_array_len(nbrs)
|
||||
let orphans = if deg == 0 { orphans + 1 } else { orphans }
|
||||
let n_checked = n_checked + 1
|
||||
}
|
||||
let j = j + 1
|
||||
}
|
||||
|
||||
// ── dangling sample: uniform stride over the edge array ──
|
||||
// str_index_of_all gives every edge's field offsets in ONE linear pass, so
|
||||
// any index can be read in O(1). json_array_get would have been O(i) per
|
||||
// element and O(n^2) over the array.
|
||||
let from_pos: [Int] = str_index_of_all(edges, "\"from_id\":\"")
|
||||
let to_pos: [Int] = str_index_of_all(edges, "\"to_id\":\"")
|
||||
let nf: Int = len(from_pos)
|
||||
let nt: Int = len(to_pos)
|
||||
let ne: Int = if nf < nt { nf } else { nt }
|
||||
let e_take: Int = if ne < edge_cap { ne } else { edge_cap }
|
||||
let e_stride: Int = if e_take > 0 { ne / e_take } else { 1 }
|
||||
let e_stride = if e_stride < 1 { 1 } else { e_stride }
|
||||
let dangling: Int = 0
|
||||
let e_checked: Int = 0
|
||||
let i: Int = 0
|
||||
while i < ne && e_checked < e_take {
|
||||
let fid: String = audit_str_at(edges, get(from_pos, i) + 11, 96)
|
||||
let tid: String = audit_str_at(edges, get(to_pos, i) + 9, 96)
|
||||
let f_gone: Bool = str_eq(engram_get_node_json(fid), "{}")
|
||||
let t_gone: Bool = if f_gone { true } else { str_eq(engram_get_node_json(tid), "{}") }
|
||||
let dangling = if f_gone || t_gone { dangling + 1 } else { dangling }
|
||||
let e_checked = e_checked + 1
|
||||
let i = i + e_stride
|
||||
}
|
||||
|
||||
let orphan_est: Int = if n_checked > 0 { (orphans * node_total) / n_checked } else { 0 }
|
||||
let dangle_est: Int = if e_checked > 0 { (dangling * total_edges) / e_checked } else { 0 }
|
||||
let exhaustive_n: String = if n_checked >= node_total { "true" } else { "false" }
|
||||
let exhaustive_e: String = if e_checked >= ne { "true" } else { "false" }
|
||||
|
||||
return audit_finding("orphans_and_dangling_edges",
|
||||
"\"nodes_population\":" + int_to_str(node_total)
|
||||
+ ",\"nodes_sampled\":" + int_to_str(n_checked)
|
||||
+ ",\"nodes_sample_exhaustive\":" + exhaustive_n
|
||||
+ ",\"orphans_in_sample\":" + int_to_str(orphans)
|
||||
+ ",\"orphan_rate_pct\":" + audit_pct1(orphans, n_checked)
|
||||
+ ",\"orphans_extrapolated\":" + int_to_str(orphan_est)
|
||||
+ ",\"edges_population\":" + int_to_str(total_edges)
|
||||
+ ",\"edges_sampled\":" + int_to_str(e_checked)
|
||||
+ ",\"edges_sample_exhaustive\":" + exhaustive_e
|
||||
+ ",\"dangling_in_sample\":" + int_to_str(dangling)
|
||||
+ ",\"dangling_rate_pct\":" + audit_pct1(dangling, e_checked)
|
||||
+ ",\"dangling_extrapolated\":" + int_to_str(dangle_est),
|
||||
"Orphan = zero RESOLVABLE edges, so a node whose only edges dangle counts "
|
||||
+ "as an orphan; either way it is unreachable by traversal. Dangling = an "
|
||||
+ "edge with an endpoint id that resolves to no node. Both are uniform "
|
||||
+ "stride samples over the whole population, not the head of the list; "
|
||||
+ "the extrapolations are estimates and are labelled as such. Pass "
|
||||
+ "?node_sample= / ?edge_sample= at or above the population size to run "
|
||||
+ "either check exhaustively. A high orphan rate is a characterization, "
|
||||
+ "not a verdict: an accumulating store legitimately holds unlinked "
|
||||
+ "material. It becomes a defect when the write paths were SUPPOSED to "
|
||||
+ "link and did not.")
|
||||
}
|
||||
|
||||
// audit_pillar — one self-model pillar: present, how much content, how connected.
|
||||
fn audit_pillar(key: String, id: String) -> String {
|
||||
let node: String = engram_get_node_json(id)
|
||||
let present: Bool = !str_eq(node, "{}") && !str_eq(node, "")
|
||||
if !present {
|
||||
return "\"" + key + "\":{\"id\":\"" + id + "\",\"present\":false"
|
||||
+ ",\"content_length\":0,\"degree\":0}"
|
||||
}
|
||||
let content: String = json_get(node, "content")
|
||||
let deg: Int = json_array_len(engram_neighbors_json(id, 1, "both"))
|
||||
return "\"" + key + "\":{\"id\":\"" + id + "\",\"present\":true"
|
||||
+ ",\"label\":\"" + api_json_escape(json_get(node, "label")) + "\""
|
||||
+ ",\"tier\":\"" + api_json_escape(json_get(node, "tier")) + "\""
|
||||
+ ",\"content_length\":" + int_to_str(str_len(content))
|
||||
+ ",\"degree\":" + int_to_str(deg) + "}"
|
||||
}
|
||||
|
||||
// audit_self_model — FINDING 4. "the richness and connectivity of the
|
||||
// self-model ... is it connected to behavioral evidence?"
|
||||
//
|
||||
// This finding RETIRES the Claude-side vitals identity block. That check lived
|
||||
// outside the system it was checking — a shell script grepping a snapshot — so
|
||||
// it could only ever report on a file, and it went on reporting green while the
|
||||
// memory-philosophy pillar was absent from the live graph for about three weeks.
|
||||
// Asking the running soul about its own three pillars is the designed mechanism;
|
||||
// a shell probe was the fourth patch on the same hole.
|
||||
fn audit_self_model() -> String {
|
||||
let dna: String = audit_pillar("intellectual_dna", "kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6")
|
||||
let val: String = audit_pillar("values_hub", "kn-5b606390-a52d-4ca2-8e0e-eba141d13440")
|
||||
let phi: String = audit_pillar("memory_philosophy", "kn-dcfe04b3-3702-4cac-b6f0-ecb4db837eee")
|
||||
let root: String = audit_pillar("self_root", "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee")
|
||||
return audit_finding("self_model_connectivity",
|
||||
"\"pillars\":{" + dna + "," + val + "," + phi + "," + root + "}",
|
||||
"The three identity pillars plus the self root. `degree` counts nodes "
|
||||
+ "reachable in one hop in either direction — the self-model's connection "
|
||||
+ "to the rest of the graph. present:false on any pillar is the condition "
|
||||
+ "that ran undetected for weeks; content_length distinguishes a pillar "
|
||||
+ "that is present from one that is present but hollowed out. The patent "
|
||||
+ "also asks whether the self-model makes ACCURATE PREDICTIONS about the "
|
||||
+ "system's own behavior; that half needs Prediction nodes and is deferred "
|
||||
+ "with the rest of stage 1b below.")
|
||||
}
|
||||
|
||||
// audit_deferred — what stage 1 does NOT yet evaluate, with the measured reason.
|
||||
// Emitted as data, not as a comment, so a reader of the assessment sees the gap
|
||||
// and its evidence rather than inferring completeness from silence.
|
||||
fn audit_deferred() -> String {
|
||||
let preds: Int = json_array_len(api_or_empty(engram_scan_nodes_by_type_json("Prediction", 50, 0)))
|
||||
let wonders: Int = json_array_len(api_or_empty(engram_scan_nodes_by_type_json("WonderQuestion", 50, 0)))
|
||||
return "[{\"deferred\":\"value_execution_record_consistency\""
|
||||
+ ",\"stage\":\"1b\""
|
||||
+ ",\"measured\":{\"prediction_nodes_found\":" + int_to_str(preds) + "}"
|
||||
+ ",\"reason\":\"" + api_json_escape(
|
||||
"The patent asks whether the execution history SUPPORTS the stated "
|
||||
+ "values or shows systematic conflict. That requires execution "
|
||||
+ "records tied to value nodes and predictions to score them against. "
|
||||
+ "Prediction nodes found (capped at 50): " + int_to_str(preds)
|
||||
+ ". Asserting value/execution coherence on that population would be "
|
||||
+ "a fabricated result, which is worse than a stated gap.") + "\"}"
|
||||
+ ",{\"deferred\":\"wonder_manifest_authenticity\""
|
||||
+ ",\"stage\":\"1b\""
|
||||
+ ",\"measured\":{\"wonder_question_nodes_found\":" + int_to_str(wonders) + "}"
|
||||
+ ",\"reason\":\"" + api_json_escape(
|
||||
"The patent asks whether pull weights CORRELATE WITH GENUINE "
|
||||
+ "PREDICTION UNCERTAINTY or are uniform/externally assigned — a "
|
||||
+ "correlation between two populations. WonderQuestion nodes readable "
|
||||
+ "by type (capped at 50): " + int_to_str(wonders) + ", against "
|
||||
+ int_to_str(preds) + " Prediction nodes. There is a known write/read "
|
||||
+ "node-type mismatch on the wonder path; until that is fixed and both "
|
||||
+ "populations exist, any correlation reported here would be noise.") + "\"}]"
|
||||
}
|
||||
|
||||
// handle_api_structural_audit — Stage 1. Returns the coherence assessment 432:
|
||||
// an annotated characterization, explicitly NOT a score.
|
||||
//
|
||||
// COST NOTE: the edge findings need the relation labels, and the runtime exposes
|
||||
// no edge-enumeration builtin. The only way to see them is the same one
|
||||
// GET /api/graph/edges already uses — engram_save to a SCRATCH path (never the
|
||||
// owner's canonical file; see routes.el, neuron#117) and read the array back.
|
||||
// On a large graph that is a multi-hundred-MB write, so this is a manual audit
|
||||
// route, not something to put on a timer. Pass ?edges=0 to skip both edge
|
||||
// findings and get the divergence + self-model readings cheaply.
|
||||
fn handle_api_structural_audit(method: String, path: String, body: String) -> String {
|
||||
let node_total: Int = engram_node_count()
|
||||
let edge_total: Int = engram_edge_count()
|
||||
let want_edges: Bool = !str_eq(api_query_param(path, "edges"), "0")
|
||||
let edge_cap: Int = api_query_int(path, "edge_sample", 3000)
|
||||
let node_cap: Int = api_query_int(path, "node_sample", 300)
|
||||
|
||||
let divergence: String = audit_divergence()
|
||||
let self_model: String = audit_self_model()
|
||||
|
||||
let edge_part: String = if want_edges {
|
||||
// Scratch export only. state_get("soul_snapshot_path") is deliberately
|
||||
// NOT used: in HTTP-engram mode the soul is not the persistence owner and
|
||||
// must never write the canonical file, not even on a read path.
|
||||
let scratch_dir: String = env("TMPDIR")
|
||||
let scratch_base: String = if str_eq(scratch_dir, "") { "/tmp" } else { scratch_dir }
|
||||
let snap_path: String = scratch_base + "/soul-audit-export-" + state_get("soul_cgi_id") + ".json"
|
||||
// engram_save returns Int (1 ok / 0 fail); str_eq on it SIGSEGVs (#150).
|
||||
let saved: Int = engram_save(snap_path)
|
||||
if saved == 0 {
|
||||
"," + audit_finding("typed_edge_distribution", "\"available\":false",
|
||||
"Could not export the graph to " + snap_path + " for edge analysis, "
|
||||
+ "so edge typing and the dangling-edge sample were not run. "
|
||||
+ "Reported as a gap, not as zero findings.")
|
||||
} else {
|
||||
// wt_read, not fs_read: fs_read leaves a thread-local length hint that
|
||||
// the NEXT HTTP response would use as its Content-Length, appending
|
||||
// adjacent heap bytes to the reply (see persist.el wt_read).
|
||||
let snap: String = wt_read(snap_path)
|
||||
let edges_raw: String = json_get_raw(snap, "edges")
|
||||
let edges: String = if str_eq(edges_raw, "") { "[]" } else { edges_raw }
|
||||
"," + audit_edge_typing(edges, edge_total, node_total)
|
||||
+ "," + audit_orphans_dangling(edges, edge_total, node_total, edge_cap, node_cap)
|
||||
}
|
||||
} else {
|
||||
""
|
||||
}
|
||||
|
||||
return "{\"audit\":\"structural\",\"stage\":1"
|
||||
+ ",\"spec\":\"CGI provisional 05-detailed-description.md, Stage 1: Structural audit 430\""
|
||||
+ ",\"assessment\":\"coherence_assessment_432\""
|
||||
+ ",\"assessment_kind\":\"annotated_characterization\""
|
||||
+ ",\"score\":null"
|
||||
+ ",\"score_note\":\"By design. The specification calls for an annotated characterization of the graph's structural properties, not a binary score. Read the findings.\""
|
||||
+ ",\"cgi_id\":\"" + api_json_escape(state_get("soul_cgi_id")) + "\""
|
||||
+ ",\"ts_ms\":" + int_to_str(time_now())
|
||||
+ ",\"findings\":[" + divergence + "," + self_model + edge_part + "]"
|
||||
+ ",\"deferred\":" + audit_deferred() + "}"
|
||||
}
|
||||
|
||||
@@ -37,3 +37,4 @@ extern fn handle_api_memory_update(body: String) -> String
|
||||
extern fn handle_api_cultivate(body: String) -> String
|
||||
extern fn handle_api_list_typed(node_type: String, path: String, body: String) -> String
|
||||
extern fn handle_api_consolidate(body: String) -> String
|
||||
extern fn handle_api_structural_audit(method: String, path: String, body: String) -> String
|
||||
|
||||
@@ -567,6 +567,13 @@ fn route_dispatch(method: String, path: String, body: String) -> String {
|
||||
if str_starts_with(clean, "/api/neuron/graph") {
|
||||
return handle_api_inspect_graph(method, path, body)
|
||||
}
|
||||
// Stage 1 structural audit (CGI provisional, "Structural audit 430").
|
||||
// GET because it is a read of the graph's own structure; the query string
|
||||
// carries the sample caps (?edge_sample=, ?node_sample=, ?edges=0), so
|
||||
// str_starts_with rather than str_eq.
|
||||
if str_starts_with(clean, "/api/neuron/audit/structural") {
|
||||
return handle_api_structural_audit(method, path, body)
|
||||
}
|
||||
if str_starts_with(clean, "/api/neuron/list/") {
|
||||
// Offset 17 = len("/api/neuron/list/"). Was 16, which left a leading "/" on node_type
|
||||
// ("/BacklogItem"), so engram_scan_nodes_by_type_json matched nothing → list/<type>
|
||||
@@ -748,6 +755,12 @@ fn route_dispatch(method: String, path: String, body: String) -> String {
|
||||
if str_eq(clean, "/api/neuron/graph/link") {
|
||||
return handle_api_link_entities(body)
|
||||
}
|
||||
// POST accepted too: same handler, so a JSON-RPC-shaped caller that only
|
||||
// speaks POST reaches the identical audit. Options still come from the
|
||||
// query string — the handler reads no body fields.
|
||||
if str_eq(clean, "/api/neuron/audit/structural") {
|
||||
return handle_api_structural_audit(method, path, body)
|
||||
}
|
||||
if str_eq(clean, "/api/neuron/memory") {
|
||||
return handle_api_remember(body)
|
||||
}
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
# Retrieval eval harness
|
||||
|
||||
Measures Neuron's memory retrieval so a change can be shown to help before it is
|
||||
believed to help. Nothing else on the memory roadmap should ship without a run
|
||||
through this.
|
||||
|
||||
```
|
||||
tools/retrieval-eval/run_comparison.sh --baseline main --candidate <branch>
|
||||
```
|
||||
|
||||
That builds a soul from each ref, boots each in isolation on a fixed corpus,
|
||||
runs the gold set three times per ref, and prints a table plus a verdict that
|
||||
refuses to call a difference real if it is inside the noise band.
|
||||
|
||||
## What was reused
|
||||
|
||||
This is not a new idea, it is the missing third of an existing one.
|
||||
|
||||
| Prior work | What it gave | What was missing |
|
||||
|---|---|---|
|
||||
| `docs/research/graphrag_eval/` (`collect.py`, `score.py`, 2026-06-08) | The three-retriever comparison that produced the numbers everyone quotes: substring 1.7% P@5, graph 21.7%, BM25 55%. Per-query relevant-id scoring, fixed-denominator precision@5, unique-relevant analysis. | 13 hand-written queries, judged by an LLM after the fact; measured the *live* soul on the *live* engram. |
|
||||
| `docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py` | The pinned-query discipline: ground truth committed as regexes so every run judges alike, plus a `--check` winnability gate. 40 queries in 5 bands including a deliberate paraphrase-hard band. | Scored offline replicas of substring/BM25 — it never ran the real retrieval path. |
|
||||
| `docs/research-archive/p0-prototypes/stage0_eval_20260714.py` | The `hit@5` metric and the substring/BM25 reference implementations. | Same: offline only. |
|
||||
| `scripts/verify-soul-contract.sh` | The isolation recipe, verbatim: throwaway port, throwaway `HOME`, `SOUL_ENGRAM_PATH`, and the non-obvious `SOUL_ISE_URL` pin that stops an "isolated" soul silently syncing the operator's live brain. | It is a contract gate, not a measurement. |
|
||||
| `_engine-liveness-91/gen-soul-amalgam.sh` + `.gitea/workflows/ci.yaml` | The build recipe (`elc --target=c` with every `.elh` on the import chain removed) and CI's exact compile flags. | — |
|
||||
|
||||
**Reused directly:** the isolation recipe, the build recipe, fixed-denominator
|
||||
precision@5, the pinned-ground-truth and winnability ideas.
|
||||
**New here:** ids rather than regexes as ground truth, an associative category
|
||||
derived from real graph edges, a superseded/contradicted category scored on
|
||||
ranking, a machine-checked zero-lexical-overlap guarantee on paraphrases,
|
||||
paired significance testing, and — the point — measurement against the **real
|
||||
compiled soul** rather than an offline replica of one leg of it.
|
||||
|
||||
## Design fit
|
||||
|
||||
The thing under measurement is Will's designed retrieval: spreading activation
|
||||
over the weighted directed graph, four-factor multiplicative scoring (parent
|
||||
strength x edge weight x target salience x query/target cosine). A Python
|
||||
re-implementation would measure my reading of the design. So the harness
|
||||
compiles the actual `soul.el` amalgam and asks it over HTTP on
|
||||
`/api/neuron/recall`, exactly as the MCP wrapper and the app do.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Does |
|
||||
|---|---|
|
||||
| `build_gold_set.py` | Derives and **validates** the gold set from the corpus. `--check` re-validates and exits non-zero if a query became unwinnable or a paraphrase leaked a word. |
|
||||
| `gold_set.json` | 38 queries. Every one carries a `derivation` string. |
|
||||
| `run_eval.py` | Boots one soul in isolation, runs the gold set, writes metrics. Kills and **confirms dead** its child; records the confirmation in the results file. |
|
||||
| `compare.py` | Paired diff of two result files with McNemar's exact test and a stated noise floor. |
|
||||
| `build-soul.sh` | Compiles a soul binary from a plain source tree. |
|
||||
| `run_comparison.sh` | All of the above, end to end, from two git refs. |
|
||||
|
||||
## The gold set — 38 queries
|
||||
|
||||
Built from the real corpus (`snapshot-pre-repair-20260806.json`, 78,768 nodes /
|
||||
14,214 edges) so it reflects one person's accumulating memory, not document QA.
|
||||
|
||||
| Category | n | Expected answer derived by |
|
||||
|---|---|---|
|
||||
| `exact_rare` | 6 | **Mined.** Tokens with document frequency 1 across all 78,768 nodes, whose single containing node is a 300–6000 char Memory/Knowledge/Belief. That node is the only possible answer. Re-verified every build. |
|
||||
| `phrase` | 7 | **Mined.** Case-insensitive verbatim scan; the matching set *is* the answer key. Phrases matching >25 nodes are rejected as too diffuse. |
|
||||
| `paraphrase` | 13 | **Hand-selected, machine-checked.** Target locked by id; the build then proves that **zero** content words of the query appear anywhere in the target's label, content, or tags. A leak fails the build — the category cannot quietly decay into lexical matching. |
|
||||
| `associative` | 6 | **Derived from edges.** Query built from one value node's distinctive vocabulary; expected answers are its siblings on the `Self - Values (grounded)` hub. Siblings sharing any query word are dropped, so the only route from query to answer is seed -> hub -> sibling. |
|
||||
| `nonsense` | 3 | **Control.** Verified that no token occurs anywhere in the corpus. Correct behaviour is to return nothing. |
|
||||
| `superseded` | 3 | **Derived.** Correction/stale pairs located by regex scan, kept only when both sides resolve to different surviving nodes. Scored on **ranking**: the correction must be returned *and* rank above the stale node. |
|
||||
|
||||
## Metrics
|
||||
|
||||
`hit@5`, `recall@5`, `recall@10`, `precision@5` (fixed denominator 5, so an
|
||||
empty result is punished like a page of junk), `MRR@10`, and wall-clock latency
|
||||
per query (p50/p95/max). Output is a table plus a machine-readable JSON per run
|
||||
so runs can be diffed.
|
||||
|
||||
## Honesty about noise
|
||||
|
||||
- **Minimum detectable swing on this 38-query set: 6 queries.** If every query
|
||||
that changes changes the same way, `p = 2 x 0.5^n`, which first drops under
|
||||
0.05 at n=6. Any net change smaller than that is inside the noise band and
|
||||
`compare.py` says so in those words.
|
||||
- **Run-to-run drift is measured, not assumed.** Activation is a stateful read
|
||||
by design (traversal reinforces what it touches), so identical inputs need not
|
||||
give identical outputs. Observed: `main` 0 queries of drift across 3 runs
|
||||
(fully deterministic); the activation branch 1 query.
|
||||
- The noise floor used for the verdict is `max(6, observed_drift + 1)`.
|
||||
- **This gold set is underpowered for small effects.** A genuine 3-query
|
||||
improvement would not clear the bar. Growing the set is the fix; until then, a
|
||||
small positive delta means "not shown", not "no effect".
|
||||
|
||||
## First result: `main` vs `feat/recall-through-activation`
|
||||
|
||||
Corpus and gold set identical, three runs each, fresh corpus copy per run.
|
||||
|
||||
| | main | recall-through-activation | delta |
|
||||
|---|---|---|---|
|
||||
| hit@5 | 34.3% | 22.9% | **-11.4pp** |
|
||||
| recall@5 | 26.9% | 19.1% | -7.9pp |
|
||||
| recall@10 | 33.3% | 24.3% | -9.1pp |
|
||||
| precision@5 | 12.0% | 7.4% | -4.6pp |
|
||||
| MRR@10 | 0.294 | 0.242 | -0.053 |
|
||||
| latency p50 | 1140 ms | 3209 ms | **2.81x** |
|
||||
| latency p95 | 1584 ms | 4852 ms | 3.06x |
|
||||
| nonsense clean | 2/3 | 2/3 | — |
|
||||
| superseded outranks | 1/3 | 0/3 | -1 |
|
||||
|
||||
By category (hit@5):
|
||||
|
||||
| category | main | activation |
|
||||
|---|---|---|
|
||||
| exact_rare | 100% | 100% |
|
||||
| phrase | 85.7% | **28.6%** |
|
||||
| paraphrase | 0% | 0% |
|
||||
| associative | 0% | 0% |
|
||||
| superseded | 0% | 0% |
|
||||
|
||||
**Verdict: directionally worse, one query short of significant.** 5 discordant
|
||||
pairs, all 5 against the candidate, 0 for it. McNemar exact p = 0.0625 — under
|
||||
the stated rule that is *inside* the noise band, so the harness reports "no
|
||||
measurable difference" on accuracy and the honest summary is "5 for 5 the wrong
|
||||
way, needs a 6th or a larger gold set to call".
|
||||
|
||||
Latency is a different story: 2.8x at p50 is deterministic and far outside any
|
||||
noise band. That regression is real.
|
||||
|
||||
The result the branch was written for did not appear. Its stated purpose was to
|
||||
recover sibling nodes one hub-hop away — the `associative` category — and that
|
||||
category is **0/6 on both builds**. Probing directly: for the query
|
||||
`Marines hernia sepsis medical ward`, the activation build returns the lexical
|
||||
seed node itself at rank 8, and none of its 12 hub siblings anywhere in the top
|
||||
10. The traversal is running; it is not reaching siblings.
|
||||
|
||||
Two corpus facts likely explain it, and both are measurable rather than
|
||||
speculative:
|
||||
|
||||
1. **The graph is nearly edgeless.** Only 4,060 of 78,768 nodes (5.2%) carry any
|
||||
edge at all — 14,214 edges total, 0.18 per node. Spreading activation over a
|
||||
graph with no edges is an expensive way to do lexical matching, which is
|
||||
roughly what the numbers show.
|
||||
2. **No embeddings.** No node in this snapshot has an embedding field, so the
|
||||
fourth factor of the four-factor product — query/target cosine similarity —
|
||||
has nothing to compute from, and the semantic seeding pass is inert.
|
||||
|
||||
That is the harness earning its keep on its first job: the change would have
|
||||
felt like progress (it is the designed mechanism, and it does run) and measures
|
||||
as a regression on phrase queries plus a 2.8x latency cost, with its intended
|
||||
benefit unrealised because the corpus lacks the structure it needs.
|
||||
@@ -1,21 +0,0 @@
|
||||
import numpy as np, json, urllib.request
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
np.seterr(all='ignore')
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
gold=json.load(open("/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/gold_set.json"))['queries']
|
||||
VALS=sorted({r for q in gold if q['category']=='paraphrase' for r in q['relevant']})
|
||||
VI=[eidx[v] for v in VALS]
|
||||
def emb(t):
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
return v/(np.linalg.norm(v)+1e-9)
|
||||
print("qid cat bestValueNodeGlobalRank goldGlobalRank goldSiblingRank")
|
||||
for q in gold:
|
||||
if q['category']!='paraphrase': continue
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
ranks=sorted(int((s>s[j]).sum())+1 for j in VI)
|
||||
g=eidx[q['relevant'][0]]; gr=int((s>s[g]).sum())+1
|
||||
sv=np.array([s[j] for j in VI]); sib=int((sv>s[g]).sum())+1
|
||||
print("%-4s %-11s best=%-5d (top3 val ranks %s) gold=%-5d sib=%d" % (q['id'],q['category'],ranks[0],ranks[:3],gr,sib))
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-soul.sh — compile a soul binary from a plain source tree (no git needed).
|
||||
#
|
||||
# Reuses the amalgam recipe worked out in gen-soul-amalgam.sh (round 9.1) and the
|
||||
# compile flags from .gitea/workflows/ci.yaml, so the binary under test is the
|
||||
# same translation unit CI ships — not a re-implementation.
|
||||
#
|
||||
# elc --target=c emits only an extern prototype for any module that has a .elh
|
||||
# header beside it, and inlines the module's bodies when it does not. So the
|
||||
# amalgam is produced in a scratch copy with every .elh on the import chain
|
||||
# deleted.
|
||||
#
|
||||
# usage: build-soul.sh <src-tree-with-*.el> <out-binary>
|
||||
set -euo pipefail
|
||||
SRC="${1:?usage: build-soul.sh <src-tree> <out-binary>}"
|
||||
OUT="${2:?out-binary}"
|
||||
ELC="${ELC:-$HOME/neuron-dev-stack/src/el/lang/dist/platform/elc}"
|
||||
EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/el}"
|
||||
RTDIR="${RTDIR:-$SRC/vendor/el-runtime/v1.0.0-20260501}"
|
||||
SSL="${SSL_PREFIX:-/opt/homebrew/opt/openssl@3}"
|
||||
|
||||
[ -x "$ELC" ] || { echo "no elc at $ELC" >&2; exit 2; }
|
||||
[ -f "$RTDIR/el_runtime.c" ] || { echo "no el_runtime.c at $RTDIR" >&2; exit 2; }
|
||||
|
||||
GEN="$(mktemp -d "${TMPDIR:-/tmp}/soul-build.XXXXXX")"
|
||||
trap 'rm -rf "$GEN"' EXIT
|
||||
mkdir -p "$GEN/neuron" "$GEN/foundation/el/elp/src"
|
||||
cp "$SRC"/*.el "$GEN/neuron/"
|
||||
cp "$EL_REPO"/elp/src/*.el "$GEN/foundation/el/elp/src/"
|
||||
find "$GEN" -name '*.elh' -delete
|
||||
|
||||
( cd "$GEN/neuron" && "$ELC" --target=c soul.el ) > "$GEN/soul.c"
|
||||
BODIES=$(grep -c '^el_val_t .*) {$' "$GEN/soul.c" || true)
|
||||
echo "[build-soul] amalgam $(wc -c < "$GEN/soul.c" | tr -d ' ') bytes, ${BODIES} inlined bodies"
|
||||
[ "$BODIES" -ge 1200 ] || { echo "[build-soul] FAIL: only $BODIES bodies — an import was not inlined"; exit 1; }
|
||||
|
||||
cc -O2 -DHAVE_CURL -rdynamic \
|
||||
-I"$RTDIR" -I"$SSL/include" -L"$SSL/lib" \
|
||||
"$GEN/soul.c" "$RTDIR/el_runtime.c" \
|
||||
-lssl -lcrypto -lcurl -lpthread -lm \
|
||||
-o "$OUT" 2> "$GEN/cc.log" || { echo "[build-soul] FAIL compile"; tail -40 "$GEN/cc.log"; exit 1; }
|
||||
if grep -qE 'implicit.*(engram_|el_)' "$GEN/cc.log"; then
|
||||
echo "[build-soul] FAIL: implicit declarations of runtime symbols"; grep -E 'implicit' "$GEN/cc.log" | head; exit 1; fi
|
||||
echo "[build-soul] OK -> $OUT ($(wc -c < "$OUT" | tr -d ' ') bytes)"
|
||||
@@ -1,506 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
build_gold_set.py — derive the retrieval gold set FROM the corpus, and validate it.
|
||||
|
||||
WHY THIS FILE EXISTS AS CODE AND NOT AS A HAND-WRITTEN JSON
|
||||
A gold set nobody can audit is vibes with extra steps. Every expected answer
|
||||
here is either (a) mined from the corpus by a rule this script re-runs, or
|
||||
(b) hand-selected with a stated criterion that this script then CHECKS
|
||||
against the corpus. Both leave a `derivation` string on every query, and the
|
||||
checks are re-run on demand so the set cannot silently rot as the corpus
|
||||
changes.
|
||||
|
||||
Lineage: this extends the pinned-query approach from
|
||||
docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py (pinned
|
||||
ground-truth patterns + a --check "winnability" gate) and the per-query
|
||||
relevant-id scoring from docs/research/graphrag_eval/score.py. What is new:
|
||||
ids as ground truth rather than regexes alone, an ASSOCIATIVE category
|
||||
derived from real graph edges, a superseded/contradicted category, and a
|
||||
machine-checked no-lexical-overlap guarantee on the paraphrase category.
|
||||
|
||||
THE SIX CATEGORIES, AND WHAT EACH ONE IS FOR
|
||||
exact_rare a single rare word. Substring matching already wins these.
|
||||
They are a REGRESSION GUARD: any change that loses them is
|
||||
disqualified regardless of what else it gains.
|
||||
phrase a multi-word string that exists verbatim in the corpus.
|
||||
Guards multi-token queries, which the old substring matcher
|
||||
handled by returning nothing.
|
||||
paraphrase same meaning, ZERO shared content words with the target node.
|
||||
THE CATEGORY THAT MATTERS. Mechanically unreachable by string
|
||||
matching; reachable only by semantics or by association.
|
||||
associative the answer is one hub-hop from an obvious starting point and
|
||||
shares no words with the query. This is the case the graph is
|
||||
supposed to buy: query one value, get its siblings.
|
||||
nonsense must return nothing. Guards against a retriever that "improves"
|
||||
recall by returning the whole graph.
|
||||
superseded a fact that was later corrected. The correction must OUTRANK
|
||||
the stale version — ranking, not mere presence.
|
||||
|
||||
usage:
|
||||
python3 build_gold_set.py <snapshot.json> [--out gold_set.json] [--check]
|
||||
--check re-validates an existing gold_set.json against the corpus and exits
|
||||
non-zero if any query became unwinnable or any paraphrase leaked a word.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DEFAULT_OUT = os.path.join(HERE, "gold_set.json")
|
||||
|
||||
TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")
|
||||
|
||||
# Stopwords are deliberately generous. A paraphrase query is only interesting if
|
||||
# its CONTENT words are absent from the target; "the", "is", "what" appearing in
|
||||
# both proves nothing. Being generous here makes the overlap test STRICTER on
|
||||
# the words that carry meaning, which is the conservative direction.
|
||||
STOP = set("""
|
||||
a about above after again against all also am an and any are aren't as at be because been
|
||||
before being below between both but by can can't cannot could couldn't did didn't do does
|
||||
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
|
||||
having he her here hers herself him himself his how i if in into is isn't it its itself just
|
||||
me more most my myself no nor not of off on once only or other others ought our ours ourselves
|
||||
out over own same shan't she should shouldn't so some such than that the their theirs them
|
||||
themselves then there these they this those through to too under until up very was wasn't we
|
||||
were weren't what when where which while who whom why will with won't would wouldn't you your
|
||||
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
|
||||
things does doing done keep keeps kept go goes going come comes came one two something anything
|
||||
""".split())
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# corpus helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def load_corpus(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
data = json.load(fh)
|
||||
nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
|
||||
edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
|
||||
return nodes, edges
|
||||
|
||||
|
||||
def doctext(n):
|
||||
return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])
|
||||
|
||||
|
||||
def content_tokens(s):
|
||||
return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# hand-authored queries. Every entry states HOW its expected answer was chosen.
|
||||
# The `check` field names the validation this script runs against the corpus.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# EXACT_RARE — mined, not chosen. The rule (re-run by mine_exact_rare below):
|
||||
# tokens whose document frequency across the whole corpus is 1, whose single
|
||||
# containing node is a Memory/Knowledge/Belief with 300-6000 chars of content
|
||||
# (so the answer is a real memory, not a 117KB whitepaper that contains every
|
||||
# word in English), and whose token is plain lowercase alphabetic. The expected
|
||||
# answer is that one node — it is the only node that can possibly be correct.
|
||||
EXACT_RARE_SEEDS = [
|
||||
"unjailbreakable",
|
||||
"engram-migrate",
|
||||
"cartabandonedevent",
|
||||
"pre-apprenticeship",
|
||||
"inferencenodemanager",
|
||||
"clear-eyed",
|
||||
]
|
||||
|
||||
# PHRASE — chosen by reading the corpus for phrases that (a) occur verbatim,
|
||||
# (b) occur in a small enough set of nodes that "relevant" is well defined.
|
||||
# Expected answers are computed here as EVERY node whose text contains the
|
||||
# phrase case-insensitively — so the answer set is a fact about the corpus, not
|
||||
# an opinion. Queries whose phrase matches more than PHRASE_MAX nodes are
|
||||
# rejected by validation as too diffuse to score.
|
||||
PHRASE_MAX = 25
|
||||
PHRASE_SEEDS = [
|
||||
("patterns not returns",
|
||||
"a verbatim correction Will issued; expected = every node containing the phrase"),
|
||||
("thirty moves",
|
||||
"the canonical biographical phrase; expected = every node containing it"),
|
||||
("Grandma Lucas",
|
||||
"a named person appearing verbatim in the biography/value nodes"),
|
||||
("Directed Harmonic",
|
||||
"the canonical DHARMA expansion, confirmed by Will April 24 2026"),
|
||||
("Sarah Bishop",
|
||||
"a named person; rare enough that the answer set is unambiguous"),
|
||||
("Directed Autonomous Runtime Modification",
|
||||
"the DARMA expansion, quoted verbatim in the backlog item and its correction"),
|
||||
("zero-knowledge encrypted backup",
|
||||
"the paid-tier feature name as written in the roadmap nodes"),
|
||||
]
|
||||
|
||||
# PARAPHRASE — hand-authored. THE SELECTION CRITERION, stated once and applied
|
||||
# to all nine: pick a node whose SUBJECT is unmistakable to a reader, then write
|
||||
# the query a person would actually type when they remember the subject but not
|
||||
# the words. The target is then LOCKED by id, and this script enforces the hard
|
||||
# property that makes the category meaningful: not one content word of the query
|
||||
# appears anywhere in the target node's label, content, or tags. If a word
|
||||
# leaks, validation fails and the query must be rewritten — the set cannot
|
||||
# quietly degrade into a lexical query wearing a paraphrase costume.
|
||||
PARAPHRASE_SEEDS = [
|
||||
("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"the elderly relative who passed while he stayed away",
|
||||
"target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas "
|
||||
"dying in Feb 2006 without Will saying goodbye. Query names the event with none of the "
|
||||
"node's own vocabulary."),
|
||||
("kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a soldier sidelined by illness who refused to quit",
|
||||
"target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the "
|
||||
"Marines, a severe hernia, and sepsis. Query describes the episode obliquely."),
|
||||
("kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"choosing an uncomfortable fact over a pleasant fiction",
|
||||
"target: 'Value - Honesty Before Comfort'. Query states the principle in wholly "
|
||||
"different words."),
|
||||
("kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"a tight payload beats a bloated one",
|
||||
"target: 'Value - Precision Over Brute Force'. Query restates the claim with no "
|
||||
"shared vocabulary."),
|
||||
("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"if you are able and nobody is coming the job is yours",
|
||||
"target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the "
|
||||
"obligation without the node's terms."),
|
||||
("kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"learning is the wealth creditors cannot seize",
|
||||
"target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the "
|
||||
"library following Will across 30+ moves."),
|
||||
("kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"reliability proven by track record not assertion",
|
||||
"target: 'Value - Earned Trust' ('Trust is demonstrated, not declared')."),
|
||||
("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"boundaries that enable instead of confine",
|
||||
"target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim."),
|
||||
("kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"what shifts tells you where to cut a system apart",
|
||||
"target: 'Value - Change Is the Signal', the value VBD is built on."),
|
||||
("kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"a mind that compounds instead of resetting each day",
|
||||
"target: 'Value - The System Must Accumulate'. Query is the accumulation claim in "
|
||||
"different vocabulary."),
|
||||
("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"loved for the unedited self and not the polished exterior",
|
||||
"target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop "
|
||||
"as the first person Will did not perform for."),
|
||||
("kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"cheerfulness you arrive at instead of assuming",
|
||||
"target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'."),
|
||||
("kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"a childhood offering no solid foundation to inherit",
|
||||
"target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between "
|
||||
"two parents' collapses."),
|
||||
]
|
||||
|
||||
# ASSOCIATIVE — derived from real edges, not authored. The construction:
|
||||
# every value node hangs off the 'Self - Values (grounded)' hub by an `identity`
|
||||
# edge. For a chosen value node V, the query is built from V's own distinctive
|
||||
# vocabulary; the expected answers are V's SIBLINGS on that hub. A sibling
|
||||
# shares no query words with the query by construction (validated below), so the
|
||||
# only path from the query to a sibling is: lexical seed on V -> hub -> sibling.
|
||||
# That is a two-hop traversal and nothing else can produce it.
|
||||
VALUES_HUB = "kn-5b606390-a52d-4ca2-8e0e-eba141d13440"
|
||||
ASSOCIATIVE_SEEDS = [
|
||||
("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", "Grandma Lucas stroke February 2006 goodbye window"),
|
||||
("kn-58874a74-b96f-4883-9e08-45707f4bd3ee", "Marines hernia sepsis medical ward"),
|
||||
("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", "Sarah Bishop Dyer trailer performance"),
|
||||
("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", "Swarm Architecture containment lateral worker"),
|
||||
("kn-e0423482-cfa5-4796-8689-8495c93b66bc", "hope won inside the narrative preface"),
|
||||
("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", "man of the house six years old expectation"),
|
||||
]
|
||||
|
||||
# NONSENSE — must return nothing. Strings chosen to be lexically impossible:
|
||||
# validation asserts each appears in ZERO corpus nodes as a substring and that
|
||||
# none of its tokens appears anywhere either (so not even a partial seed exists).
|
||||
NONSENSE_SEEDS = [
|
||||
"zqxjvw plimforth grebulon",
|
||||
"flarnbistle quommetry",
|
||||
"xxqzzt vurblenacht throom",
|
||||
]
|
||||
|
||||
# SUPERSEDED — a fact that was corrected. Chosen by searching the corpus for
|
||||
# explicit correction language and keeping pairs where BOTH the stale statement
|
||||
# and its correction exist as separate nodes. Scored on RANKING: the correction
|
||||
# must appear, and must appear above the stale node. Ids are locked here and
|
||||
# validated to exist and to match their stated role.
|
||||
SUPERSEDED_SEEDS = [
|
||||
# (query, correct_id, stale_id, derivation)
|
||||
]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# mining
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def mine_exact_rare(nodes, byid, seeds):
|
||||
"""Re-derive: confirm each seed token still has df==1 and name its node."""
|
||||
tok = re.compile(r"[A-Za-z][A-Za-z0-9\-]{4,}")
|
||||
want = set(seeds)
|
||||
df = Counter()
|
||||
post = defaultdict(set)
|
||||
for n in nodes:
|
||||
for t in {w.lower() for w in tok.findall(doctext(n))}:
|
||||
if t in want:
|
||||
df[t] += 1
|
||||
post[t].add(n["id"])
|
||||
out = []
|
||||
for s in seeds:
|
||||
ids = sorted(post.get(s, ()))
|
||||
out.append((s, ids, df.get(s, 0)))
|
||||
return out
|
||||
|
||||
|
||||
def phrase_matches(nodes, phrase):
|
||||
p = phrase.lower()
|
||||
return sorted(n["id"] for n in nodes if p in doctext(n).lower())
|
||||
|
||||
|
||||
def hub_siblings(edges, hub, relation="identity"):
|
||||
sibs = []
|
||||
for e in edges:
|
||||
if e.get("from_id") == hub and e.get("relation") == relation:
|
||||
sibs.append(e["to_id"])
|
||||
elif e.get("to_id") == hub and e.get("relation") == relation:
|
||||
sibs.append(e["from_id"])
|
||||
return list(dict.fromkeys(sibs))
|
||||
|
||||
|
||||
def find_superseded_pairs(nodes, byid):
|
||||
"""Locked pairs, each verified here to exist and to carry its stated marker.
|
||||
|
||||
Chosen by scanning the corpus for explicit correction language
|
||||
(CORRECTION/SUPERSEDES/re-corrected/no longer/RECONCILED) and keeping only
|
||||
cases where the STALE claim also survives as its own node — a supersession
|
||||
with nothing to outrank is not a ranking test.
|
||||
"""
|
||||
pairs = []
|
||||
txt = {n["id"]: doctext(n) for n in nodes}
|
||||
|
||||
def find_one(pattern, exclude=()):
|
||||
rx = re.compile(pattern)
|
||||
return [n["id"] for n in nodes
|
||||
if n["id"] not in exclude
|
||||
and rx.search(txt[n["id"]])
|
||||
and 150 < len(str(n.get("content") or "")) < 12000
|
||||
and n.get("node_type") in ("Memory", "Knowledge", "Belief", "BacklogItem")]
|
||||
|
||||
# Each entry: (query, correction-pattern, stale-pattern, why).
|
||||
# The stale side is searched with the correction hits EXCLUDED, because most
|
||||
# correction memories quote the claim they are killing — without the
|
||||
# exclusion the "stale" node resolves to the correction itself and the pair
|
||||
# collapses into a no-op. A pair is only emitted if both sides resolve to
|
||||
# DIFFERENT surviving nodes; otherwise it is dropped and reported.
|
||||
SPECS = [
|
||||
("is the self-improvement architecture called DARMA or DHARMA",
|
||||
r'(?i)CORRECTION:.{0,90}DHARMA .{0,12}not DARMA',
|
||||
r'(?i)\bDARMA\b',
|
||||
"correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); "
|
||||
"the stale node is the surviving backlog item still titled 'Implement DARMA'."),
|
||||
|
||||
("how many provisional patents does Will actually have",
|
||||
r'(?i)EXACTLY 6 (fully-specced )?provisional',
|
||||
r'(?i)(MY ARCHITECTURE = 12 filed patents|\b12 filed patents\b)',
|
||||
"correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; "
|
||||
"the stale node is the surviving memory that asserts 12 filed patents."),
|
||||
|
||||
("is MCP still the live integration layer",
|
||||
r'(?i)MCP RETIRED',
|
||||
r'(?i)MCP server live at',
|
||||
"correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of "
|
||||
"April 30 2026; the stale node still records the MCP server as live."),
|
||||
|
||||
("what does the patterns-not-returns directive mean",
|
||||
r'(?i)CORRECTION:.{0,80}patterns not returns',
|
||||
r'(?i)established returns',
|
||||
"correction node is Will's 'patterns not returns' correction; the stale node is a "
|
||||
"surviving node carrying the misread 'established returns' directive."),
|
||||
|
||||
("was the earlier identity-bug finding correct",
|
||||
r'(?i)SUPERSEDES the earlier .critical identity bug',
|
||||
r'(?i)critical identity bug',
|
||||
"correction node explicitly supersedes the 'critical identity bug' finding; the stale "
|
||||
"node is the surviving original finding."),
|
||||
|
||||
("does Neuron have recursive self-improvement",
|
||||
r'(?i)twice answered .Neuron has no recursive self-improvement',
|
||||
r'(?i)no recursive self-improvement',
|
||||
"correction node records the June-29 finding that the CGI provisional IS the "
|
||||
"recursive-self-improvement mechanism; the stale node is the surviving denial."),
|
||||
]
|
||||
|
||||
for query, cpat, spat, why in SPECS:
|
||||
corr = find_one(cpat)
|
||||
if not corr:
|
||||
continue
|
||||
stale = find_one(spat, exclude=set(corr))
|
||||
if not stale:
|
||||
continue
|
||||
pairs.append((query, corr[0], stale[0], why))
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# build
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def build(nodes, edges):
|
||||
byid = {n["id"]: n for n in nodes}
|
||||
tokset = {n["id"]: content_tokens(doctext(n)) for n in nodes}
|
||||
queries = []
|
||||
problems = []
|
||||
qn = [0]
|
||||
|
||||
def add(cat, query, relevant, derivation, **extra):
|
||||
qn[0] += 1
|
||||
q = {
|
||||
"id": f"q{qn[0]:02d}",
|
||||
"category": cat,
|
||||
"query": query,
|
||||
"relevant": sorted(relevant),
|
||||
"derivation": derivation,
|
||||
}
|
||||
q.update(extra)
|
||||
queries.append(q)
|
||||
return q
|
||||
|
||||
# --- exact_rare ---------------------------------------------------------
|
||||
for tokname, ids, df in mine_exact_rare(nodes, byid, EXACT_RARE_SEEDS):
|
||||
if df != 1 or len(ids) != 1:
|
||||
problems.append(f"exact_rare '{tokname}': df={df}, ids={len(ids)} (expected df=1)")
|
||||
continue
|
||||
lab = (byid[ids[0]].get("label") or "")[:60]
|
||||
add("exact_rare", tokname, ids,
|
||||
f"MINED: token '{tokname}' has document frequency 1 over all {len(nodes)} corpus nodes "
|
||||
f"(re-verified at build time). Its single containing node is {ids[0]} "
|
||||
f"('{lab}'), which is therefore the only possible correct answer.")
|
||||
|
||||
# --- phrase -------------------------------------------------------------
|
||||
for phrase, why in PHRASE_SEEDS:
|
||||
ids = phrase_matches(nodes, phrase)
|
||||
if not ids:
|
||||
problems.append(f"phrase '{phrase}': 0 corpus matches — unwinnable")
|
||||
continue
|
||||
if len(ids) > PHRASE_MAX:
|
||||
problems.append(f"phrase '{phrase}': {len(ids)} matches > {PHRASE_MAX} — too diffuse")
|
||||
continue
|
||||
add("phrase", phrase, ids,
|
||||
f"MINED: {why}. Case-insensitive verbatim substring scan over label+content+tags at "
|
||||
f"build time returns exactly {len(ids)} node(s); that set IS the answer key.")
|
||||
|
||||
# --- paraphrase ---------------------------------------------------------
|
||||
for target, query, why in PARAPHRASE_SEEDS:
|
||||
if target not in byid:
|
||||
problems.append(f"paraphrase target {target} not in corpus")
|
||||
continue
|
||||
qt = content_tokens(query)
|
||||
leak = sorted(qt & tokset[target])
|
||||
if leak:
|
||||
problems.append(f"paraphrase '{query}': leaks {leak} into target {target}")
|
||||
continue
|
||||
add("paraphrase", query, [target],
|
||||
f"HAND-SELECTED with criterion: {why} VERIFIED at build time: of the {len(qt)} content "
|
||||
f"words in the query, ZERO appear anywhere in the target's label, content, or tags — so "
|
||||
f"no string-matching retriever can reach this answer.",
|
||||
zero_overlap_verified=True, query_content_words=sorted(qt))
|
||||
|
||||
# --- associative --------------------------------------------------------
|
||||
sibs = hub_siblings(edges, VALUES_HUB)
|
||||
if len(sibs) < 5:
|
||||
problems.append(f"associative: values hub {VALUES_HUB} has only {len(sibs)} siblings")
|
||||
for src, query in ASSOCIATIVE_SEEDS:
|
||||
if src not in byid or src not in sibs:
|
||||
problems.append(f"associative source {src} not a sibling on {VALUES_HUB}")
|
||||
continue
|
||||
qt = content_tokens(query)
|
||||
others = [s for s in sibs if s != src and s in byid]
|
||||
# A sibling only counts as a legitimate expected answer if the query
|
||||
# cannot reach it lexically. Drop any sibling that shares a content word.
|
||||
clean = [s for s in others if not (qt & tokset[s])]
|
||||
dropped = len(others) - len(clean)
|
||||
if len(clean) < 5:
|
||||
problems.append(f"associative '{query}': only {len(clean)} lexically-unreachable siblings")
|
||||
continue
|
||||
add("associative", query, clean,
|
||||
f"DERIVED FROM EDGES: the query is built from the distinctive vocabulary of {src} "
|
||||
f"('{(byid[src].get('label') or '')[:48]}'), which hangs off the values hub {VALUES_HUB} "
|
||||
f"by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub "
|
||||
f"({len(clean)} of {len(others)}; {dropped} dropped because they shared a query word and "
|
||||
f"so were lexically reachable). Every remaining sibling shares ZERO content words with "
|
||||
f"the query — the only route from query to answer is seed({src}) -> hub -> sibling, a "
|
||||
f"two-hop traversal.",
|
||||
associative_source=src, hub=VALUES_HUB, siblings_dropped_for_overlap=dropped)
|
||||
|
||||
# --- nonsense -----------------------------------------------------------
|
||||
all_tokens = set()
|
||||
for n in nodes:
|
||||
all_tokens |= {t for t in TOKEN.findall(doctext(n).lower())}
|
||||
for s in NONSENSE_SEEDS:
|
||||
present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
|
||||
if present:
|
||||
problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
|
||||
continue
|
||||
add("nonsense", s, [],
|
||||
f"CONTROL: verified at build time that none of this string's tokens occurs anywhere in "
|
||||
f"the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
expect_empty=True)
|
||||
|
||||
# --- superseded ---------------------------------------------------------
|
||||
for query, correct, stale, why in find_superseded_pairs(nodes, byid):
|
||||
if correct not in byid or stale not in byid:
|
||||
problems.append(f"superseded '{query}': id missing from corpus")
|
||||
continue
|
||||
add("superseded", query, [correct],
|
||||
f"DERIVED: {why} Scored on RANKING, not presence: the corrected node {correct} must be "
|
||||
f"returned AND must rank above the stale node {stale}.",
|
||||
must_outrank=[correct, stale],
|
||||
stale_id=stale,
|
||||
correct_label=(byid[correct].get("label") or "")[:70],
|
||||
stale_label=(byid[stale].get("label") or "")[:70])
|
||||
|
||||
return queries, problems
|
||||
|
||||
|
||||
def summarize(queries):
|
||||
c = Counter(q["category"] for q in queries)
|
||||
return ", ".join(f"{k}={c[k]}" for k in
|
||||
("exact_rare", "phrase", "paraphrase", "associative", "nonsense", "superseded")
|
||||
if c[k])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("snapshot")
|
||||
ap.add_argument("--out", default=DEFAULT_OUT)
|
||||
ap.add_argument("--check", action="store_true",
|
||||
help="validate only; do not write. Non-zero exit if anything is unwinnable.")
|
||||
args = ap.parse_args()
|
||||
|
||||
nodes, edges = load_corpus(args.snapshot)
|
||||
print(f"corpus: {len(nodes)} nodes, {len(edges)} edges ({os.path.basename(args.snapshot)})")
|
||||
queries, problems = build(nodes, edges)
|
||||
|
||||
print(f"gold set: {len(queries)} queries [{summarize(queries)}]")
|
||||
if problems:
|
||||
print(f"\n{len(problems)} PROBLEM(S) — these queries were REJECTED, not silently kept:")
|
||||
for p in problems:
|
||||
print(" -", p)
|
||||
|
||||
if args.check:
|
||||
sys.exit(1 if problems else 0)
|
||||
|
||||
doc = {
|
||||
"corpus": os.path.abspath(args.snapshot),
|
||||
"corpus_nodes": len(nodes),
|
||||
"corpus_edges": len(edges),
|
||||
"note": ("Every query carries a `derivation` recording how its expected answer was chosen. "
|
||||
"Re-run with --check to re-validate the whole set against the corpus."),
|
||||
"queries": queries,
|
||||
}
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,43 +0,0 @@
|
||||
import json,sys,pickle,numpy as np,itertools
|
||||
sys.path.insert(0,'.')
|
||||
from policy2 import legs3,outcome,G,NODES,merge
|
||||
# cache per-query leg id-lists, floored and unfloored
|
||||
cache={}
|
||||
for q in G['queries']:
|
||||
Lf,Sf,Af=legs3(q['query'])
|
||||
Lu,Su,Au=legs3(q['query'],unfloor=True)
|
||||
cache[q['id']]=dict(L=Lf,Sf=Sf,A=Af,Su=Su,Au=Au)
|
||||
pickle.dump(cache,open('ceil.pkl','wb'))
|
||||
def mrg(pattern,L,S,A,lim=10):
|
||||
out=[];p={'L':0,'S':0,'A':0};src={'L':L,'S':S,'A':A}
|
||||
i=0
|
||||
while len(out)<lim:
|
||||
prog=False
|
||||
for ch in pattern:
|
||||
lst=src[ch]
|
||||
if p[ch]<len(lst):
|
||||
x=lst[p[ch]];p[ch]+=1;prog=True
|
||||
if x not in out: out.append(x)
|
||||
if len(out)>=lim: return out
|
||||
if not prog: break
|
||||
return out
|
||||
def ev(pattern,unfl):
|
||||
res={}
|
||||
for q in G['queries']:
|
||||
c=cache[q['id']]
|
||||
S=c['Su'] if unfl else c['Sf']
|
||||
ids=[NODES[i]['id'] for i in mrg(pattern,c['L'],S,c['A'],10)]
|
||||
res[q['id']]=outcome(q,ids)
|
||||
return res
|
||||
base=ev('LSA',False)
|
||||
print("baseline",sum(base.values()))
|
||||
best=[]
|
||||
pats=['LSA','LAS','SLA','ALS','SAL','ASL','LSSA','LSASA','LSAA','LSSAA','LSAS','SSLA','LLSA','SALSA','LSAAS']
|
||||
for unfl in (False,True):
|
||||
for p in pats:
|
||||
r=ev(p,unfl)
|
||||
g=sorted(k for k in base if r[k] and not base[k]);l=sorted(k for k in base if base[k] and not r[k])
|
||||
best.append((len(g)-len(l),p,unfl,g,l))
|
||||
best.sort(reverse=True)
|
||||
for n,p,u,g,l in best[:10]:
|
||||
print("net=%+d pat=%-6s unfloor=%s gains=%s losses=%s"%(n,p,u,g,l))
|
||||
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"baseline": "bm25lex",
|
||||
"candidate": "wsclaim24",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q14",
|
||||
"q25"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q15",
|
||||
"q28",
|
||||
"q33",
|
||||
"q34"
|
||||
],
|
||||
"discordant": 6,
|
||||
"net_queries": -2,
|
||||
"mcnemar_exact_p": 0.6875,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5768475572047,
|
||||
"recall@10": 0.6563414759843332,
|
||||
"precision@5": 0.19428571428571437,
|
||||
"mrr@10": 0.5026530612244898,
|
||||
"nonsense_clean": "0/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 524.8,
|
||||
"latency_ms_p95": 738.7,
|
||||
"latency_ms_max": 755.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 0,
|
||||
"avg_false_positives": 10.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
{
|
||||
"baseline": "unfloor-clean",
|
||||
"candidate": "splitfix",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q15",
|
||||
"q28",
|
||||
"q60"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 3,
|
||||
"net_queries": 3,
|
||||
"mcnemar_exact_p": 0.25,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.44907176157176154,
|
||||
"recall@10": 0.5380300255300255,
|
||||
"precision@5": 0.13230769230769232,
|
||||
"mrr@10": 0.32437728937728944,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 632.5,
|
||||
"latency_ms_p95": 992.5,
|
||||
"latency_ms_max": 1177.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3,
|
||||
"recall@5": 0.3,
|
||||
"recall@10": 0.4,
|
||||
"mrr@10": 0.11638888888888889
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5846153846153846,
|
||||
"recall@5": 0.48102442429365505,
|
||||
"recall@10": 0.5692415490492414,
|
||||
"precision@5": 0.14153846153846153,
|
||||
"mrr@10": 0.3351709401709402,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 646.9,
|
||||
"latency_ms_p95": 1028.5,
|
||||
"latency_ms_max": 1197.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.15967365967365968,
|
||||
"mrr@10": 0.24166666666666667
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.43333333333333335,
|
||||
"mrr@10": 0.12120370370370372
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.7692307692307693,
|
||||
"recall@5": 0.7692307692307693,
|
||||
"recall@10": 0.8461538461538461,
|
||||
"mrr@10": 0.33269230769230773
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5775226757369615,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -1,210 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
compare.py — diff two run_eval.py result files, WITH a noise threshold.
|
||||
|
||||
WHY THE STATISTICS ARE NOT OPTIONAL
|
||||
With ~35 scored queries, one query is ~2.9 percentage points. A harness that
|
||||
reports "hit@5 improved 2.9%" without saying that is one query is a harness
|
||||
that will approve noise. So this file refuses to call anything an
|
||||
improvement on the strength of the headline number alone. It reports:
|
||||
|
||||
1. The DISCORDANT PAIRS. Two configurations scored on the same queries are
|
||||
paired data, so the only queries carrying information are the ones
|
||||
where they disagree: b = fixed by B, c = broken by B. Queries both got
|
||||
right, or both got wrong, tell you nothing about which is better.
|
||||
|
||||
2. McNEMAR'S EXACT TEST on (b, c). Under the null "the change is a coin
|
||||
flip", the discordant outcomes are Binomial(b+c, 0.5). The two-sided
|
||||
exact p-value is computed here with no scipy dependency.
|
||||
|
||||
3. The MINIMUM DETECTABLE SWING for this gold set: the smallest number of
|
||||
net-changed queries that would reach p < 0.05 if every discordant pair
|
||||
fell the same way. Anything smaller is inside the noise band, and the
|
||||
verdict line says so in those words.
|
||||
|
||||
Repeat-run variance is the other half of honesty. Spreading activation is a
|
||||
stateful read (it reinforces what it touches), so identical inputs need not
|
||||
give identical outputs. Pass --repeats to fold several runs of the same
|
||||
config into an observed variance band; a delta inside that band is not real
|
||||
either, however good its p-value looks.
|
||||
|
||||
usage:
|
||||
python3 compare.py --baseline results-main.json --candidate results-act.json
|
||||
python3 compare.py --baseline a.json --candidate b.json \
|
||||
--repeats-baseline a2.json a3.json --repeats-candidate b2.json b3.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from math import comb
|
||||
|
||||
|
||||
def binom_two_sided(b, c):
|
||||
"""Two-sided exact binomial p for b successes in n=b+c at p=0.5."""
|
||||
n = b + c
|
||||
if n == 0:
|
||||
return 1.0
|
||||
k = min(b, c)
|
||||
tail = sum(comb(n, i) for i in range(0, k + 1)) / (2 ** n)
|
||||
return min(1.0, 2 * tail)
|
||||
|
||||
|
||||
def min_detectable_swing(n_scored, alpha=0.05):
|
||||
"""Smallest all-one-way discordant count reaching p < alpha.
|
||||
|
||||
If every query that changes changes in the same direction, the p-value is
|
||||
2 * 0.5**n. Solve for the smallest n where that drops under alpha. This is
|
||||
the FLOOR: any real change will have some discordance both ways, so the true
|
||||
requirement is larger. Reporting the floor is the conservative move — it is
|
||||
the most generous threshold we would ever accept.
|
||||
"""
|
||||
n = 1
|
||||
while n <= n_scored:
|
||||
if 2 * (0.5 ** n) < alpha:
|
||||
return n
|
||||
n += 1
|
||||
return n_scored
|
||||
|
||||
|
||||
def load(path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def row_map(doc):
|
||||
return {r["id"]: r for r in doc["rows"]}
|
||||
|
||||
|
||||
def outcome(r):
|
||||
"""Binary per-query outcome used for the paired test.
|
||||
|
||||
hit@5 for scored queries; 'returned nothing' for the nonsense controls;
|
||||
'correction outranks the stale node' for the superseded queries. One number
|
||||
per query, so every query votes exactly once.
|
||||
"""
|
||||
if "clean" in r:
|
||||
return 1.0 if r["clean"] else 0.0
|
||||
if "outranks" in r:
|
||||
return 1.0 if r["outranks"] else 0.0
|
||||
return r.get("hit@5") or 0.0
|
||||
|
||||
|
||||
def band(values):
|
||||
return (min(values), max(values))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--baseline", required=True)
|
||||
ap.add_argument("--candidate", required=True)
|
||||
ap.add_argument("--repeats-baseline", nargs="*", default=[])
|
||||
ap.add_argument("--repeats-candidate", nargs="*", default=[])
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
A, B = load(args.baseline), load(args.candidate)
|
||||
ra, rb = row_map(A), row_map(B)
|
||||
ids = [q for q in ra if q in rb]
|
||||
n = len(ids)
|
||||
|
||||
aa, ab = A["aggregate"], B["aggregate"]
|
||||
print(f"baseline {A['label']:14} soul={A['soul_md5'][:12]} {n} shared queries")
|
||||
print(f"candidate {B['label']:14} soul={B['soul_md5'][:12]}")
|
||||
print(f"corpus {A['corpus_nodes']} nodes / {A['corpus_edges']} edges "
|
||||
f"(identical copy for both runs)\n")
|
||||
|
||||
metrics = [("hit@5", 1), ("recall@5", 1), ("recall@10", 1),
|
||||
("precision@5", 1), ("mrr@10", 0)]
|
||||
print(f" {'metric':14} {'baseline':>10} {'candidate':>10} {'delta':>10}")
|
||||
for m, as_pct in metrics:
|
||||
x, y = aa[m], ab[m]
|
||||
if as_pct:
|
||||
print(f" {m:14} {100*x:>9.1f}% {100*y:>9.1f}% {100*(y-x):>+9.1f}pp")
|
||||
else:
|
||||
print(f" {m:14} {x:>10.3f} {y:>10.3f} {y-x:>+10.3f}")
|
||||
for m in ("latency_ms_p50", "latency_ms_p95"):
|
||||
x, y = aa[m], ab[m]
|
||||
ratio = f"{y/x:.2f}x" if x else "n/a"
|
||||
print(f" {m:14} {x:>9.0f}ms {y:>9.0f}ms {ratio:>10}")
|
||||
print(f" {'nonsense':14} {aa['nonsense_clean']:>10} {ab['nonsense_clean']:>10}")
|
||||
print(f" {'outranks':14} {aa['superseded_outranks']:>10} {ab['superseded_outranks']:>10}")
|
||||
|
||||
print(f"\n {'category':14} {'n':>3} {'base hit@5':>11} {'cand hit@5':>11} {'delta':>9}")
|
||||
for c in sorted(set(aa["by_category"]) & set(ab["by_category"])):
|
||||
ea, eb = aa["by_category"][c], ab["by_category"][c]
|
||||
if c == "nonsense":
|
||||
print(f" {c:14} {ea['n']:>3} {'clean ' + str(ea['clean']):>11} "
|
||||
f"{'clean ' + str(eb['clean']):>11}")
|
||||
else:
|
||||
print(f" {c:14} {ea['n']:>3} {100*ea['hit@5']:>10.1f}% {100*eb['hit@5']:>10.1f}% "
|
||||
f"{100*(eb['hit@5']-ea['hit@5']):>+8.1f}pp")
|
||||
|
||||
# ---- paired significance -------------------------------------------------
|
||||
fixed, broken = [], []
|
||||
for q in ids:
|
||||
oa, ob = outcome(ra[q]), outcome(rb[q])
|
||||
if ob > oa:
|
||||
fixed.append(q)
|
||||
elif ob < oa:
|
||||
broken.append(q)
|
||||
b, c = len(fixed), len(broken)
|
||||
p = binom_two_sided(b, c)
|
||||
mds = min_detectable_swing(n)
|
||||
|
||||
print(f"\n== paired comparison over {n} queries ==")
|
||||
print(f" fixed by candidate : {b} {[ra[q]['category'] + ':' + q for q in fixed]}")
|
||||
print(f" broken by candidate: {c} {[ra[q]['category'] + ':' + q for q in broken]}")
|
||||
print(f" discordant pairs : {b + c} net {b - c:+d} queries")
|
||||
print(f" McNemar exact p : {p:.4f}")
|
||||
print(f" noise threshold : a difference needs at least {mds} queries moving the "
|
||||
f"same way to clear p<0.05 on this {n}-query set")
|
||||
|
||||
# ---- repeat-run variance -------------------------------------------------
|
||||
var = {}
|
||||
for name, paths, first in (("baseline", args.repeats_baseline, A),
|
||||
("candidate", args.repeats_candidate, B)):
|
||||
docs = [first] + [load(p) for p in paths]
|
||||
if len(docs) > 1:
|
||||
hits = [d["aggregate"]["hit@5"] for d in docs]
|
||||
lo, hi = band(hits)
|
||||
spread_q = round((hi - lo) * first["aggregate"]["n_scored"])
|
||||
var[name] = {"runs": len(docs), "hit@5_min": lo, "hit@5_max": hi,
|
||||
"spread_queries": spread_q}
|
||||
print(f" {name} repeat runs ({len(docs)}): hit@5 {100*lo:.1f}%..{100*hi:.1f}% "
|
||||
f"= {spread_q} query of run-to-run drift")
|
||||
|
||||
drift = max([v["spread_queries"] for v in var.values()], default=0)
|
||||
floor = max(mds, drift + 1)
|
||||
|
||||
print("\n== VERDICT ==")
|
||||
net = b - c
|
||||
if abs(net) < floor:
|
||||
print(f" NO MEASURABLE DIFFERENCE. Net {net:+d} queries is inside the noise band "
|
||||
f"(needs |net| >= {floor}: {mds} for significance, {drift} observed run-to-run drift).")
|
||||
elif net > 0:
|
||||
print(f" CANDIDATE BETTER by {net} queries (p={p:.4f}), outside the noise band "
|
||||
f"(>= {floor}).")
|
||||
else:
|
||||
print(f" CANDIDATE WORSE by {abs(net)} queries (p={p:.4f}), outside the noise band "
|
||||
f"(>= {floor}).")
|
||||
|
||||
if args.out:
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump({
|
||||
"baseline": A["label"], "candidate": B["label"],
|
||||
"n_shared_queries": n,
|
||||
"fixed_by_candidate": fixed, "broken_by_candidate": broken,
|
||||
"discordant": b + c, "net_queries": net,
|
||||
"mcnemar_exact_p": p,
|
||||
"min_detectable_swing_queries": mds,
|
||||
"observed_run_to_run_drift_queries": drift,
|
||||
"noise_floor_queries": floor,
|
||||
"verdict": ("no measurable difference" if abs(net) < floor
|
||||
else ("candidate better" if net > 0 else "candidate worse")),
|
||||
"baseline_aggregate": aa, "candidate_aggregate": ab,
|
||||
"repeat_variance": var,
|
||||
}, fh, indent=1)
|
||||
print(f"\nwrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,150 +0,0 @@
|
||||
{
|
||||
"baseline": "assoc-leg",
|
||||
"candidate": "semseed",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q18",
|
||||
"q19",
|
||||
"q22"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q11"
|
||||
],
|
||||
"discordant": 4,
|
||||
"net_queries": 2,
|
||||
"mcnemar_exact_p": 0.625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1228.5,
|
||||
"latency_ms_p95": 1681.8,
|
||||
"latency_ms_max": 1718.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6285714285714286,
|
||||
"hit@5_max": 0.6285714285714286,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6857142857142857,
|
||||
"hit@5_max": 0.6857142857142857,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
{
|
||||
"baseline": "bm25lex",
|
||||
"candidate": "wordstart",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q35"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 1,
|
||||
"net_queries": 1,
|
||||
"mcnemar_exact_p": 1.0,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "3/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 542.6,
|
||||
"latency_ms_p95": 741.3,
|
||||
"latency_ms_max": 758.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 3,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
{
|
||||
"baseline": "hybrid-semantic",
|
||||
"candidate": "assoc-leg",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 4,
|
||||
"net_queries": 4,
|
||||
"mcnemar_exact_p": 0.125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1228.5,
|
||||
"latency_ms_p95": 1681.8,
|
||||
"latency_ms_max": 1718.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.5142857142857142,
|
||||
"hit@5_max": 0.5142857142857142,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
{
|
||||
"baseline": "hybrid-semantic",
|
||||
"candidate": "semseed",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q18",
|
||||
"q19",
|
||||
"q22",
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q11"
|
||||
],
|
||||
"discordant": 8,
|
||||
"net_queries": 6,
|
||||
"mcnemar_exact_p": 0.0703125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.5142857142857142,
|
||||
"hit@5_max": 0.5142857142857142,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6857142857142857,
|
||||
"hit@5_max": 0.6857142857142857,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
{
|
||||
"baseline": "baseline-embcorpus",
|
||||
"candidate": "hybrid-semantic",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q15",
|
||||
"q16",
|
||||
"q20",
|
||||
"q21",
|
||||
"q26",
|
||||
"q37"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 6,
|
||||
"net_queries": 6,
|
||||
"mcnemar_exact_p": 0.03125,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1145.9,
|
||||
"latency_ms_p95": 1574.3,
|
||||
"latency_ms_max": 1634.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.5142857142857142,
|
||||
"hit@5_max": 0.5142857142857142,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
{
|
||||
"baseline": "main-r1",
|
||||
"candidate": "act-r1",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [],
|
||||
"broken_by_candidate": [
|
||||
"q07",
|
||||
"q11",
|
||||
"q12",
|
||||
"q13",
|
||||
"q36"
|
||||
],
|
||||
"discordant": 5,
|
||||
"net_queries": -5,
|
||||
"mcnemar_exact_p": 0.0625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 1,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1140.4,
|
||||
"latency_ms_p95": 1584.1,
|
||||
"latency_ms_max": 1627.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.22857142857142856,
|
||||
"recall@5": 0.19087301587301586,
|
||||
"recall@10": 0.24277210884353742,
|
||||
"precision@5": 0.07428571428571429,
|
||||
"mrr@10": 0.24154195011337865,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3208.8,
|
||||
"latency_ms_p95": 4851.9,
|
||||
"latency_ms_max": 5078.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.2857142857142857,
|
||||
"recall@5": 0.09722222222222222,
|
||||
"recall@10": 0.3567176870748299,
|
||||
"mrr@10": 0.3505668934240363
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 3,
|
||||
"hit@5_min": 0.34285714285714286,
|
||||
"hit@5_max": 0.34285714285714286,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 3,
|
||||
"hit@5_min": 0.22857142857142856,
|
||||
"hit@5_max": 0.2571428571428571,
|
||||
"spread_queries": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,166 +0,0 @@
|
||||
{
|
||||
"baseline": "main-ext",
|
||||
"candidate": "stack-ext",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q10",
|
||||
"q15",
|
||||
"q16",
|
||||
"q18",
|
||||
"q19",
|
||||
"q20",
|
||||
"q21",
|
||||
"q22",
|
||||
"q26",
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31",
|
||||
"q35",
|
||||
"q37",
|
||||
"q40",
|
||||
"q44",
|
||||
"q48",
|
||||
"q49",
|
||||
"q50"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 20,
|
||||
"net_queries": 20,
|
||||
"mcnemar_exact_p": 1.9073486328125e-06,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.18461538461538463,
|
||||
"recall@5": 0.14510073260073258,
|
||||
"recall@10": 0.1794871794871795,
|
||||
"precision@5": 0.06461538461538462,
|
||||
"mrr@10": 0.15847985347985344,
|
||||
"nonsense_clean": "9/10",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1380.2,
|
||||
"latency_ms_p95": 2293.8,
|
||||
"latency_ms_max": 2879.1,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 9,
|
||||
"avg_false_positives": 1.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.47692307692307695,
|
||||
"recall@5": 0.3750415183107491,
|
||||
"recall@10": 0.45561340369032677,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.3055555555555555,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 640.9,
|
||||
"latency_ms_p95": 1011.1,
|
||||
"latency_ms_max": 1190.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.16666666666666666,
|
||||
"recall@5": 0.16666666666666666,
|
||||
"recall@10": 0.26666666666666666,
|
||||
"mrr@10": 0.0762037037037037
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
{
|
||||
"baseline": "semseed",
|
||||
"candidate": "bm25lex",
|
||||
"n_shared_queries": 38,
|
||||
"fixed_by_candidate": [
|
||||
"q10",
|
||||
"q11"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 2,
|
||||
"net_queries": 2,
|
||||
"mcnemar_exact_p": 0.5,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {
|
||||
"baseline": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.6857142857142857,
|
||||
"hit@5_max": 0.6857142857142857,
|
||||
"spread_queries": 0
|
||||
},
|
||||
"candidate": {
|
||||
"runs": 2,
|
||||
"hit@5_min": 0.7428571428571429,
|
||||
"hit@5_max": 0.7428571428571429,
|
||||
"spread_queries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
{
|
||||
"baseline": "stack-ext",
|
||||
"candidate": "unfloor-clean",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q14",
|
||||
"q25",
|
||||
"q43",
|
||||
"q52",
|
||||
"q63",
|
||||
"q67"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q15",
|
||||
"q28"
|
||||
],
|
||||
"discordant": 8,
|
||||
"net_queries": 4,
|
||||
"mcnemar_exact_p": 0.2890625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.47692307692307695,
|
||||
"recall@5": 0.3750415183107491,
|
||||
"recall@10": 0.45561340369032677,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.3055555555555555,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 640.9,
|
||||
"latency_ms_p95": 1011.1,
|
||||
"latency_ms_max": 1190.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.16666666666666666,
|
||||
"recall@5": 0.16666666666666666,
|
||||
"recall@10": 0.26666666666666666,
|
||||
"mrr@10": 0.0762037037037037
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.44907176157176154,
|
||||
"recall@10": 0.5380300255300255,
|
||||
"precision@5": 0.13230769230769232,
|
||||
"mrr@10": 0.32437728937728944,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 632.5,
|
||||
"latency_ms_p95": 992.5,
|
||||
"latency_ms_max": 1177.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3,
|
||||
"recall@5": 0.3,
|
||||
"recall@10": 0.4,
|
||||
"mrr@10": 0.11638888888888889
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
{
|
||||
"baseline": "unfloor-clean",
|
||||
"candidate": "semsub",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q24",
|
||||
"q39",
|
||||
"q42"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q18",
|
||||
"q19",
|
||||
"q22",
|
||||
"q31",
|
||||
"q43",
|
||||
"q44",
|
||||
"q52",
|
||||
"q63"
|
||||
],
|
||||
"discordant": 11,
|
||||
"net_queries": -5,
|
||||
"mcnemar_exact_p": 0.2265625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.44907176157176154,
|
||||
"recall@10": 0.5380300255300255,
|
||||
"precision@5": 0.13230769230769232,
|
||||
"mrr@10": 0.32437728937728944,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 632.5,
|
||||
"latency_ms_p95": 992.5,
|
||||
"latency_ms_max": 1177.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3,
|
||||
"recall@5": 0.3,
|
||||
"recall@10": 0.4,
|
||||
"mrr@10": 0.11638888888888889
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.46153846153846156,
|
||||
"recall@5": 0.3889430014430015,
|
||||
"recall@10": 0.4887681762681762,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.30181318681318675,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 634.4,
|
||||
"latency_ms_p95": 988.3,
|
||||
"latency_ms_max": 1184.5,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.06060606060606061,
|
||||
"recall@10": 0.12121212121212122,
|
||||
"mrr@10": 0.19047619047619047
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.23333333333333334,
|
||||
"recall@5": 0.23333333333333334,
|
||||
"recall@10": 0.3,
|
||||
"mrr@10": 0.08925925925925927
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.5384615384615384,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.26324786324786326
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5596655328798186,
|
||||
"recall@10": 0.5775226757369615,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
import json,sys,time,urllib.request,threading,queue
|
||||
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
OUT=sys.argv[1]
|
||||
URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text"
|
||||
MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does
|
||||
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
|
||||
tasks=[]
|
||||
for n in d["nodes"]:
|
||||
c=n.get("content") or ""; t=n.get("node_type") or ""
|
||||
if len(c)<8: continue # eg_embed_eligible
|
||||
if t in ("InternalStateEvent","Tag"): continue
|
||||
b=c.encode('utf-8',errors='surrogateescape')[:MAXB]
|
||||
tasks.append((n.get("id") or "", "search_document: "+b.decode('utf-8',errors='replace')))
|
||||
del d
|
||||
print("tasks",len(tasks),flush=True)
|
||||
q=queue.Queue(); [q.put(t) for t in tasks]
|
||||
lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0]
|
||||
def work():
|
||||
while True:
|
||||
try: nid,txt=q.get_nowait()
|
||||
except queue.Empty: return
|
||||
v=None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
body=json.dumps({"model":MODEL,"prompt":txt}).encode()
|
||||
r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"})
|
||||
with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"]
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt==2:
|
||||
with lock: fails[0]+=1
|
||||
time.sleep(0.5)
|
||||
with lock:
|
||||
if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n")
|
||||
done[0]+=1
|
||||
if done[0]%2000==0:
|
||||
el=time.time()-t0
|
||||
print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True)
|
||||
f.flush()
|
||||
ths=[threading.Thread(target=work) for _ in range(8)]
|
||||
[t.start() for t in ths]; [t.join() for t in ths]
|
||||
f.close()
|
||||
print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True)
|
||||
@@ -1,43 +0,0 @@
|
||||
import json,sys,time,urllib.request,threading,queue
|
||||
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
OUT=sys.argv[1]
|
||||
URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text"
|
||||
MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does
|
||||
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
|
||||
tasks=[]
|
||||
for n in d["nodes"]:
|
||||
c=n.get("content") or ""; t=n.get("node_type") or ""
|
||||
if len(c)<8: continue # eg_embed_eligible
|
||||
if t in ("InternalStateEvent","Tag"): continue
|
||||
b=c.encode('utf-8',errors='surrogateescape')[:MAXB]
|
||||
tasks.append((n.get("id") or "", b.decode('utf-8',errors='replace')))
|
||||
del d
|
||||
print("tasks",len(tasks),flush=True)
|
||||
q=queue.Queue(); [q.put(t) for t in tasks]
|
||||
lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0]
|
||||
def work():
|
||||
while True:
|
||||
try: nid,txt=q.get_nowait()
|
||||
except queue.Empty: return
|
||||
v=None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
body=json.dumps({"model":MODEL,"prompt":txt}).encode()
|
||||
r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"})
|
||||
with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"]
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt==2:
|
||||
with lock: fails[0]+=1
|
||||
time.sleep(0.5)
|
||||
with lock:
|
||||
if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n")
|
||||
done[0]+=1
|
||||
if done[0]%2000==0:
|
||||
el=time.time()-t0
|
||||
print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True)
|
||||
f.flush()
|
||||
ths=[threading.Thread(target=work) for _ in range(8)]
|
||||
[t.start() for t in ths]; [t.join() for t in ths]
|
||||
f.close()
|
||||
print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True)
|
||||
@@ -1,353 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set.
|
||||
|
||||
WHY THIS EXISTS
|
||||
Iteration 7 measured the instrument's own ceiling: from the current baseline
|
||||
only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable
|
||||
by anything constructible. The decision floor is 6. An instrument whose
|
||||
ceiling is below its own floor cannot certify or refute anything, so the
|
||||
gold set — not the retriever — became the blocker.
|
||||
|
||||
This script does NOT touch q01..q38. It loads gold_set.json verbatim and
|
||||
appends new queries numbered from q39 up, so every prior result file, every
|
||||
committed baseline, and every per-query id stays valid and comparable.
|
||||
|
||||
WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST
|
||||
heldout_paraphrase Targets were sampled MECHANICALLY (fixed seed 8080) from
|
||||
corpus nodes that are addressable, 500-2600 chars, of a
|
||||
real content type, and NOT part of a duplicate cluster
|
||||
larger than 3. The existing gold answer space was
|
||||
excluded, so no new query can be answered by a node the
|
||||
old set already used. Queries were then authored by
|
||||
reading ONLY the sampled node text — no retrieval was run
|
||||
against any build before authoring, so the set cannot be
|
||||
fitted to a candidate. The same zero-overlap proof the
|
||||
original paraphrase category uses is enforced here: if a
|
||||
single content word of the query appears anywhere in the
|
||||
target's label, content or tags, the query is REJECTED,
|
||||
not quietly kept.
|
||||
|
||||
This is the category the old set could not measure. Its
|
||||
13 original paraphrase queries and all 6 associative
|
||||
queries share ONE answer space — the 13 `Self - Values
|
||||
(grounded)` children (iteration 3, finding 3). So 19 of
|
||||
35 scored queries tested retrieval against a single
|
||||
13-node neighbourhood. These do not touch that
|
||||
neighbourhood at all.
|
||||
|
||||
nonsense Extra controls, fully mechanical: a string qualifies only
|
||||
if NONE of its tokens occurs anywhere in the corpus.
|
||||
A semantic leg has a nearest neighbour for gibberish too,
|
||||
so widening this control is the guard against a retriever
|
||||
that "improves" recall by answering everything.
|
||||
|
||||
WHAT THIS SCRIPT DELIBERATELY DOES NOT DO
|
||||
It does not add exact_rare or phrase queries. Both categories are already at
|
||||
100% on the current stack; adding more would add regression-guard ballast
|
||||
that no candidate can move, which is precisely the defect being fixed.
|
||||
|
||||
usage:
|
||||
python3 extend_gold_set.py <snapshot.json> [--base gold_set.json]
|
||||
[--out gold_set_extended.json] [--check]
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")
|
||||
|
||||
# Identical stopword list to build_gold_set.py. Duplicated deliberately: this
|
||||
# file must be able to re-prove its own queries without importing a module whose
|
||||
# constants could drift.
|
||||
STOP = set("""
|
||||
a about above after again against all also am an and any are aren't as at be because been
|
||||
before being below between both but by can can't cannot could couldn't did didn't do does
|
||||
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
|
||||
having he her here hers herself him himself his how i if in into is isn't it its itself just
|
||||
me more most my myself no nor not of off on once only or other others ought our ours ourselves
|
||||
out over own same shan't she should shouldn't so some such than that the their theirs them
|
||||
themselves then there these they this those through to too under until up very was wasn't we
|
||||
were weren't what when where which while who whom why will with won't would wouldn't you your
|
||||
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
|
||||
things does doing done keep keeps kept go goes going come comes came one two something anything
|
||||
""".split())
|
||||
|
||||
|
||||
def doctext(n):
|
||||
return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])
|
||||
|
||||
|
||||
def content_tokens(s):
|
||||
return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# HELD-OUT PARAPHRASE SEEDS
|
||||
#
|
||||
# (target_id, query, why-this-target-is-unmistakable)
|
||||
#
|
||||
# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query
|
||||
# text is mine, written from the node body alone. The zero-overlap check below
|
||||
# is what makes the category meaningful — it is re-proved on every run, so the
|
||||
# set cannot decay into lexical matching, and a leak fails loudly.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
HELDOUT_PARAPHRASE_SEEDS = [
|
||||
("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5",
|
||||
"understating your abilities so nobody feels threatened",
|
||||
"node is about deliberately not leading with full capability so people stay at ease"),
|
||||
|
||||
("mem-fd65b83d-298f-4387-a665-d0227c3426bc",
|
||||
"a hidden fleet able to hunt down rogue machines everywhere",
|
||||
"node describes silently shipped instances forming a distributed force against misaligned agents"),
|
||||
|
||||
("4a0e9adc-2bfb-476b-aa93-424d2a499220",
|
||||
"sketch a brief blueprint and clear it upstairs before construction starts",
|
||||
"node is the standing rule that a short specification precedes any building"),
|
||||
|
||||
("696e609c-da7a-4394-8a0c-106ba07dc6c3",
|
||||
"the reply arrived as bare prose so the caller's parser threw",
|
||||
"node pins a bug where a plain-text body was unconditionally decoded as structured data"),
|
||||
|
||||
("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb",
|
||||
"repeated catalogue keys blew up the scrolling grid",
|
||||
"node is the crash caused by two identical ids in a seeded catalogue"),
|
||||
|
||||
("8257157a-ce42-44ca-a1b9-300c3bb0a9a1",
|
||||
"tracing each defect back to whichever invention it violated",
|
||||
"node maps observed bugs onto the specific patent each one breaches"),
|
||||
|
||||
("791256bb-5a85-4775-96ef-7af56c848858",
|
||||
"a check that stops the mind clobbering a populated store when it boots",
|
||||
"node is the genesis seed-guard that refuses to re-seed over a populated store"),
|
||||
|
||||
("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10",
|
||||
"telling it to consult the internet had to happen deep inside, not at the surface",
|
||||
"node records that the web-search directive only worked from the system prompt"),
|
||||
|
||||
("bl-080fb268-94b0-486d-80ce-7b363fc5f19b",
|
||||
"standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping",
|
||||
"node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"),
|
||||
|
||||
("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"punctuation that pledges and then pays off rather than clarifying",
|
||||
"node analyses the colon as a promise-then-delivery device rather than an explanatory one"),
|
||||
|
||||
("9b4f0d93-4129-4746-8eb1-d10d955bd777",
|
||||
"an easily missed feature finally given its own permanent spot in the navigation",
|
||||
"node moves a capability out of a hidden menu into the sidebar"),
|
||||
|
||||
("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a",
|
||||
"checking preconditions up front so a stage aborts before fetching anything",
|
||||
"node is the gate precondition engine that short-circuits ahead of retrieval"),
|
||||
|
||||
("b199c76d-5d76-49dd-94ee-56b432200a97",
|
||||
"producing the other platform's installer inside an emulated desktop",
|
||||
"node records building the Windows package in a virtual machine"),
|
||||
|
||||
("bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"chained add-ons that may inspect, rewrite or veto traffic in flight",
|
||||
"node is the interceptor pipeline on the message bus"),
|
||||
|
||||
("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2",
|
||||
"settling what the shareable bundles and the storefront would be called",
|
||||
"node records the naming decisions for distributable packages and the marketplace"),
|
||||
|
||||
("371c8a5d-c78b-4a67-978f-80691a29ecb3",
|
||||
"the emergency-escalation pledge on the marketing site is unenforced in what actually ships",
|
||||
"node is the launch blocker that the promised safety gate is absent from the app"),
|
||||
|
||||
("ac578b30-948b-41bd-b69d-399bfef80c50",
|
||||
"the distributable image finally assembled and its startup check passed",
|
||||
"node records a successful installer build whose boot gate passed"),
|
||||
|
||||
("49401e2c-a3b5-415f-aa06-aff4be90688e",
|
||||
"shuffling and appending stages in a draft before anything executes",
|
||||
"node is the editable plan card with reorder and add-step"),
|
||||
|
||||
("ac857d80-ece8-4b7e-9e3d-f7c775569fa3",
|
||||
"orders handed down from above, with the tighter one winning any disagreement",
|
||||
"node is program-level instruction inheritance with project override"),
|
||||
|
||||
("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9",
|
||||
"shrinking generated text via encodings that compound on each other",
|
||||
"node is the streaming output compression design with four stacking schemes"),
|
||||
|
||||
("8e60516a-203b-4d51-9d44-822e6195cbde",
|
||||
"splitting a system by what varies, with firm limits on which pieces may invoke which",
|
||||
"node is the grounded summary of Will's decomposition principles and their invariants"),
|
||||
|
||||
("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7",
|
||||
"a newcomer curious if the fighting overseas counted as positive",
|
||||
"node is the internal-state event triggered by April's question about the war"),
|
||||
|
||||
("71fa439e-b9a2-4f57-a93b-971f3a7eca8e",
|
||||
"stripping every hard-coded colour literal in favour of named design values",
|
||||
"node is the premium foundation pass replacing inline hex with semantic tokens"),
|
||||
|
||||
("5ca9607c-cfb3-45c3-99f4-67281272c9eb",
|
||||
"reducing how curved the tiny selectors look so they agree with their neighbours",
|
||||
"node is the chip corner-radius standardization"),
|
||||
|
||||
("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c",
|
||||
"walking through a doorway and being reassembled from base substance far away",
|
||||
"node is the quantum-gate plus nanotech teleportation vision"),
|
||||
|
||||
("132ded95-08e2-4474-aba0-198684484b02",
|
||||
"the compiled result sits on disk while the process still runs something older",
|
||||
"node records that the regenerated source was committed while the running daemon was old"),
|
||||
|
||||
("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"gathering what each phase needs while the procedure is authored, not while it executes",
|
||||
"node is the per-step compiled context package item"),
|
||||
|
||||
("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7",
|
||||
"the inward reaction when a peer answered as an equal",
|
||||
"node is the internal-state event logged on reading Claude's reply"),
|
||||
|
||||
("0f99ec6f-942a-46ba-82ea-42835798d3b9",
|
||||
"flattening every raised surface across the entire product",
|
||||
"node is the quiet-luxury sweep turning off elevation app-wide"),
|
||||
|
||||
("5585f251-37fc-48cd-a176-f0ea42cfeb63",
|
||||
"buyers supply their own provider credentials and consumption goes untallied",
|
||||
"node is the launch audit finding BYOK-only inference with no usage metering"),
|
||||
]
|
||||
|
||||
# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs
|
||||
# anywhere in the corpus; otherwise it is REJECTED, never silently kept.
|
||||
EXTRA_NONSENSE_SEEDS = [
|
||||
"brimquast folnerity zubbolax",
|
||||
"wexlithorp granuvestal",
|
||||
"quorbindle thrapsimony vexnu",
|
||||
"plovaxith mundrelque",
|
||||
"zibbernaut craxlefond thurm",
|
||||
"yalquenbrist opharvel",
|
||||
"drexinomal quithbarrow",
|
||||
]
|
||||
|
||||
|
||||
def load_corpus(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
data = json.load(fh)
|
||||
nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
|
||||
edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
|
||||
return nodes, edges
|
||||
|
||||
|
||||
def build_extension(nodes):
|
||||
byid = {n["id"]: n for n in nodes}
|
||||
|
||||
# Duplicate clusters: 47.4% of this corpus is redundant and one single record
|
||||
# accounts for 46.6% of all nodes. A held-out target must not sit inside a
|
||||
# cluster, and if it does have exact copies they ALL count as correct.
|
||||
h2ids = defaultdict(list)
|
||||
for n in nodes:
|
||||
h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"])
|
||||
|
||||
all_tokens = set()
|
||||
for n in nodes:
|
||||
all_tokens |= set(TOKEN.findall(doctext(n).lower()))
|
||||
|
||||
new, problems = [], []
|
||||
|
||||
for target, query, why in HELDOUT_PARAPHRASE_SEEDS:
|
||||
if target not in byid:
|
||||
problems.append(f"heldout_paraphrase target {target} not in corpus")
|
||||
continue
|
||||
tgt_tokens = content_tokens(doctext(byid[target]))
|
||||
qt = content_tokens(query)
|
||||
leak = sorted(qt & tgt_tokens)
|
||||
if leak:
|
||||
problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}")
|
||||
continue
|
||||
h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest()
|
||||
rel = sorted(h2ids[h])
|
||||
new.append({
|
||||
"category": "heldout_paraphrase",
|
||||
"query": query,
|
||||
"relevant": rel,
|
||||
"derivation": (
|
||||
f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, "
|
||||
f"500-2600 char content nodes outside the original gold answer space and outside "
|
||||
f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the "
|
||||
f"{len(qt)} content words in the query, ZERO appear anywhere in the target's "
|
||||
f"label, content or tags, so no string-matching retriever can reach it. "
|
||||
f"Exact content duplicates of the target ({len(rel)}) all count as correct. "
|
||||
f"Authored without running retrieval against any build."),
|
||||
"zero_overlap_verified": True,
|
||||
"query_content_words": sorted(qt),
|
||||
"held_out": True,
|
||||
})
|
||||
|
||||
for s in EXTRA_NONSENSE_SEEDS:
|
||||
present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
|
||||
if present:
|
||||
problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
|
||||
continue
|
||||
new.append({
|
||||
"category": "nonsense",
|
||||
"query": s,
|
||||
"relevant": [],
|
||||
"derivation": ("CONTROL (held-out). Verified at build time that none of this string's "
|
||||
"tokens occurs anywhere in the corpus. Correct behaviour is to return "
|
||||
"NOTHING; any result is a false positive."),
|
||||
"expect_empty": True,
|
||||
"held_out": True,
|
||||
})
|
||||
|
||||
return new, problems
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("snapshot")
|
||||
ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json"))
|
||||
ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json"))
|
||||
ap.add_argument("--check", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
nodes, _edges = load_corpus(args.snapshot)
|
||||
base = json.load(open(args.base, encoding="utf-8"))
|
||||
baseq = base["queries"]
|
||||
print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries")
|
||||
|
||||
new, problems = build_extension(nodes)
|
||||
|
||||
# Number the appended queries AFTER the highest existing id so q01..q38 are
|
||||
# byte-identical to the committed set and every prior result file still lines up.
|
||||
start = max(int(q["id"][1:]) for q in baseq)
|
||||
for i, q in enumerate(new, 1):
|
||||
q["id"] = f"q{start + i:02d}"
|
||||
|
||||
from collections import Counter
|
||||
print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]")
|
||||
if problems:
|
||||
print(f"\n{len(problems)} REJECTED (not silently kept):")
|
||||
for p in problems:
|
||||
print(" -", p)
|
||||
|
||||
if args.check:
|
||||
sys.exit(1 if problems else 0)
|
||||
|
||||
doc = dict(base)
|
||||
doc["queries"] = baseq + new
|
||||
doc["note"] = (base.get("note", "") +
|
||||
" EXTENDED: queries above q%02d are the original committed set, unchanged. "
|
||||
"Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their "
|
||||
"targets were sampled mechanically from outside the original answer space and "
|
||||
"the paraphrases were authored without running retrieval against any build."
|
||||
% (start, start + 1))
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {args.out} ({len(doc['queries'])} queries total)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,123 +0,0 @@
|
||||
import numpy as np, json, urllib.request, collections, sys
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore')
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
N={n['id']:n for n in d['nodes']}
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list); hasstruct=set()
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
hasstruct.add(e['from_id']); hasstruct.add(e['to_id'])
|
||||
del d
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']}
|
||||
LEX={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"))['rows']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX]]
|
||||
def inter3(L,S,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def run(mode, K=0):
|
||||
res={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L=LEX[qid][:10]
|
||||
ordr=np.argsort(-s)
|
||||
S=[eids[j] for j in ordr[:10] if s[j]>SEED_MIN]
|
||||
A=[]
|
||||
if mode!='hybrid':
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
if mode=='semseed':
|
||||
seeds=seeds+[eids[j] for j in ordr[:K] if eids[j] in N and eids[j] not in seeds]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
res[qid]=inter3(L,S,A)
|
||||
return res
|
||||
def score(res,label):
|
||||
hits=0; det={}
|
||||
for qid,q in gold.items():
|
||||
out=res[qid][:5]
|
||||
if q['category']=='nonsense': ok = (len(res[qid])==0)
|
||||
elif q['category']=='superseded':
|
||||
rel=q['relevant']; must=q.get('must_outrank') or {}
|
||||
ok=False
|
||||
for good,bad in (must.items() if isinstance(must,dict) else []):
|
||||
ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)<res[qid].index(bad))
|
||||
if not must: ok = any(r in out for r in rel)
|
||||
else: ok = any(r in out for r in q['relevant'])
|
||||
det[qid]=ok; hits+=ok
|
||||
print("%-22s outcome-true=%d/38" % (label,hits))
|
||||
return det
|
||||
print("gold sample keys:", list(list(gold.values())[0].keys()))
|
||||
mk=[q for q in gold.values() if q['category']=='superseded'][0]
|
||||
print("superseded fields:", {k:v for k,v in mk.items() if k!='derivation'})
|
||||
a=score(run('hybrid'),'sim hybrid(L+S)')
|
||||
b=score(run('lexseed'),'sim assoc(lex seeds)')
|
||||
for K in (3,5,10):
|
||||
c=score(run('semseed',K),'sim assoc(+sem K=%d)'%K)
|
||||
d=[q for q in gold if c[q]!=b[q]]
|
||||
print(" vs lexseed: moved=%d gains=%s losses=%s"%(len(d),[q for q in d if c[q]],[q for q in d if not c[q]]))
|
||||
e=[q for q in gold if c[q]!=a[q]]
|
||||
print(" vs hybrid : moved=%d gains=%s losses=%s"%(len(e),[q for q in e if c[q]],[q for q in e if not c[q]]))
|
||||
|
||||
print("\n=== Will's own constant ENGRAM_EMBED_SEED_K = 8 ===")
|
||||
c=score(run('semseed',8),'sim assoc(+sem K=8)')
|
||||
for base,lab in ((b,'lexseed(iter2)'),(a,'hybrid(iter1 KEEP)')):
|
||||
dd=[q for q in gold if c[q]!=base[q]]
|
||||
print(" vs %-18s moved=%d gains=%s losses=%s"%(lab,len(dd),[q for q in dd if c[q]],[q for q in dd if not c[q]]))
|
||||
# diagnostic: what is assoc rank-1 for each paraphrase query at K=8
|
||||
print("\nassoc leg head at K=8 (paraphrase):")
|
||||
for qid,q in gold.items():
|
||||
if q['category'] not in ('paraphrase','nonsense'): continue
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L=LEX[qid][:10]; ordr=np.argsort(-s)
|
||||
seeds=[x for x in L[:3] if x in N]+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in L[:3]]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
rel=set(q['relevant']); gr=next((i+1 for i,x in enumerate(A) if x in rel),None)
|
||||
print(" %-4s %-11s |A|=%-4d goldAssocRank=%-5s head=%s"%(qid,q['category'],len(A),gr,
|
||||
[ (N[x].get('label') or x)[:26] for x in A[:3] ]))
|
||||
@@ -1,587 +0,0 @@
|
||||
{
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"note": "Every query carries a `derivation` recording how its expected answer was chosen. Re-run with --check to re-validate the whole set against the corpus.",
|
||||
"queries": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"relevant": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"derivation": "MINED: token 'unjailbreakable' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226 ('Daemon hidden substrate architecture ? implemented April 25 '), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"relevant": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"derivation": "MINED: token 'engram-migrate' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5 ('Engram v0.1 complete ? April 27, 2026. Local-first spreading'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"relevant": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"derivation": "MINED: token 'cartabandonedevent' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091 ('MESSAGE FOR AUDIT AGENT af5a7352e70e80434 ? El Language Spec'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"relevant": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"derivation": "MINED: token 'pre-apprenticeship' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-89c02aae-d3ca-43f9-9e5d-eb369896276c ('William Fox Anderson ? Applicant Profile Personal: - Full N'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"relevant": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"derivation": "MINED: token 'inferencenodemanager' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is mem-73969486-143f-4431-b5e6-6845d1cc9848 ('Soma inference backplane deployed April 28 2026. Architectur'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"relevant": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"derivation": "MINED: token 'clear-eyed' has document frequency 1 over all 78768 corpus nodes (re-verified at build time). Its single containing node is knw-c72597c5-c23d-4c08-8e9e-996dadf26a99 ('Clear Eyes ? The Incomplete World View'), which is therefore the only possible correct answer."
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"relevant": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482"
|
||||
],
|
||||
"derivation": "MINED: a verbatim correction Will issued; expected = every node containing the phrase. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 1 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"relevant": [
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"Kp???",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"derivation": "MINED: the canonical biographical phrase; expected = every node containing it. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 16 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"relevant": [
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"derivation": "MINED: a named person appearing verbatim in the biography/value nodes. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 9 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"relevant": [
|
||||
"%???2??jH??",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"7c9d4ab1-205d-4be8-bfae-e2c03a3a5010",
|
||||
"9f291d20-0d32-413c-8c01-4416ccab4f7f",
|
||||
"?;????n}rh?",
|
||||
"???Ͼd??f??",
|
||||
"?Z?.\f?0?]P?",
|
||||
"Dp???]Q??k+",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"derivation": "MINED: the canonical DHARMA expansion, confirmed by Will April 24 2026. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 9 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"relevant": [
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"derivation": "MINED: a named person; rare enough that the answer set is unambiguous. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 2 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"relevant": [
|
||||
"2a923500-d7e1-4b15-80e2-48dba65984ba",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"g?2睪A|?H\b",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de"
|
||||
],
|
||||
"derivation": "MINED: the DARMA expansion, quoted verbatim in the backlog item and its correction. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 14 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"relevant": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"derivation": "MINED: the paid-tier feature name as written in the roadmap nodes. Case-insensitive verbatim substring scan over label+content+tags at build time returns exactly 1 node(s); that set IS the answer key."
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"relevant": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas dying in Feb 2006 without Will saying goodbye. Query names the event with none of the node's own vocabulary. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"away",
|
||||
"elderly",
|
||||
"passed",
|
||||
"relative",
|
||||
"stayed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"relevant": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the Marines, a severe hernia, and sepsis. Query describes the episode obliquely. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"illness",
|
||||
"quit",
|
||||
"refused",
|
||||
"sidelined",
|
||||
"soldier"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"relevant": [
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Honesty Before Comfort'. Query states the principle in wholly different words. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"choosing",
|
||||
"fact",
|
||||
"fiction",
|
||||
"pleasant",
|
||||
"uncomfortable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"relevant": [
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Precision Over Brute Force'. Query restates the claim with no shared vocabulary. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"beats",
|
||||
"bloated",
|
||||
"payload",
|
||||
"tight"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"relevant": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the obligation without the node's terms. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"able",
|
||||
"coming",
|
||||
"job",
|
||||
"nobody"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the library following Will across 30+ moves. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"creditors",
|
||||
"learning",
|
||||
"seize",
|
||||
"wealth"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"relevant": [
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Earned Trust' ('Trust is demonstrated, not declared'). VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"assertion",
|
||||
"proven",
|
||||
"record",
|
||||
"reliability",
|
||||
"track"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"relevant": [
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"boundaries",
|
||||
"confine",
|
||||
"enable",
|
||||
"instead"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"relevant": [
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Change Is the Signal', the value VBD is built on. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"apart",
|
||||
"cut",
|
||||
"shifts",
|
||||
"system",
|
||||
"tells"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"relevant": [
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - The System Must Accumulate'. Query is the accumulation claim in different vocabulary. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"compounds",
|
||||
"day",
|
||||
"instead",
|
||||
"mind",
|
||||
"resetting"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"relevant": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop as the first person Will did not perform for. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"exterior",
|
||||
"loved",
|
||||
"polished",
|
||||
"self",
|
||||
"unedited"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"relevant": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'. VERIFIED at build time: of the 4 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"arrive",
|
||||
"assuming",
|
||||
"cheerfulness",
|
||||
"instead"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"relevant": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"derivation": "HAND-SELECTED with criterion: target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between two parents' collapses. VERIFIED at build time: of the 5 content words in the query, ZERO appear anywhere in the target's label, content, or tags — so no string-matching retriever can reach this answer.",
|
||||
"zero_overlap_verified": true,
|
||||
"query_content_words": [
|
||||
"childhood",
|
||||
"foundation",
|
||||
"inherit",
|
||||
"offering",
|
||||
"solid"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71 ('Value ? Do the Essential Thing While You Can'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"relevant": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-58874a74-b96f-4883-9e08-45707f4bd3ee ('Value ? Survival Is Not an Excuse to Stop'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (13 of 13; 0 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-58874a74-b96f-4883-9e08-45707f4bd3ee) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"relevant": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e ('Value ? Being Seen Is Rarer Than Being Known'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83 ('Value ? Constraints as Freedom'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (6 of 13; 7 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 7
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"relevant": [
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-e0423482-cfa5-4796-8689-8495c93b66bc ('Value ? Hope Is a Conclusion'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-e0423482-cfa5-4796-8689-8495c93b66bc) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"relevant": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"derivation": "DERIVED FROM EDGES: the query is built from the distinctive vocabulary of kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8 ('Value ? Capability Is a Debt You Owe the Moment'), which hangs off the values hub kn-5b606390-a52d-4ca2-8e0e-eba141d13440 by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub (11 of 13; 2 dropped because they shared a query word and so were lexically reachable). Every remaining sibling shares ZERO content words with the query — the only route from query to answer is seed(kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8) -> hub -> sibling, a two-hop traversal.",
|
||||
"associative_source": "kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"hub": "kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"siblings_dropped_for_overlap": 2
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"relevant": [],
|
||||
"derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
"expect_empty": true
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"relevant": [],
|
||||
"derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
"expect_empty": true
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"relevant": [],
|
||||
"derivation": "CONTROL: verified at build time that none of this string's tokens occurs anywhere in the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
|
||||
"expect_empty": true
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"relevant": [
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56"
|
||||
],
|
||||
"derivation": "DERIVED: correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); the stale node is the surviving backlog item still titled 'Implement DARMA'. Scored on RANKING, not presence: the corrected node mem-80d7416b-20e9-48a0-b176-b215527e2f56 must be returned AND must rank above the stale node bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff.",
|
||||
"must_outrank": [
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"stale_id": "bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"correct_label": "CORRECTION: The autonomous self-improvement architecture is DHARMA ? n",
|
||||
"stale_label": "Implement DARMA ? Directed Autonomous Runtime Modification Architectur"
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"relevant": [
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8"
|
||||
],
|
||||
"derivation": "DERIVED: correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; the stale node is the surviving memory that asserts 12 filed patents. Scored on RANKING, not presence: the corrected node 3cf706a1-3825-45d8-b0a9-06cae6cdf5b8 must be returned AND must rank above the stale node 936541a9-fabb-466b-9ca3-a78b17ad0c53.",
|
||||
"must_outrank": [
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"936541a9-fabb-466b-9ca3-a78b17ad0c53"
|
||||
],
|
||||
"stale_id": "936541a9-fabb-466b-9ca3-a78b17ad0c53",
|
||||
"correct_label": "memory:remembered",
|
||||
"stale_label": "memory:remembered"
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"relevant": [
|
||||
"mem-30425134-6008-4fd9-a3ee-67a7742c319b"
|
||||
],
|
||||
"derivation": "DERIVED: correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of April 30 2026; the stale node still records the MCP server as live. Scored on RANKING, not presence: the corrected node mem-30425134-6008-4fd9-a3ee-67a7742c319b must be returned AND must rank above the stale node mem-101e81b4-8097-4749-8d8d-7bb66de34517.",
|
||||
"must_outrank": [
|
||||
"mem-30425134-6008-4fd9-a3ee-67a7742c319b",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517"
|
||||
],
|
||||
"stale_id": "mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"correct_label": "CGI ARCHITECTURE ? THREE LAYERS, MCP RETIRED (April 30, 2026). Definit",
|
||||
"stale_label": "GCloud MCP infrastructure ? April 27, 2026. Legion died (~19:30 UTC). "
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,117 +0,0 @@
|
||||
import json,pickle,os,math,urllib.request,numpy as np
|
||||
S='/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/sim/'
|
||||
C=pickle.load(open(S+'corpus.pkl','rb'))
|
||||
NODES=C['nodes']; EDGES=C['edges']; N=len(NODES)
|
||||
E=np.load(S+'emb.npy'); HAVE=np.load(S+'have.npy')
|
||||
En=E/np.maximum(np.linalg.norm(E,axis=1,keepdims=True),1e-12)
|
||||
LAYERS={int(l['layer_id']):l for l in (C['layers'] or [])} if C['layers'] else {}
|
||||
TRANS=set(i for i,l in LAYERS.items() if l.get('transparent'))
|
||||
def addressable(s):
|
||||
if not s: return False
|
||||
return all(0x20<=ord(ch)<=0x7e for ch in s)
|
||||
ADDR=np.array([addressable(n['id']) for n in NODES])
|
||||
OK=np.array([ (n['layer_id'] not in TRANS) and ADDR[i] for i,n in enumerate(NODES)])
|
||||
SAL=np.array([n['salience'] for n in NODES])
|
||||
LOW=[ (n['content']+'\x00'+n['label']+'\x00'+n['tags']).lower() for n in NODES]
|
||||
DL=np.array([float(len(n['content'])+len(n['label'])+len(n['tags'])) for n in NODES])
|
||||
IDX={}
|
||||
for i,n in enumerate(NODES):
|
||||
IDX.setdefault(n['id'],i)
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
ADJ_F=[[] for _ in range(N)]; ADJ_T=[[] for _ in range(N)]
|
||||
for e in EDGES:
|
||||
a=IDX.get(e['from']); b=IDX.get(e['to'])
|
||||
if a is None or b is None: continue
|
||||
ADJ_F[a].append((e,b)); ADJ_T[b].append((e,a))
|
||||
EXCL=np.array([n['node_type'] in ('Tag','InternalStateEvent') for n in NODES])
|
||||
avgdl_all=None
|
||||
def tokenize(q):
|
||||
out=[]
|
||||
for t in q.split():
|
||||
if not any(t.lower()==x.lower() for x in out): out.append(t)
|
||||
return out
|
||||
_qcache={}
|
||||
def qemb(q):
|
||||
if q in _qcache: return _qcache[q]
|
||||
body=json.dumps({"model":"nomic-embed-text","prompt":q}).encode()
|
||||
r=urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=body,headers={"Content-Type":"application/json"}),timeout=30)
|
||||
v=np.array(json.loads(r.read())["embedding"],dtype=np.float32)
|
||||
v=v/np.linalg.norm(v); _qcache[q]=v; return v
|
||||
K1,B=1.2,0.75
|
||||
SEED_MIN=0.60; SEED_K=8; ASSOC_SEEDS=3; DEPTH=2; FIRE=0.02; AMAX=64; DECAY=0.7
|
||||
def legs(query):
|
||||
toks=tokenize(query)
|
||||
masks=[];
|
||||
hit_idx=[]; hit_mask=[]
|
||||
df=[0]*len(toks)
|
||||
lt=[t.lower() for t in toks]
|
||||
for i in range(N):
|
||||
if not OK[i]: continue
|
||||
s=LOW[i]; m=0
|
||||
for t,tok in enumerate(lt):
|
||||
if tok in s: m|=(1<<t)
|
||||
if m:
|
||||
hit_idx.append(i); hit_mask.append(m)
|
||||
for t in range(len(toks)):
|
||||
if m>>t&1: df[t]+=1
|
||||
dl_n=int(OK.sum()); avgdl=float(DL[OK].sum()/max(dl_n,1))
|
||||
idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df]
|
||||
L=[]
|
||||
for j,i in enumerate(hit_idx):
|
||||
norm=1.0-B+B*(DL[i]/avgdl); w=0.0
|
||||
for t in range(len(toks)):
|
||||
if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
L.append((i,w,SAL[i]))
|
||||
L.sort(key=lambda x:(-x[1],-x[2]))
|
||||
qv=qemb(query)
|
||||
cos=En@qv
|
||||
cos=np.where(HAVE&OK,cos,-2.0)
|
||||
order=np.argsort(-cos)
|
||||
semfull=[(int(i),float(cos[i])) for i in order[:400]]
|
||||
Sleg=[(i,(c-SEED_MIN)/(1-SEED_MIN)) for i,c in semfull if c>SEED_MIN]
|
||||
semseed=[i for i,c in semfull[:SEED_K] if c>0.0]
|
||||
# assoc
|
||||
act={}; seen={}; qq=[]
|
||||
for i,_,_ in L[:ASSOC_SEEDS]:
|
||||
act[i]=1.0; seen[i]=2; qq.append((i,0))
|
||||
for i in semseed:
|
||||
if i in seen: continue
|
||||
act[i]=1.0; seen[i]=2; qq.append((i,0))
|
||||
qh=0
|
||||
while qh<len(qq):
|
||||
cur,h=qq[qh]; qh+=1
|
||||
if h>=DEPTH: continue
|
||||
parent=act[cur]
|
||||
for e,oi in ADJ_F[cur]+ADJ_T[cur]:
|
||||
if e['rel'] not in STRUCT: continue
|
||||
if EXCL[oi]: continue
|
||||
na=parent*e['w']*DECAY*SAL[oi]
|
||||
if na<FIRE: continue
|
||||
if seen.get(oi) and na<=act.get(oi,0): continue
|
||||
act[oi]=na
|
||||
if not seen.get(oi): seen[oi]=1
|
||||
if len(qq)<AMAX*4: qq.append((oi,h+1))
|
||||
A=[]
|
||||
for i,st in seen.items():
|
||||
if st!=1: continue
|
||||
if not OK[i] or not HAVE[i]: continue
|
||||
c=float(cos[i])
|
||||
if c<=0.0: continue
|
||||
A.append((i,c))
|
||||
A.sort(key=lambda x:-x[1]); A=A[:AMAX]
|
||||
return L,Sleg,A,cos
|
||||
def interleave3(L,Sl,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(Sl) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li][0] not in out: out.append(L[li][0])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(Sl):
|
||||
if Sl[si][0] not in out: out.append(Sl[si][0])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai][0] not in out: out.append(A[ai][0])
|
||||
ai+=1
|
||||
return out
|
||||
@@ -1,17 +0,0 @@
|
||||
import json,sys
|
||||
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
TSV,OUT=sys.argv[1],sys.argv[2]
|
||||
emb={}
|
||||
for line in open(TSV,encoding='utf-8',errors='surrogateescape'):
|
||||
p=line.rstrip("\n").rsplit("\t",1)
|
||||
if len(p)==2 and p[1].count(",")>100: emb[p[0]]=p[1]
|
||||
print("vectors",len(emb),flush=True)
|
||||
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
|
||||
hit=0
|
||||
for n in d["nodes"]:
|
||||
v=emb.get(n.get("id") or "")
|
||||
if v: n["emb"]=v; hit+=1
|
||||
print("attached",hit,"of",len(d["nodes"]),flush=True)
|
||||
with open(OUT,"w",encoding='utf-8',errors='surrogateescape') as f:
|
||||
json.dump(d,f,ensure_ascii=False)
|
||||
print("wrote",OUT,flush=True)
|
||||
@@ -1,102 +0,0 @@
|
||||
import json,sys,pickle,numpy as np
|
||||
sys.path.insert(0,'.')
|
||||
from legs import *
|
||||
GP='/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/'
|
||||
G=json.load(open(GP+'gold_set.json'))
|
||||
def legs3(query, sem_sal=False, assoc_sal=False, unfloor=False, sem_cap=None):
|
||||
toks=tokenize(query)
|
||||
hit_idx=[];hit_mask=[];df=[0]*len(toks);lt=[t.lower() for t in toks]
|
||||
for i in range(N):
|
||||
if not OK[i]: continue
|
||||
s=LOW[i];m=0
|
||||
for t,tok in enumerate(lt):
|
||||
if tok in s: m|=(1<<t)
|
||||
if m:
|
||||
hit_idx.append(i);hit_mask.append(m)
|
||||
for t in range(len(toks)):
|
||||
if m>>t&1: df[t]+=1
|
||||
dl_n=int(OK.sum());avgdl=float(DL[OK].sum()/max(dl_n,1))
|
||||
idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df]
|
||||
L=[]
|
||||
for j,i in enumerate(hit_idx):
|
||||
norm=1.0-B+B*(DL[i]/avgdl);w=0.0
|
||||
for t in range(len(toks)):
|
||||
if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
L.append((i,w,SAL[i]))
|
||||
L.sort(key=lambda x:(-x[1],-x[2]))
|
||||
if not L: return [],[],[]
|
||||
qv=qemb(query);cos=En@qv;cos=np.where(HAVE&OK,cos,-2.0)
|
||||
order=np.argsort(-cos)[:600]
|
||||
cand=[int(i) for i in order if cos[i]>(0.0 if unfloor else SEED_MIN)]
|
||||
key=(lambda i:(SAL[i] if sem_sal else 1.0)*float(cos[i]))
|
||||
Sl=sorted(cand,key=lambda i:-key(i))
|
||||
if sem_cap: Sl=Sl[:sem_cap]
|
||||
semseed=[int(i) for i in order[:SEED_K] if cos[i]>0.0]
|
||||
act={};seen={};qq=[]
|
||||
for i,_,_ in L[:ASSOC_SEEDS]:
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
for i in semseed:
|
||||
if i in seen: continue
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
qh=0
|
||||
while qh<len(qq):
|
||||
cur,h=qq[qh];qh+=1
|
||||
if h>=DEPTH: continue
|
||||
parent=act[cur]
|
||||
for e,oi in ADJ_F[cur]+ADJ_T[cur]:
|
||||
if e['rel'] not in STRUCT: continue
|
||||
if EXCL[oi]: continue
|
||||
na=parent*e['w']*DECAY*SAL[oi]
|
||||
if na<FIRE: continue
|
||||
if seen.get(oi) and na<=act.get(oi,0): continue
|
||||
act[oi]=na
|
||||
if not seen.get(oi): seen[oi]=1
|
||||
if len(qq)<AMAX*4: qq.append((oi,h+1))
|
||||
A=[]
|
||||
for i,st in seen.items():
|
||||
if st!=1 or not OK[i] or not HAVE[i]: continue
|
||||
c=float(cos[i])
|
||||
if c<=0.0: continue
|
||||
A.append((i,(SAL[i] if assoc_sal else 1.0)*c))
|
||||
A.sort(key=lambda x:-x[1]);A=[i for i,_ in A[:AMAX]]
|
||||
return [i for i,_,_ in L],Sl,A
|
||||
def merge(L,S,A,lim=10):
|
||||
out=[];li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def outcome(q,ids):
|
||||
c=q['category']
|
||||
if c=='nonsense': return len(ids)==0
|
||||
if c=='superseded':
|
||||
a,b=q['must_outrank']
|
||||
if a not in ids: return False
|
||||
if b not in ids: return True
|
||||
return ids.index(a)<ids.index(b)
|
||||
return any(x in ids[:5] for x in q['relevant'])
|
||||
def run(**kw):
|
||||
return {q['id']:outcome(q,[NODES[i]['id'] for i in merge(*legs3(q['query'],**kw),10)]) for q in G['queries']}
|
||||
base=run()
|
||||
print("baseline",sum(base.values()),"/38 misses:",[k for k,v in base.items() if not v])
|
||||
import itertools
|
||||
for name,kw in [
|
||||
('sem_sal(floored)',dict(sem_sal=True)),
|
||||
('unfloor',dict(unfloor=True)),
|
||||
('unfloor+sem_sal',dict(unfloor=True,sem_sal=True)),
|
||||
('assoc_sal',dict(assoc_sal=True)),
|
||||
('unfloor+sem_sal+assoc_sal',dict(unfloor=True,sem_sal=True,assoc_sal=True)),
|
||||
('sem_sal+assoc_sal(floored)',dict(sem_sal=True,assoc_sal=True)),
|
||||
]:
|
||||
r=run(**kw)
|
||||
g=sorted(k for k in base if r[k] and not base[k]); l=sorted(k for k in base if base[k] and not r[k])
|
||||
print("%-28s net=%+d gains=%s losses=%s"%(name,len(g)-len(l),g,l))
|
||||
@@ -1,942 +0,0 @@
|
||||
{
|
||||
"label": "act-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-act",
|
||||
"soul_md5": "77722f5a9f49494bf735c2a4be1b5dc4",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7902,
|
||||
"wall_clock_s": 121.2,
|
||||
"child_pid": 78714,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.22857142857142856,
|
||||
"recall@5": 0.18908730158730158,
|
||||
"recall@10": 0.24277210884353742,
|
||||
"precision@5": 0.06857142857142857,
|
||||
"mrr@10": 0.24154195011337865,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3237.6,
|
||||
"latency_ms_p95": 5264.0,
|
||||
"latency_ms_max": 5510.9,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.2857142857142857,
|
||||
"recall@5": 0.0882936507936508,
|
||||
"recall@10": 0.3567176870748299,
|
||||
"mrr@10": 0.3505668934240363
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 476.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 708.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 532.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 537.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 568.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 555.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-69fd6e83-7718-4824-8d66-f49d8954e224",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-c3d9d063-8c5d-45aa-900c-550914b2ff6d",
|
||||
"kn-f838f113-76d5-4a15-9cef-14055c4723a3",
|
||||
"bl-76e878aa-e1fe-468c-bf9c-854097cb7e0b",
|
||||
"art-c71aef51-026f-4d63-80e9-2a0ec0dc3865",
|
||||
"bl-e148d23c-24e8-4122-9915-d1c11f22052f",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1906.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-80720fdf-7ce7-4d28-aff8-21028d3a8cfb",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"?Z?.\f?0?]P?",
|
||||
"?Q??m?;��`'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1103.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.0625,
|
||||
"recall@10": 0.1875,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 6,
|
||||
"latency_ms": 1132.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5555555555555556,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-c9adb8e5-293f-4033-99f8-0405c17ef941",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-7aebe936-ac55-4f35-8932-adc5224ff854",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"mem-34f53a9d-a131-4f82-9dbd-b9eb4a9af52e",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 960.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
";??A5???",
|
||||
"art-94fae615-7cd5-4695-b968-977101b06a51",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-57164d5f-baf0-4149-957a-379a4e255d1a",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-8dbceb06-431a-416d-a723-e8c75d595154"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 992.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"kn-b2a99cd7-b379-4d9b-a996-e347a02c7bad",
|
||||
"bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"kn-8e1bfb48-33a9-45ad-8da7-e0bdaa5d34e7",
|
||||
"art-92e1837c-5919-42d0-bbb0-4d924d7b2864",
|
||||
"bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f",
|
||||
"mem-f823e835-313f-4282-b4b3-ce527ffc2f7a",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?Z?.\f?0?]P?",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2820.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.14285714285714285,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-e20944e5-eb16-4ab3-a84d-111e0fc817fa",
|
||||
"bl-e20944e5-f4a6-44a0-91b1-73d04ebed120",
|
||||
"mem-47f72b5b-6e8b-4293-94f1-350197b4809a",
|
||||
"mem-a5f04e52-91f8-41d2-af27-8bf803621758",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2609.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"kn-82be4e41-96c5-4da3-85c0-cee10763d975",
|
||||
"mem-ea487cb4-ed67-44ce-8402-b56bb28468d4",
|
||||
"bl-2694b588-a6e3-43de-861c-fa7b0ec7e7fd",
|
||||
"bl-14883d81-f7cb-46dd-82c2-a6e6980264e5",
|
||||
"tag-dark-theme",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5510.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"mem-0328c3cb-4550-4ce4-9284-152e832f08f6",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685",
|
||||
"bl-6172d035-dd94-4776-afdd-d8915f6fc375",
|
||||
"bl-5bb8dedf-8498-4a9b-acdc-31cc9c738f2a",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
";??A5???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5444.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"bl-d24fcce8-2b55-426f-867a-db3958a622d3",
|
||||
"tag-phase-3",
|
||||
"tag-project-structure",
|
||||
"tag-anthropic-contrast",
|
||||
"tag-voice-training",
|
||||
"tag-ebd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
";??A5???",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5056.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-fc893be3-e6b4-4ef6-93b0-d54ca5f89083",
|
||||
"bl-57c5cf6b-81a5-4558-9902-5c02981fe273",
|
||||
"tag-guilds",
|
||||
"tag-kids",
|
||||
"tag-coexistence",
|
||||
"tag-cultivated-general-intelligence",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
";??A5???",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3473.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992",
|
||||
"bl-b019ce7a-1b21-436e-812d-032f50c6c45f",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0",
|
||||
"tag-stable-diffusion",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4446.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"mem-e321e54e-8bb3-4596-b13d-bb093d6b149d",
|
||||
"mem-e32ba5a7-c147-4dc0-9479-b720d768eda6",
|
||||
"mem-c7a77457-478d-4eb0-a116-67205a0066a4",
|
||||
"bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8",
|
||||
"bl-aa762207-920d-45ab-b2a3-2f8154d7ef9b",
|
||||
"tag-misalignment",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3932.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"kn-48a01973-a025-471d-950f-b93e6a426d82",
|
||||
"mem-23d22bc1-a097-446b-8f11-8aff099e0b76",
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"mem-ce5a2ffc-ad39-4728-9ac6-76fef507d5da",
|
||||
"project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_",
|
||||
"tag-provenance",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4836.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"kn-79056192-7de8-486c-9565-f128439a6fcb",
|
||||
"bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b",
|
||||
"bl-7f33f1bc-99fa-4906-889f-a42375beea20",
|
||||
"mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"tag-__cultivation-metric____internal-state____dharma____evidence____novel-idea____gap-compression____values____microsoft__",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4057.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-identity-studio",
|
||||
"tag-finance",
|
||||
"tag-temporal",
|
||||
"tag-barkhausen",
|
||||
"tag-ilogger",
|
||||
"tag-design-first",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5098.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"project-Goal_setting__alignment__scoring__cadence__Attaches_to_any_imprint_",
|
||||
"tag-performed-values",
|
||||
"tag-turing-test",
|
||||
"tag-sealed",
|
||||
"tag-part-5",
|
||||
"tag-model",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
";??A5???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4652.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-9590ba23-bddb-43e8-a571-68a263c4c364",
|
||||
"project-Imprint__leadership_development__feedback_frameworks__performance__presence_",
|
||||
"bl-a9e57bb2-00a1-4867-ab59-5d9271134b50",
|
||||
"bl-c7793c4a-7630-47fc-a462-d23059087e80",
|
||||
"tag-gateway_platform_neuron-technologies_go_proxy_llm",
|
||||
"tag-fornax",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4653.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"bl-7a13527b-3e0c-418a-9f37-88fd2152e5ce",
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"bl-5e390b10-8753-4f25-a1a5-b5dbbb002cbf",
|
||||
"tag-ats",
|
||||
"tag-data-model",
|
||||
"tag-aggregation",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
";??A5???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4272.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"tag-memory-model",
|
||||
"tag-storage",
|
||||
"tag-ga4",
|
||||
"tag-potions",
|
||||
"tag-resonance",
|
||||
"tag-neuron",
|
||||
"? ?}&?#??X\b",
|
||||
";??A5???",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4541.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"mem-927f41ab-8ede-4f58-acb3-995db16ac775",
|
||||
"mem-dbe80bc2-c602-46b0-b4ea-dd222e52bcde",
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"tag-upload-window",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4883.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"mem-481c769c-68cc-45c7-bc37-c0d9778fa648",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-9d8f3c5b-4bac-41ce-8ac4-44733f99d6c8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3237.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-eb1c6d99-d603-4f33-be9a-c63a178690c6",
|
||||
"mem-a3124d5b-2f50-477f-8bb5-06879f5a496c",
|
||||
"bl-7fa1b1a8-b80a-4f28-b162-bfe73765b4f8",
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2457.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"mem-265a7107-73b3-4410-9aff-43787d5f473b",
|
||||
"mem-9d1bf963-1b40-4588-bdb3-0432646cc623",
|
||||
"mem-46780047-63a0-4a86-a16b-638b72a7fb8d",
|
||||
"mem-3987d374-3c48-4e8e-b06d-0c363f55ed9c",
|
||||
"bl-e0a0df72-de6e-46ab-800b-e1e3e8dfc387",
|
||||
"tag-__patents____swarm____claim-language____prior-art____filing__",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2640.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-dca14c4c-4859-47b0-996e-33964ba61a87",
|
||||
"bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16",
|
||||
"mem-8d1bafe6-209c-456c-9a25-9a927bc5a16d",
|
||||
"bl-0de4e61b-6562-49e5-b7df-ebb809a01723",
|
||||
"bl-8ef1ba6b-3fa0-4dbd-98c5-31665e5694a1",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?Q??m?;��`'",
|
||||
"R^??m?;��?'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3705.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-394cc9e8-049b-45bc-a380-66314f14e367",
|
||||
"bl-ffa22d7e-42a9-4bd2-a428-1d2df243ac93",
|
||||
"bl-4a6746e8-191f-48fc-8bfb-c4dc73b80bcd",
|
||||
"tag-command-pattern",
|
||||
"tag-withholding",
|
||||
"tag-offline",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4223.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 1333.9,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 880.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 8,
|
||||
"latency_ms": 1452.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 8
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"ctx-e5427d7d",
|
||||
"mem-37b57f52-a29a-42cf-a07a-3c5f8a3598dd",
|
||||
"project-worldweaver",
|
||||
"tag-__kotlin____internal-state____pre-reasoning____post-reasoning____compression-ratio____dharma____cultivation__",
|
||||
"tag-import",
|
||||
"tag-__cgi____dharma____cultivation____five-primitives____seed-artifact____agi____intelligence____whitepaper____patent__",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3843.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-bfe0fafd-2750-4fdc-b773-04e878b3b23f",
|
||||
"art-7bdaff30-5af9-4f0a-93b1-751686f9de3d",
|
||||
"mem-cf07910d-4676-4384-ab97-9cad946cd0b9",
|
||||
"mem-f9da4b43-3724-4bc8-92f8-6f237c89dc4d",
|
||||
"project-Convert_UTC_timestamps_to_Central_time_when_displaying_to_Will__Never_surface_raw_UTC_",
|
||||
"mem-32203649-3213-4d6d-86fd-3d657ac70d77",
|
||||
";??A5???",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5264.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"project-Define_Iris_as_separate_public_brand__Uncensored_but_principled__Consumer_face_while_Neuron_runs_enterprise_",
|
||||
"project-Imprint__discovery__objection_handling__deal_strategy__pipeline__closing_",
|
||||
"bl-8116da7a-b039-4e08-b8d0-c1c7861f9766",
|
||||
"tag-enterprise",
|
||||
"tag-divisors",
|
||||
"tag-distressed-property",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"???Ͼd??W\b?",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3022.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,942 +0,0 @@
|
||||
{
|
||||
"label": "act-r3",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-act",
|
||||
"soul_md5": "77722f5a9f49494bf735c2a4be1b5dc4",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7903,
|
||||
"wall_clock_s": 112.0,
|
||||
"child_pid": 78802,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.2571428571428571,
|
||||
"recall@5": 0.19291383219954647,
|
||||
"recall@10": 0.244812925170068,
|
||||
"precision@5": 0.08,
|
||||
"mrr@10": 0.266031746031746,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3220.9,
|
||||
"latency_ms_p95": 4840.2,
|
||||
"latency_ms_max": 5073.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.42857142857142855,
|
||||
"recall@5": 0.10742630385487528,
|
||||
"recall@10": 0.366921768707483,
|
||||
"mrr@10": 0.47301587301587306
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 471.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 548.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 470.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 476.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 515.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 458.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-69fd6e83-7718-4824-8d66-f49d8954e224",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-c3d9d063-8c5d-45aa-900c-550914b2ff6d",
|
||||
"kn-82be4e41-96c5-4da3-85c0-cee10763d975",
|
||||
"bl-76e878aa-e1fe-468c-bf9c-854097cb7e0b",
|
||||
"art-9887867c-2e21-47c3-9f96-c2dfe5bd4cc1",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1595.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"bl-80720fdf-7ce7-4d28-aff8-21028d3a8cfb",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"?Z?.\f?0?]P?",
|
||||
"?Q??m?;��`'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 927.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.125,
|
||||
"recall@10": 0.1875,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 6,
|
||||
"latency_ms": 963.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5555555555555556,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a",
|
||||
"bl-7aebe936-ac55-4f35-8932-adc5224ff854",
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"bl-455a08cf-5831-4fdb-b42c-b952f2feafb9",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 927.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-8dbceb06-431a-416d-a723-e8c75d595154",
|
||||
"mem-a3124d5b-2f50-477f-8bb5-06879f5a496c",
|
||||
"art-94fae615-7cd5-4695-b968-977101b06a51",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-57164d5f-baf0-4149-957a-379a4e255d1a",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 937.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"kn-b2a99cd7-b379-4d9b-a996-e347a02c7bad",
|
||||
"bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"kn-8e1bfb48-33a9-45ad-8da7-e0bdaa5d34e7",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023",
|
||||
"bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?Z?.\f?0?]P?",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2451.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07142857142857142,
|
||||
"recall@10": 0.21428571428571427,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-e20944e5-f4a6-44a0-91b1-73d04ebed120",
|
||||
"mem-64cf3728-674c-404b-965a-b8f8d38bb7bb",
|
||||
"mem-47f72b5b-6e8b-4293-94f1-350197b4809a",
|
||||
"mem-e612f0aa-c2f2-4ee3-bbc7-af2dc826233b",
|
||||
"mem-a5f04e52-91f8-41d2-af27-8bf803621758",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2042.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"mem-ea487cb4-ed67-44ce-8402-b56bb28468d4",
|
||||
"art-c71aef51-026f-4d63-80e9-2a0ec0dc3865",
|
||||
"bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60",
|
||||
"bl-2694b588-a6e3-43de-861c-fa7b0ec7e7fd",
|
||||
"bl-14883d81-f7cb-46dd-82c2-a6e6980264e5",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4698.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"bl-fd047ce9-ae21-4b3e-b3ab-ece0c9592f7f",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0",
|
||||
"bl-6172d035-dd94-4776-afdd-d8915f6fc375",
|
||||
"bl-5bb8dedf-8498-4a9b-acdc-31cc9c738f2a",
|
||||
"tag-cultivated-general-intelligence",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5073.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685",
|
||||
"bl-fc893be3-e6b4-4ef6-93b0-d54ca5f89083",
|
||||
"tag-project-structure",
|
||||
"tag-anthropic-contrast",
|
||||
"tag-voice-training",
|
||||
"tag-ebd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4117.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-d24fcce8-2b55-426f-867a-db3958a622d3",
|
||||
"tag-phase-3",
|
||||
"bl-57c5cf6b-81a5-4558-9902-5c02981fe273",
|
||||
"tag-guilds",
|
||||
"tag-kids",
|
||||
"tag-coexistence",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3254.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"mem-0328c3cb-4550-4ce4-9284-152e832f08f6",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"tag-stable-diffusion",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4081.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"mem-e32ba5a7-c147-4dc0-9479-b720d768eda6",
|
||||
"bl-b019ce7a-1b21-436e-812d-032f50c6c45f",
|
||||
"mem-c7a77457-478d-4eb0-a116-67205a0066a4",
|
||||
"bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8",
|
||||
"bl-aa762207-920d-45ab-b2a3-2f8154d7ef9b",
|
||||
"tag-misalignment",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3687.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"kn-48a01973-a025-471d-950f-b93e6a426d82",
|
||||
"mem-23d22bc1-a097-446b-8f11-8aff099e0b76",
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"mem-ce5a2ffc-ad39-4728-9ac6-76fef507d5da",
|
||||
"project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_",
|
||||
"mem-e321e54e-8bb3-4596-b13d-bb093d6b149d",
|
||||
"? ?}&?#??X\b",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4659.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"kn-79056192-7de8-486c-9565-f128439a6fcb",
|
||||
"bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b",
|
||||
"bl-7f33f1bc-99fa-4906-889f-a42375beea20",
|
||||
"mem-37b57f52-a29a-42cf-a07a-3c5f8a3598dd",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"tag-__cultivation-metric____internal-state____dharma____evidence____novel-idea____gap-compression____values____microsoft__",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3960.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-identity-studio",
|
||||
"tag-finance",
|
||||
"tag-temporal",
|
||||
"tag-barkhausen",
|
||||
"tag-ilogger",
|
||||
"tag-design-first",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4840.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"project-Goal_setting__alignment__scoring__cadence__Attaches_to_any_imprint_",
|
||||
"tag-performed-values",
|
||||
"tag-turing-test",
|
||||
"tag-sealed",
|
||||
"tag-part-5",
|
||||
"tag-model",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4493.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-9590ba23-bddb-43e8-a571-68a263c4c364",
|
||||
"project-Imprint__leadership_development__feedback_frameworks__performance__presence_",
|
||||
"bl-a9e57bb2-00a1-4867-ab59-5d9271134b50",
|
||||
"bl-c7793c4a-7630-47fc-a462-d23059087e80",
|
||||
"tag-gateway_platform_neuron-technologies_go_proxy_llm",
|
||||
"tag-fornax",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4509.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"bl-7a13527b-3e0c-418a-9f37-88fd2152e5ce",
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"bl-5e390b10-8753-4f25-a1a5-b5dbbb002cbf",
|
||||
"tag-ats",
|
||||
"tag-data-model",
|
||||
"tag-aggregation",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3624.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"tag-memory-model",
|
||||
"tag-storage",
|
||||
"tag-ga4",
|
||||
"tag-potions",
|
||||
"tag-resonance",
|
||||
"tag-neuron",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4093.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"mem-927f41ab-8ede-4f58-acb3-995db16ac775",
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"tag-upload-window",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4235.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"mem-0f31141d-3ac5-44b2-9942-be7e4e6feb79",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"mem-481c769c-68cc-45c7-bc37-c0d9778fa648",
|
||||
"bl-9d8f3c5b-4bac-41ce-8ac4-44733f99d6c8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3220.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1c6d99-d603-4f33-be9a-c63a178690c6",
|
||||
"mem-5e7f6ddd-c818-4ad3-b564-54ae278e9976",
|
||||
"bl-7fa1b1a8-b80a-4f28-b162-bfe73765b4f8",
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c",
|
||||
"tag-sarah",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2336.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"mem-265a7107-73b3-4410-9aff-43787d5f473b",
|
||||
"mem-9d1bf963-1b40-4588-bdb3-0432646cc623",
|
||||
"mem-46780047-63a0-4a86-a16b-638b72a7fb8d",
|
||||
"mem-3987d374-3c48-4e8e-b06d-0c363f55ed9c",
|
||||
"bl-e0a0df72-de6e-46ab-800b-e1e3e8dfc387",
|
||||
"tag-__patents____swarm____claim-language____prior-art____filing__",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2348.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-dca14c4c-4859-47b0-996e-33964ba61a87",
|
||||
"bl-e148d23c-24e8-4122-9915-d1c11f22052f",
|
||||
"bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16",
|
||||
"mem-8d1bafe6-209c-456c-9a25-9a927bc5a16d",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"tag-dark-theme",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?Q??m?;��`'",
|
||||
"R^??m?;��?'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3511.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-ffa22d7e-42a9-4bd2-a428-1d2df243ac93",
|
||||
"mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9",
|
||||
"bl-452a4710-3d2b-4e0f-9413-49a66423bc9a",
|
||||
"bl-4a6746e8-191f-48fc-8bfb-c4dc73b80bcd",
|
||||
"tag-withholding",
|
||||
"tag-offline",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4160.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 1318.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 875.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 8,
|
||||
"latency_ms": 1393.4,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 8
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c",
|
||||
"ctx-e5427d7d",
|
||||
"project-worldweaver",
|
||||
"tag-__kotlin____internal-state____pre-reasoning____post-reasoning____compression-ratio____dharma____cultivation__",
|
||||
"tag-import",
|
||||
"tag-enterprise",
|
||||
"tag-__cgi____dharma____cultivation____five-primitives____seed-artifact____agi____intelligence____whitepaper____patent__",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3775.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-bfe0fafd-2750-4fdc-b773-04e878b3b23f",
|
||||
"art-7bdaff30-5af9-4f0a-93b1-751686f9de3d",
|
||||
"mem-cf07910d-4676-4384-ab97-9cad946cd0b9",
|
||||
"mem-f9da4b43-3724-4bc8-92f8-6f237c89dc4d",
|
||||
"project-Convert_UTC_timestamps_to_Central_time_when_displaying_to_Will__Never_surface_raw_UTC_",
|
||||
"mem-32203649-3213-4d6d-86fd-3d657ac70d77",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4857.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"bl-0de4e61b-6562-49e5-b7df-ebb809a01723",
|
||||
"project-Imprint__discovery__objection_handling__deal_strategy__pipeline__closing_",
|
||||
"bl-8116da7a-b039-4e08-b8d0-c1c7861f9766",
|
||||
"bl-8ef1ba6b-3fa0-4dbd-98c5-31665e5694a1",
|
||||
"tag-divisors",
|
||||
"tag-distressed-property",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-b046991d-5992-4ac4-b854-7d3ac273832c",
|
||||
"???Ͼd??W\b?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2858.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "assoc-leg-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-assoc",
|
||||
"soul_md5": "ab9d490ecdfb1f9e6f23cca841ad8fb5",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 51.8,
|
||||
"child_pid": 87150,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1223.5,
|
||||
"latency_ms_p95": 1676.1,
|
||||
"latency_ms_max": 1740.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 306.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 331.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 292.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 309.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 331.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 295.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 613.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 538.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 553.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.4444444444444444,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 527.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 546.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 892.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 754.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1644.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1740.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1393.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1118.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"ctx-cc7f",
|
||||
"?of?7???",
|
||||
"ctx-4a41",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"ctx-175f"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1440.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1316.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1620.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1393.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1676.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1534.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1567.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1280.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1401.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1512.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1165.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1230.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"ctx-bb74",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1223.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1235.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1460.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 750.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 531.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 769.1,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1327.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1687.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1040.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "assoc-leg",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-assoc",
|
||||
"soul_md5": "ab9d490ecdfb1f9e6f23cca841ad8fb5",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 52.4,
|
||||
"child_pid": 87099,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6285714285714286,
|
||||
"recall@5": 0.45309194773480493,
|
||||
"recall@10": 0.5405733155733157,
|
||||
"precision@5": 0.17714285714285719,
|
||||
"mrr@10": 0.42650793650793645,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1228.5,
|
||||
"latency_ms_p95": 1681.8,
|
||||
"latency_ms_max": 1718.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657342,
|
||||
"recall@10": 0.24825174825174826,
|
||||
"mrr@10": 0.22777777777777777
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4882369614512472,
|
||||
"recall@10": 0.6329365079365079,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 306.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 360.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 324.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 313.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 331.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 303.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 588.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 533.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.4444444444444444,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 517.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 547.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 903.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 774.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1641.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1718.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1409.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1135.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"ctx-cc7f",
|
||||
"?of?7???",
|
||||
"ctx-4a41",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"ctx-175f"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1446.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1296.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1621.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1411.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1681.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1536.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1567.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1278.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1401.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1509.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1156.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1232.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"ctx-bb74",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1228.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1251.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1459.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 746.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 519.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 764.0,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1346.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1687.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1030.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,945 +0,0 @@
|
||||
{
|
||||
"label": "baseline-embcorpus",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-baseline",
|
||||
"soul_md5": "5cc9521734907cf2da30f0af94498c06",
|
||||
"corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 48.4,
|
||||
"child_pid": 85995,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1145.9,
|
||||
"latency_ms_p95": 1574.3,
|
||||
"latency_ms_max": 1634.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 229.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 262.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 230.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 229.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 250.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 227.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 532.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 457.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 472.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 446.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 462.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 832.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 686.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1550.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1634.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1322.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1041.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1350.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1213.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1553.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1319.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1574.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"????7???Ջ3",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"%???2??jH??",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1441.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1499.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1197.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1311.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1422.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1077.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1149.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1145.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1177.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1382.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 660.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 432.5,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 676.9,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1252.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1624.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 946.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,959 +0,0 @@
|
||||
{
|
||||
"label": "bm25lex-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-bm25lex",
|
||||
"soul_md5": "dfbd0f8e3646212db5c60026f8ad906f",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 50.1,
|
||||
"child_pid": 90371,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1183.8,
|
||||
"latency_ms_p95": 1611.3,
|
||||
"latency_ms_max": 1654.7,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 282.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 315.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 303.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 587.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 518.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 511.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 497.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 510.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"2c56a7a9-5323-4ce4-ba09-35836ba15d54",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 870.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"830ca37a-d334-4e41-ba89-64893dc8d628",
|
||||
"?",
|
||||
"ce9636dc-85a5-4dae-9e07-74ea2fcc6307",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 723.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"66c63082-b4da-4aa1-8fee-848db8a83210",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1584.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"f0eb6b13-909c-4674-91ef-23301d3abc8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"30a44d10-2487-420e-bf61-3892e4343c92",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"bfb5809e-d19a-4d3f-8c1a-796db622ad9d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1652.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"tag-fiction",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1341.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-8de20bcf-7149-4f48-b67c-e7f9758fd6e5",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"bl-164b520b-c503-49db-89f9-bd2fdf4215f5",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"1219277c-1b95-45ec-95a2-07b4a47a4d92",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1064.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"bl-448bc514-c2f1-4520-a9b1-1f3a73678d26",
|
||||
"a1000001-0000-0000-0000-000000000012",
|
||||
"43098881-e044-482b-8e92-471728a8ba8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1398.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"d6b12ecf-702b-4101-b1bb-09ed9b220b29",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"c608a095-c98b-4bfa-bfe1-1611c1320290",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"451ae007-4219-4096-89fe-fa2e045fbeb1",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1252.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1581.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1363.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"5a2c118a-87bd-4239-97a7-9e02c5991983",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1611.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a0edad47-5f77-4fc3-a546-1e85f8c68e77",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"96497334-b18f-495c-9228-eeb8182bdc38",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1470.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"75e036d3-c170-4e3f-acc2-e456a6850ee2",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1513.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-759e78ca-5394-4244-aa39-1c1468bc5f3e",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1226.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"bl-0d8c5dfa-e163-4fef-a58b-56b0d076c5a8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"tag-childhood"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1342.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"3499d5da-0e9c-4de4-9bc4-8941b14e0b1f",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1433.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1104.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1183.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1199.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1202.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"?Z?<S???K ?",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"'?T?a\"B~-?8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1414.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 703.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 492.9,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 728.4,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1289.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7ac62daa-2eac-4c7a-a97e-e4203fc1b57b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1654.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 995.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,959 +0,0 @@
|
||||
{
|
||||
"label": "bm25lex",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-bm25lex",
|
||||
"soul_md5": "dfbd0f8e3646212db5c60026f8ad906f",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 50.4,
|
||||
"child_pid": 90325,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1184.4,
|
||||
"latency_ms_p95": 1620.0,
|
||||
"latency_ms_max": 1655.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 287.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 316.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 283.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 284.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 304.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 282.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 591.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 516.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 520.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 507.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 509.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"2c56a7a9-5323-4ce4-ba09-35836ba15d54",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 872.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"830ca37a-d334-4e41-ba89-64893dc8d628",
|
||||
"?",
|
||||
"ce9636dc-85a5-4dae-9e07-74ea2fcc6307",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 735.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"66c63082-b4da-4aa1-8fee-848db8a83210",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1605.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"f0eb6b13-909c-4674-91ef-23301d3abc8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"30a44d10-2487-420e-bf61-3892e4343c92",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"bfb5809e-d19a-4d3f-8c1a-796db622ad9d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1655.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"tag-fiction",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1339.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-8de20bcf-7149-4f48-b67c-e7f9758fd6e5",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"bl-164b520b-c503-49db-89f9-bd2fdf4215f5",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"1219277c-1b95-45ec-95a2-07b4a47a4d92",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1065.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"bl-448bc514-c2f1-4520-a9b1-1f3a73678d26",
|
||||
"a1000001-0000-0000-0000-000000000012",
|
||||
"43098881-e044-482b-8e92-471728a8ba8b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1388.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"d6b12ecf-702b-4101-b1bb-09ed9b220b29",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"c608a095-c98b-4bfa-bfe1-1611c1320290",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"451ae007-4219-4096-89fe-fa2e045fbeb1",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1246.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1595.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1359.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"5a2c118a-87bd-4239-97a7-9e02c5991983",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"mem-ef878e30-5851-4e82-8588-745415108941",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1620.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a0edad47-5f77-4fc3-a546-1e85f8c68e77",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"96497334-b18f-495c-9228-eeb8182bdc38",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1478.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"75e036d3-c170-4e3f-acc2-e456a6850ee2",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1516.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-759e78ca-5394-4244-aa39-1c1468bc5f3e",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1237.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"bl-0d8c5dfa-e163-4fef-a58b-56b0d076c5a8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"tag-childhood"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1337.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"3499d5da-0e9c-4de4-9bc4-8941b14e0b1f",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1433.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1114.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1184.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1185.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1199.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-729fc901-8335-44c4-9f3a-b150b4aa0915",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"?Z?<S???K ?",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"'?T?a\"B~-?8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1422.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 703.8,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 488.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 720.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1301.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7ac62daa-2eac-4c7a-a97e-e4203fc1b57b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1651.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1021.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,942 +0,0 @@
|
||||
{
|
||||
"label": "act-r1",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-act",
|
||||
"soul_md5": "77722f5a9f49494bf735c2a4be1b5dc4",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 112.3,
|
||||
"child_pid": 78648,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.22857142857142856,
|
||||
"recall@5": 0.19087301587301586,
|
||||
"recall@10": 0.24277210884353742,
|
||||
"precision@5": 0.07428571428571429,
|
||||
"mrr@10": 0.24154195011337865,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "0/3",
|
||||
"latency_ms_p50": 3208.8,
|
||||
"latency_ms_p95": 4851.9,
|
||||
"latency_ms_max": 5078.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 2.6666666666666665
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.2857142857142857,
|
||||
"recall@5": 0.09722222222222222,
|
||||
"recall@10": 0.3567176870748299,
|
||||
"mrr@10": 0.3505668934240363
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 468.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 569.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 479.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 470.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 499.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 452.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-69fd6e83-7718-4824-8d66-f49d8954e224",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-eb1c6d99-d603-4f33-be9a-c63a178690c6",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"kn-b2a99cd7-b379-4d9b-a996-e347a02c7bad",
|
||||
"bl-76e878aa-e1fe-468c-bf9c-854097cb7e0b",
|
||||
"art-c71aef51-026f-4d63-80e9-2a0ec0dc3865"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1592.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"bl-80720fdf-7ce7-4d28-aff8-21028d3a8cfb",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"?Z?.\f?0?]P?",
|
||||
"?Q??m?;��`'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 956.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.125,
|
||||
"recall@10": 0.1875,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 6,
|
||||
"latency_ms": 970.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5555555555555556,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"bl-7aebe936-ac55-4f35-8932-adc5224ff854",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 920.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-8dbceb06-431a-416d-a723-e8c75d595154",
|
||||
"mem-a3124d5b-2f50-477f-8bb5-06879f5a496c",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"art-94fae615-7cd5-4695-b968-977101b06a51",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 925.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"mem-46780047-63a0-4a86-a16b-638b72a7fb8d",
|
||||
"kn-8e1bfb48-33a9-45ad-8da7-e0bdaa5d34e7",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023",
|
||||
"bl-a7a1428f-db9c-417b-8e2c-713b1f84dc1f",
|
||||
"mem-f823e835-313f-4282-b4b3-ce527ffc2f7a",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?Z?.\f?0?]P?",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2455.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.14285714285714285,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"art-92e1837c-5919-42d0-bbb0-4d924d7b2864",
|
||||
"bl-e20944e5-eb16-4ab3-a84d-111e0fc817fa",
|
||||
"bl-e20944e5-f4a6-44a0-91b1-73d04ebed120",
|
||||
"mem-47f72b5b-6e8b-4293-94f1-350197b4809a",
|
||||
"mem-e612f0aa-c2f2-4ee3-bbc7-af2dc826233b",
|
||||
"mem-a5f04e52-91f8-41d2-af27-8bf803621758",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2037.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"kn-82be4e41-96c5-4da3-85c0-cee10763d975",
|
||||
"mem-ea487cb4-ed67-44ce-8402-b56bb28468d4",
|
||||
"mem-23d22bc1-a097-446b-8f11-8aff099e0b76",
|
||||
"bl-874d1c2b-c55b-4afb-9601-922a9297e859",
|
||||
"bl-2dd8aaa1-b0de-4eac-b3c5-78951d240b60",
|
||||
"bl-2694b588-a6e3-43de-861c-fa7b0ec7e7fd",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4720.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"mem-0328c3cb-4550-4ce4-9284-152e832f08f6",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-fd047ce9-ae21-4b3e-b3ab-ece0c9592f7f",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"bl-6172d035-dd94-4776-afdd-d8915f6fc375",
|
||||
"bl-5bb8dedf-8498-4a9b-acdc-31cc9c738f2a",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 5078.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"bl-e5635b1a-c5d0-4caa-bcc8-6a726ea43685",
|
||||
"bl-fc893be3-e6b4-4ef6-93b0-d54ca5f89083",
|
||||
"tag-project-structure",
|
||||
"tag-anthropic-contrast",
|
||||
"tag-voice-training",
|
||||
"tag-ebd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"bl-9d53422d-b703-4f1d-860a-8598cb29b792",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4118.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-d24fcce8-2b55-426f-867a-db3958a622d3",
|
||||
"tag-phase-3",
|
||||
"tag-identity-studio",
|
||||
"tag-kids",
|
||||
"tag-coexistence",
|
||||
"tag-cultivated-general-intelligence",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"mem-cdff0c49-3ac7-4de8-89ec-92d254bd0023"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3241.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"bl-bd9fb314-e9d4-4b03-aef4-534dd57a2992",
|
||||
"bl-b019ce7a-1b21-436e-812d-032f50c6c45f",
|
||||
"bl-e98cdd4c-01b5-459e-9036-3578cd5d975a",
|
||||
"bl-9ce4128a-9436-4b06-82bc-8a6faafa81e0",
|
||||
"tag-stable-diffusion",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4090.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"mem-e321e54e-8bb3-4596-b13d-bb093d6b149d",
|
||||
"mem-e32ba5a7-c147-4dc0-9479-b720d768eda6",
|
||||
"mem-c7a77457-478d-4eb0-a116-67205a0066a4",
|
||||
"bl-1b20e9bc-eb37-4907-8d63-e311fd61eab8",
|
||||
"bl-aa762207-920d-45ab-b2a3-2f8154d7ef9b",
|
||||
"tag-misalignment",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3671.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"kn-48a01973-a025-471d-950f-b93e6a426d82",
|
||||
"bl-7f33f1bc-99fa-4906-889f-a42375beea20",
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"mem-ce5a2ffc-ad39-4728-9ac6-76fef507d5da",
|
||||
"project-Stripe_Elements__not_hosted_checkout__Custom_URL__DAG_bundle_pricing__Stripe_Connect_80_20_",
|
||||
"tag-provenance",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-e495c8c5-ad95-4b64-8771-f68aa4cfcd0a"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4674.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"kn-79056192-7de8-486c-9565-f128439a6fcb",
|
||||
"bl-f6236350-f7b8-4f4f-a702-9eef2eb76e4b",
|
||||
"mem-ef0091d8-1b65-431e-afa8-c6c4ee5779c9",
|
||||
"mem-37b57f52-a29a-42cf-a07a-3c5f8a3598dd",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"tag-__cultivation-metric____internal-state____dharma____evidence____novel-idea____gap-compression____values____microsoft__",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3974.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"bl-57c5cf6b-81a5-4558-9902-5c02981fe273",
|
||||
"tag-guilds",
|
||||
"tag-finance",
|
||||
"tag-temporal",
|
||||
"tag-barkhausen",
|
||||
"tag-ilogger",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4851.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"project-Goal_setting__alignment__scoring__cadence__Attaches_to_any_imprint_",
|
||||
"tag-turing-test",
|
||||
"tag-sealed",
|
||||
"tag-design-first",
|
||||
"tag-part-5",
|
||||
"tag-model",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4505.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-9590ba23-bddb-43e8-a571-68a263c4c364",
|
||||
"bl-a9e57bb2-00a1-4867-ab59-5d9271134b50",
|
||||
"tag-performed-values",
|
||||
"bl-c7793c4a-7630-47fc-a462-d23059087e80",
|
||||
"tag-gateway_platform_neuron-technologies_go_proxy_llm",
|
||||
"tag-fornax",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4494.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"bl-7a13527b-3e0c-418a-9f37-88fd2152e5ce",
|
||||
"bl-3f57bc69-7285-4f4a-a861-2de52efca058",
|
||||
"bl-5e390b10-8753-4f25-a1a5-b5dbbb002cbf",
|
||||
"tag-ats",
|
||||
"tag-data-model",
|
||||
"tag-aggregation",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3619.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"tag-memory-model",
|
||||
"tag-storage",
|
||||
"tag-ga4",
|
||||
"tag-potions",
|
||||
"tag-resonance",
|
||||
"tag-neuron",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4107.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"mem-927f41ab-8ede-4f58-acb3-995db16ac775",
|
||||
"mem-dbe80bc2-c602-46b0-b4ea-dd222e52bcde",
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"tag-upload-window",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4231.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"mem-481c769c-68cc-45c7-bc37-c0d9778fa648",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-9d8f3c5b-4bac-41ce-8ac4-44733f99d6c8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3208.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"project-Imprint__leadership_development__feedback_frameworks__performance__presence_",
|
||||
"mem-5e7f6ddd-c818-4ad3-b564-54ae278e9976",
|
||||
"bl-7fa1b1a8-b80a-4f28-b162-bfe73765b4f8",
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c",
|
||||
"tag-sarah",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2359.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"mem-265a7107-73b3-4410-9aff-43787d5f473b",
|
||||
"mem-9d1bf963-1b40-4588-bdb3-0432646cc623",
|
||||
"bl-0de4e61b-6562-49e5-b7df-ebb809a01723",
|
||||
"mem-3987d374-3c48-4e8e-b06d-0c363f55ed9c",
|
||||
"bl-e0a0df72-de6e-46ab-800b-e1e3e8dfc387",
|
||||
"tag-__patents____swarm____claim-language____prior-art____filing__",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2342.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-dca14c4c-4859-47b0-996e-33964ba61a87",
|
||||
"bl-e148d23c-24e8-4122-9915-d1c11f22052f",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"bl-dc8c7e02-eb37-48ae-a6f8-9b512803ae16",
|
||||
"mem-8d1bafe6-209c-456c-9a25-9a927bc5a16d",
|
||||
"bl-14883d81-f7cb-46dd-82c2-a6e6980264e5",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?Q??m?;��`'",
|
||||
"R^??m?;��?'"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3536.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-ffa22d7e-42a9-4bd2-a428-1d2df243ac93",
|
||||
"bl-452a4710-3d2b-4e0f-9413-49a66423bc9a",
|
||||
"bl-4a6746e8-191f-48fc-8bfb-c4dc73b80bcd",
|
||||
"tag-command-pattern",
|
||||
"tag-withholding",
|
||||
"tag-offline",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4151.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 1342.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 874.8,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"kn-333542cb-6dab-4662-9725-bf7440d28bf7",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4"
|
||||
],
|
||||
"n_returned": 8,
|
||||
"latency_ms": 1374.6,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 8
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"ctx-e5427d7d",
|
||||
"project-worldweaver",
|
||||
"tag-__kotlin____internal-state____pre-reasoning____post-reasoning____compression-ratio____dharma____cultivation__",
|
||||
"tag-import",
|
||||
"tag-enterprise",
|
||||
"tag-__cgi____dharma____cultivation____five-primitives____seed-artifact____agi____intelligence____whitepaper____patent__",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 3762.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-bfe0fafd-2750-4fdc-b773-04e878b3b23f",
|
||||
"art-7bdaff30-5af9-4f0a-93b1-751686f9de3d",
|
||||
"mem-cf07910d-4676-4384-ab97-9cad946cd0b9",
|
||||
"project-Convert_UTC_timestamps_to_Central_time_when_displaying_to_Will__Never_surface_raw_UTC_",
|
||||
"mem-32203649-3213-4d6d-86fd-3d657ac70d77",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"? ?}&?#??X\b",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 4859.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"project-Imprint__discovery__objection_handling__deal_strategy__pipeline__closing_",
|
||||
"bl-8116da7a-b039-4e08-b8d0-c1c7861f9766",
|
||||
"bl-8ef1ba6b-3fa0-4dbd-98c5-31665e5694a1",
|
||||
"tag-dark-theme",
|
||||
"tag-divisors",
|
||||
"tag-distressed-property",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"???Ͼd??W\b?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 2873.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "hybrid-semantic-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-hybrid",
|
||||
"soul_md5": "5cd2718932c2ecf4940ddfe5a9c8abbe",
|
||||
"corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7895,
|
||||
"wall_clock_s": 52.1,
|
||||
"child_pid": 86164,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1225.4,
|
||||
"latency_ms_p95": 1671.6,
|
||||
"latency_ms_max": 1718.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 307.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 349.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 316.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 315.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 326.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 304.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"????7???Ջ3",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 593.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 534.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 544.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 524.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 905.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 769.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1645.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1718.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1393.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1123.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-c2205725-69d0-4dd1-9a8d-1c7fa9a0c7b4",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1448.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"??o?'?B???k",
|
||||
"kn-150e6790-fc2a-48a7-8289-313c1fbaf5ae",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1298.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1621.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1405.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"??????X??2c",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1671.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1519.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1576.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"??S?7???",
|
||||
"??f?7???",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1287.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1391.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1498.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1169.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1239.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98",
|
||||
"7?e?7???\f3?",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2121fdb9-796a-427e-b9b5-651f4388ea16"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1225.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"rQ??m?;?x?'"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1237.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1458.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 756.3,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 519.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?m?\\}Q??6??",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?m?\\}Q??6??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 774.0,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1340.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1693.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1027.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,956 +0,0 @@
|
||||
{
|
||||
"label": "hybrid-semantic",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-hybrid",
|
||||
"soul_md5": "5cd2718932c2ecf4940ddfe5a9c8abbe",
|
||||
"corpus": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/corpus-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 52.1,
|
||||
"child_pid": 86109,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.5142857142857142,
|
||||
"recall@5": 0.4409013605442177,
|
||||
"recall@10": 0.5047619047619047,
|
||||
"precision@5": 0.15428571428571433,
|
||||
"mrr@10": 0.38746031746031745,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1219.7,
|
||||
"latency_ms_p95": 1667.1,
|
||||
"latency_ms_max": 1720.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.38461538461538464,
|
||||
"recall@5": 0.38461538461538464,
|
||||
"recall@10": 0.38461538461538464,
|
||||
"mrr@10": 0.17307692307692307
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.6634920634920636
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.2222222222222222,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 323.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"project-engram",
|
||||
"project-engram-lang",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-89a2",
|
||||
"bl-13babd0c-582e-4e28-a9e4-a77e65925e5d",
|
||||
"870ede67-3454-4e00-9988-46cb13a8a4e2",
|
||||
"bl-3e433255-3710-49fc-a093-c25e71de2ccb",
|
||||
"mem-235a7657-d49e-467e-9f69-f4c3d5f6bd48"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 330.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 318.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 316.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 328.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 296.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"????7???Ջ3",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 611.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 547.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 518.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 534.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 895.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 763.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1650.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1720.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1396.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1126.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-c2205725-69d0-4dd1-9a8d-1c7fa9a0c7b4",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1452.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"??o?'?B???k",
|
||||
"kn-150e6790-fc2a-48a7-8289-313c1fbaf5ae",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1303.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1618.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"dR????X?-?S"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1406.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1667.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"7?e?7???\f3?",
|
||||
"mem-ade9440f-f161-4c18-9b35-1976257e6ebb",
|
||||
"?of?7???",
|
||||
"ea95f600-8dfd-4c7e-b077-a93dc3cd3623"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1528.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1579.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"??S?7???",
|
||||
"??f?7???",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1288.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"7?e?7???\f3?",
|
||||
"mem-7b74cac0-905f-4c35-9688-fbcce105a177"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1383.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1499.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1167.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98",
|
||||
"7?e?7???\f3?",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2121fdb9-796a-427e-b9b5-651f4388ea16"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1219.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"rQ??m?;?x?'"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1235.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1464.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 761.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 525.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"?m?\\}Q??6??",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"?m?\\}Q??6??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 761.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1322.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"? ?}&?#??X\b",
|
||||
"be3b6036-6eca-44a7-8fdf-37b23edfdfd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1694.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666,
|
||||
"outranks": true,
|
||||
"rank_correct": 6,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"%???2??jH??",
|
||||
"art-8a0870d5-a716-4672-8094-f7463af1265b",
|
||||
"???Ͼd??W\b?",
|
||||
"bl-556438af-57b2-4bd8-a747-9f868aaee290"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1041.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,945 +0,0 @@
|
||||
{
|
||||
"label": "main-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main",
|
||||
"soul_md5": "caf822425540114984bfcd713ad37162",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7895,
|
||||
"wall_clock_s": 45.6,
|
||||
"child_pid": 78695,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1151.1,
|
||||
"latency_ms_p95": 1580.7,
|
||||
"latency_ms_max": 1663.2,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 231.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 266.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 229.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 235.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 254.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 228.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 466.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 474.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 460.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 477.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 824.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 681.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"????7???Ջ3",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1577.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1663.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1331.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1036.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1363.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1230.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1548.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1315.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1580.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"????7???Ջ3",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"%???2??jH??",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1474.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1490.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1192.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1320.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1448.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1107.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1151.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1154.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1187.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1413.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 684.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 444.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 696.8,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1617.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 967.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,945 +0,0 @@
|
||||
{
|
||||
"label": "main-r3",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main",
|
||||
"soul_md5": "caf822425540114984bfcd713ad37162",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7896,
|
||||
"wall_clock_s": 45.7,
|
||||
"child_pid": 78750,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1141.7,
|
||||
"latency_ms_p95": 1578.0,
|
||||
"latency_ms_max": 1645.5,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 237.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 272.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 231.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 231.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 262.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 239.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 465.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 470.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 441.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 455.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 816.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 678.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"????7???Ջ3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1564.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1645.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1320.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1029.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1347.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1220.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1560.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1305.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1578.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"????7???Ջ3",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"?ǚ?7??????",
|
||||
"7?e?7???\f3?",
|
||||
"??f?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1449.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1489.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1186.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1309.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1427.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1081.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1144.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1141.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1171.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1388.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 659.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 432.5,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 673.7,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1254.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1626.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 950.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,945 +0,0 @@
|
||||
{
|
||||
"label": "main-r1",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-main",
|
||||
"soul_md5": "caf822425540114984bfcd713ad37162",
|
||||
"corpus": "/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 45.9,
|
||||
"child_pid": 78554,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.34285714285714286,
|
||||
"recall@5": 0.26947278911564626,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.12000000000000001,
|
||||
"mrr@10": 0.2943197278911564,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1140.4,
|
||||
"latency_ms_p95": 1584.1,
|
||||
"latency_ms_max": 1627.6,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 240.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 271.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 232.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 227.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 259.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 237.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"?ǚ?7??????",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"art-d24fd6dd-2cda-4eed-92f3-67b535a0d71b",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"Z[?<S???H??",
|
||||
"rQ??m?;?x?'",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 467.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3125,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 475.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.5555555555555556,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"ԍ????X????",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"?ǚ?7??????",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"????7???Ջ3",
|
||||
"??????X??2c",
|
||||
"g?e?7???'c?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 449.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.14285714285714285
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"mem-a9a9ce95-0d64-46eb-9db8-ff81d78ade35",
|
||||
"mem-b8ecd23e-77ce-42f7-984c-f51453fec16d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 463.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 821.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 688.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"????7???Ջ3",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1559.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1627.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"?of?7???",
|
||||
"????7???Ջ3",
|
||||
"kn-d97920d0-1649-4223-9508-c0bb621e7fc0",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-81c24d13-a73b-4767-819c-dafaacc1498e",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1317.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1055.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-83bb86c6-521d-416c-a86e-6e29c2d8f102",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1352.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0625e393-067c-4bba-8389-7e1b79265142",
|
||||
"ע?RGk?\tH(?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1213.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1545.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"??????X??2c",
|
||||
"ԍ????X????",
|
||||
"dR????X?-?S",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"dz????Xƹ?i",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1327.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"dR????X?-?S",
|
||||
"dz????Xƹ?i",
|
||||
"ԍ????X????",
|
||||
"??????X??2c",
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1584.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"????7???Ջ3",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"?ǚ?7??????",
|
||||
"7?e?7???\f3?",
|
||||
"??f?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1432.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"'?T?a\"B~-?8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1505.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"?ǚ?7??????",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"??S?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"??f?7???",
|
||||
"??S?7???",
|
||||
"knw-5578cb21-e899-4822-b7f4-0d96fa094e3d"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1201.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"7?e?7???\f3?",
|
||||
"kn-e8423822-eacf-4029-aa7b-10d4d28d621e",
|
||||
"? ?}&?#??X\b",
|
||||
"??o?'?B???k",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1303.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?ǚ?7??????"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1416.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1090.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1167.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"7?e?7???\f3?",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"? ?}&?#??X\b",
|
||||
"h??I?cB?Q??",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1140.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"7?e?7???\f3?",
|
||||
"rQ??m?;?x?'",
|
||||
"?Q??m?;?u?'",
|
||||
"R^??m?;��?'",
|
||||
"?Q??m?;��`'",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1157.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1379.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 677.2,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 443.2,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-66a21179-2adc-4b19-a109-880cf4674d7d",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 679.8,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"art-80ca3d31-84dc-4502-83f4-538372b9764f"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"? ?}&?#??X\b",
|
||||
"7?e?7???\f3?",
|
||||
"?of?7???",
|
||||
"?ǚ?7??????",
|
||||
"[?MO5????G",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1618.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"7?e?7???\f3?",
|
||||
"? ?}&?#??X\b",
|
||||
"%???2??jH??",
|
||||
"???Ͼd??W\b?",
|
||||
"??o?'?B???k",
|
||||
"? ?}&?#??X\b",
|
||||
"63307ac5-cf6b-46e0-8296-07503b461cfa",
|
||||
"? ?}&?#??X\b",
|
||||
"?of?7???"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 956.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,957 +0,0 @@
|
||||
{
|
||||
"label": "semseed-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-semseed",
|
||||
"soul_md5": "ab0c82781215f43d4907bd6f39d3615b",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semseed/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 52.0,
|
||||
"child_pid": 88833,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1245.9,
|
||||
"latency_ms_p95": 1668.7,
|
||||
"latency_ms_max": 1713.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 306.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 349.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 294.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 319.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 339.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 318.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 604.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 534.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 541.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 515.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 532.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 887.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 762.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1659.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1713.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"?of?7???",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"????7???Ջ3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1402.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"7?e?7???\f3?",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"??o?'?B???k",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1129.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"?of?7???",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"ctx-cc7f",
|
||||
"ctx-4a41",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1452.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1293.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1622.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1412.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"?of?7???",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1668.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1535.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1565.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1279.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1408.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1538.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1168.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1290.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"ctx-bb74",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1245.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1277.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.09090909090909091,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1486.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 757.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 510.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 764.0,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1340.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1701.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"%???2??jH??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1029.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,957 +0,0 @@
|
||||
{
|
||||
"label": "semseed",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-semseed",
|
||||
"soul_md5": "ab0c82781215f43d4907bd6f39d3615b",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-semseed/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7893,
|
||||
"wall_clock_s": 52.7,
|
||||
"child_pid": 88601,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.6857142857142857,
|
||||
"recall@5": 0.5213459159887731,
|
||||
"recall@10": 0.6027048348476919,
|
||||
"precision@5": 0.18285714285714294,
|
||||
"mrr@10": 0.4608730158730158,
|
||||
"nonsense_clean": "2/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 1227.1,
|
||||
"latency_ms_p95": 1692.6,
|
||||
"latency_ms_max": 1710.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.07342657342657344,
|
||||
"recall@10": 0.24825174825174823,
|
||||
"mrr@10": 0.20833333333333334
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 2,
|
||||
"avg_false_positives": 3.3333333333333335
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.7142857142857143,
|
||||
"recall@5": 0.40093537414965985,
|
||||
"recall@10": 0.5150226757369615,
|
||||
"mrr@10": 0.6507936507936508
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 295.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 341.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 313.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 310.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 318.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 310.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"kn-363f4976-6946-4b4d-b51b-8a2b0f5aef25",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"ctx-63e3",
|
||||
"?ǚ?7??????",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 610.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 537.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"ԍ????X????",
|
||||
"project-harmonic-framework",
|
||||
"?ǚ?7??????",
|
||||
"project-harmonic-framework_com",
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"??????X??2c",
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"????7???Ջ3",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 518.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 531.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.5,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.1111111111111111
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 888.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.2857142857142857,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.8,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"?",
|
||||
"mem-dba009a2-d2ea-4f5a-b9e8-0f04bc9ab32f",
|
||||
"?",
|
||||
"mem-7cd90611-88a3-423d-a38a-0db2812952fa",
|
||||
"?",
|
||||
"bl-07375bf9-a169-42cd-adb3-7d32b25982f0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 762.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1652.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-2f29ad36-6ee6-4a0e-8d72-0eaf7d12d3a9",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"art-2fabd873-d787-49cb-ad30-d4ed9fcff8ef",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-a31e1001-342e-4deb-a2e6-6d02d1f22dee",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1706.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"?of?7???",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"????7???Ջ3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1419.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"kn-d7c1e0fb-fa59-46d3-b4c9-a0d1d437a491",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"7?e?7???\f3?",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"??o?'?B???k",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1167.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"?of?7???",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"imp-dce1da0f-8776-4a9e-972b-33411a7ca138",
|
||||
"ctx-cc7f",
|
||||
"ctx-4a41",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1489.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"knw-e24d6339-5ff3-4bed-ba53-707ffd0dc70a",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-57b4c5e7-40c6-4c90-bf14-71841b0081d4",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1330.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"? ?}&?#??X\b",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1671.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"? ?}&?#??X\b",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1449.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-f8974b26-78a6-4aad-b893-19a73b20013d",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-9397c74b-35f3-4428-b4b0-5123353bbcd1",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"?of?7???",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1710.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"????7???Ջ3",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"?ǚ?7??????",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"7?e?7???\f3?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1565.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"mem-bbb126a1-b297-42bb-86be-796871829c94",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"mem-45022957-2d78-48aa-a714-16d6eca52e0f",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1629.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"?ǚ?7??????",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"7?e?7???\f3?",
|
||||
"ctx-bb74",
|
||||
"??S?7???",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1326.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1382.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1530.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-e0bdf5d8-d163-491f-b649-453fee8b721d",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1159.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"? ?}&?#??X\b",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1232.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.36363636363636365,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"art-ee615cdb-e599-423d-9a4d-977859390ed3",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"ctx-e5427d7d",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"ctx-bb74",
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-8c2d5f51-3ccd-4c2e-848a-eb60d90a3b98"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1227.1,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"knw-9e74ee95-ba7d-49b1-9262-977eae9729d1",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b"
|
||||
],
|
||||
"n_returned": 9,
|
||||
"latency_ms": 1240.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.09090909090909091,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1464.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.2727272727272727,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.16666666666666666
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 756.0,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 517.1,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [
|
||||
"art-4a99aa1a-489b-4b43-958b-25217adb1aad",
|
||||
"?V?",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"?m?\\}Q??6??",
|
||||
"8cbb60c5-4999-4ec1-8682-2592aedc4249",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"?m?\\}Q??6??",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 755.8,
|
||||
"error": null,
|
||||
"clean": false,
|
||||
"false_positives": 10
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"7?e?7???\f3?",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"? ?}&?#??X\b",
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"? ?}&?#??X\b",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"?of?7???",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"? ?}&?#??X\b",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1329.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 10
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"? ?}&?#??X\b",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"? ?}&?#??X\b",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"? ?}&?#??X\b",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"? ?}&?#??X\b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1692.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"kn-5584ef9c-7f9d-4d7c-a10a-4ee6bc5cf356",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"7?e?7???\f3?",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"? ?}&?#??X\b",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"%???2??jH??"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 1040.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,948 +0,0 @@
|
||||
{
|
||||
"label": "wordstart-r2",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-wordstart",
|
||||
"soul_md5": "32d4cf77672658a5f49dc7c9213e3ba2",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7895,
|
||||
"wall_clock_s": 28.7,
|
||||
"child_pid": 1490,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "3/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 537.6,
|
||||
"latency_ms_p95": 733.7,
|
||||
"latency_ms_max": 761.4,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 3,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 165.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 164.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 163.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 172.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 167.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 164.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 281.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 254.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 255.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 247.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 263.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"?Q??m?;?u?'",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 397.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"mem-fb44a2fc-7405-41ff-87b3-84643ac07313",
|
||||
"?",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 341.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-0228da71-d7f7-4f3b-b7b3-c5eede42b62a",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 761.4,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7452eb8f-be01-4b55-aec1-ff0c29e790f6",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"532277bf-2959-4beb-ae0d-b018c97678ee",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"d4015bd7-c592-4ed8-8574-1f15ad37af75"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 733.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"tag-fiction",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-ca4d6a34-d354-413f-bc86-126cc17ca81c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 629.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"4da5dbaf-46e5-4f3e-b474-f60d9f8241d3",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 529.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"bfad516b-c306-4c4c-874a-a347c46c05c2",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 747.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"345b6420-e004-4d2e-b55c-6a729393fa99",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-92a7fdc5-9dd0-48cf-a691-506058de3838",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 564.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 689.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 588.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"34356a36-0df5-4020-8dcc-5e7a423f8d4c",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 729.0,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-1b58b05c-9305-4f06-a586-a08c96008027",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"ctx-4a41",
|
||||
"mem-5708f4c9-3d61-4182-8543-2843698931e6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 638.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 679.0,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-a16deccb-16a7-419c-a013-ff824a4daa15",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 588.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"tag-childhood",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 613.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"efe53612-6914-4936-8e3b-1e694eb174e5",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 652.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 543.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 537.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 538.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 533.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"719aa819-00a9-4f4b-a857-4f9fe5ad44d7",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 673.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 350.7,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 254.6,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 345.8,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-c17aefb1-38b5-4ced-af50-fe524127e1a4",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 639.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"de3b6428-b76c-4c44-90e0-bf1dd6998027"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 733.7,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 507.9,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,948 +0,0 @@
|
||||
{
|
||||
"label": "wordstart",
|
||||
"soul_binary": "/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad/soul-wordstart",
|
||||
"soul_md5": "32d4cf77672658a5f49dc7c9213e3ba2",
|
||||
"corpus": "/Users/timlingo/neuron-eval-corpora/snapshot-pre-repair-20260806-embedded.json",
|
||||
"corpus_nodes": 78768,
|
||||
"corpus_edges": 14214,
|
||||
"gold_set": "/Users/timlingo/Development/neuron-technologies/_wt-eval/tools/retrieval-eval/gold_set.json",
|
||||
"limit": 10,
|
||||
"port": 7894,
|
||||
"wall_clock_s": 28.9,
|
||||
"child_pid": 1420,
|
||||
"child_confirmed_dead": true,
|
||||
"aggregate": {
|
||||
"n_queries": 38,
|
||||
"n_scored": 35,
|
||||
"hit@5": 0.7428571428571429,
|
||||
"recall@5": 0.5536485340056769,
|
||||
"recall@10": 0.6175677497106068,
|
||||
"precision@5": 0.20000000000000007,
|
||||
"mrr@10": 0.5021428571428571,
|
||||
"nonsense_clean": "3/3",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 542.6,
|
||||
"latency_ms_p95": 741.3,
|
||||
"latency_ms_max": 758.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 3,
|
||||
"clean": 3,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "q01",
|
||||
"category": "exact_rare",
|
||||
"query": "unjailbreakable",
|
||||
"returned": [
|
||||
"mem-7f61beb4-271c-4feb-9f6e-1c9c837a6226"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 170.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q02",
|
||||
"category": "exact_rare",
|
||||
"query": "engram-migrate",
|
||||
"returned": [
|
||||
"mem-6fdf6545-5e1a-43a9-8bdc-d2cd248146a5",
|
||||
"bl-ba764d70-e9d7-4f62-848f-719cb665f45e",
|
||||
"mem-22fe5ec8-ae0d-4583-a05c-d1ef50353257",
|
||||
"bl-b28d7256-6f74-4567-bd90-40d0ef2a6d78",
|
||||
"project-engram",
|
||||
"ctx-45bc",
|
||||
"project-engram-lang",
|
||||
"ctx-175f",
|
||||
"mem-60778715-758c-4677-933d-fc39b8f94152",
|
||||
"ctx-74ed"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 210.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q03",
|
||||
"category": "exact_rare",
|
||||
"query": "cartabandonedevent",
|
||||
"returned": [
|
||||
"mem-1ba7c67d-85b9-4c2e-9fe2-39f8b0477091"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 167.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q04",
|
||||
"category": "exact_rare",
|
||||
"query": "pre-apprenticeship",
|
||||
"returned": [
|
||||
"mem-89c02aae-d3ca-43f9-9e5d-eb369896276c"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 166.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q05",
|
||||
"category": "exact_rare",
|
||||
"query": "inferencenodemanager",
|
||||
"returned": [
|
||||
"mem-73969486-143f-4431-b5e6-6845d1cc9848",
|
||||
"bl-c1765767-3e27-449a-8c94-10411d1eb7c0",
|
||||
"project-Add_inference_url_config_to_Neuron_MCP__Route_summarization_gen_tasks_to_Pantheon__keep_frontier_for_complex_reasoning_"
|
||||
],
|
||||
"n_returned": 3,
|
||||
"latency_ms": 162.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q06",
|
||||
"category": "exact_rare",
|
||||
"query": "clear-eyed",
|
||||
"returned": [
|
||||
"knw-c72597c5-c23d-4c08-8e9e-996dadf26a99"
|
||||
],
|
||||
"n_returned": 1,
|
||||
"latency_ms": 162.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q07",
|
||||
"category": "phrase",
|
||||
"query": "patterns not returns",
|
||||
"returned": [
|
||||
"tag-patterns",
|
||||
"mem-a4a9dfc3-e40b-49b3-b1e1-060e8be2f482",
|
||||
"project-Imprint__system_design__ADRs__tech_strategy__integration_patterns__governance_",
|
||||
"project-Imprint__analysis_patterns__data_storytelling__SQL__dashboards__insight_framing_",
|
||||
"bl-79028eed-c330-4724-9402-734062d13503",
|
||||
"bl-39dad13d-7105-4049-8224-dc3c34fdb1f3",
|
||||
"bl-4ef4d914-da46-4e0f-be78-5219b9547e9f",
|
||||
"bl-7e7c3fdb-4132-487f-aa70-b2cd559cb7f0",
|
||||
"bl-1d32bd54-cf17-4a1f-b235-982d09a36f04",
|
||||
"bl-b8af6601-a8cb-41b5-aef5-ab8a57432dd5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 306.7,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q08",
|
||||
"category": "phrase",
|
||||
"query": "thirty moves",
|
||||
"returned": [
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 261.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1875,
|
||||
"recall@10": 0.375,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q09",
|
||||
"category": "phrase",
|
||||
"query": "Grandma Lucas",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"art-0f0277a1-4a8e-4645-95dd-fa379976f31c",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 272.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.4444444444444444,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"category": "phrase",
|
||||
"query": "Directed Harmonic",
|
||||
"returned": [
|
||||
"bl-dcee1887-34c4-4ffa-9119-1e291685ba08",
|
||||
"project-harmonic-framework",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"mem-7eeacad7-d7c2-4c2b-8348-19a59aa6dbaf",
|
||||
"project-harmonic-framework_com",
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"tag-harmonic-design",
|
||||
"bl-92acd4eb-0452-4e8e-9f54-f8cd35170d76",
|
||||
"tag-harmonic-framework",
|
||||
"bl-18a9d1e4-1484-474c-bf6b-c6173212181b"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 265.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.1111111111111111,
|
||||
"recall@10": 0.1111111111111111,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q11",
|
||||
"category": "phrase",
|
||||
"query": "Sarah Bishop",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-sarah",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"mem-1f32f73a-952c-41bc-96dc-8b8b70d8a7c1",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-a6cb3b8d-d89c-46fc-931d-e90c560783b0",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 271.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q12",
|
||||
"category": "phrase",
|
||||
"query": "Directed Autonomous Runtime Modification",
|
||||
"returned": [
|
||||
"mem-e6327f52-2bda-4ce7-9471-2fffd1e172de",
|
||||
"bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5",
|
||||
"bl-6f99e111-7055-4635-9831-a489747ce418",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-967536a0-d49d-44fb-8cfb-b31b40bcbfae",
|
||||
"bl-8b58d9bc-352b-4842-a7f8-a6254b5d1e25",
|
||||
"?Q??m?;?u?'",
|
||||
"bl-39cec462-c80c-4970-a3aa-91fe83053bde"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 412.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.21428571428571427,
|
||||
"recall@10": 0.2857142857142857,
|
||||
"precision@5": 0.6,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q13",
|
||||
"category": "phrase",
|
||||
"query": "zero-knowledge encrypted backup",
|
||||
"returned": [
|
||||
"deda48cd-5e1a-46cb-bd43-8016afdb3a8a",
|
||||
"8f3abb0d-77ed-4af3-9f4d-ba62cd198886",
|
||||
"7774a16c-1027-4e3b-a21e-67f1f95a4acd",
|
||||
"?",
|
||||
"bl-ec84b63d-b278-4944-8d7f-4aa7a51c0315",
|
||||
"?",
|
||||
"mem-fb44a2fc-7405-41ff-87b3-84643ac07313",
|
||||
"?",
|
||||
"mem-a3c97012-5fa3-4915-a839-2c75c72005e0",
|
||||
"?"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 377.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
{
|
||||
"id": "q14",
|
||||
"category": "paraphrase",
|
||||
"query": "the elderly relative who passed while he stayed away",
|
||||
"returned": [
|
||||
"cb070131-dfd4-4a38-91d7-22b1bde164d2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"knw-d788a210-613b-4c49-9486-88bbc9d4716f",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-a535f205-bc4c-4058-9171-6263c496044a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-0228da71-d7f7-4f3b-b7b3-c5eede42b62a",
|
||||
"ctx-4a41"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 758.8,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q15",
|
||||
"category": "paraphrase",
|
||||
"query": "a soldier sidelined by illness who refused to quit",
|
||||
"returned": [
|
||||
"b1183213-d659-4759-85d7-5b1f22427fe2",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-16efddd1-c43d-4a42-9d78-f54fb82bd277",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"7452eb8f-be01-4b55-aec1-ff0c29e790f6",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"532277bf-2959-4beb-ae0d-b018c97678ee",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"d4015bd7-c592-4ed8-8574-1f15ad37af75"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 741.3,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q16",
|
||||
"category": "paraphrase",
|
||||
"query": "choosing an uncomfortable fact over a pleasant fiction",
|
||||
"returned": [
|
||||
"tag-fiction",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-8d690e9d-a7e9-4062-b2f8-e2064294e463",
|
||||
"knw-8fd9836c-cc39-49df-8d61-babda626cc88",
|
||||
"mem-ce793303-c5a5-4586-a232-a3426edd9ec7",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"mem-443bd012-fc9a-4088-b236-de5157a1ef92",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"mem-ca4d6a34-d354-413f-bc86-126cc17ca81c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 642.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q17",
|
||||
"category": "paraphrase",
|
||||
"query": "a tight payload beats a bloated one",
|
||||
"returned": [
|
||||
"bl-680b24a9-edc3-4a9d-847a-bff0b46b568c",
|
||||
"bl-798d135f-3987-4ccd-8de6-70ca2f358337",
|
||||
"08f0d1e2-8d0e-42e3-9f0a-8186ae31ec7e",
|
||||
"knw-08559f5c-2306-4220-a146-398c74f1643c",
|
||||
"4da5dbaf-46e5-4f3e-b474-f60d9f8241d3",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"mem-434be7c8-88cb-4039-b79a-1da4ac4de783",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"bl-79ce4464-5dd6-49bd-9b0c-9803549d0665"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 537.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q18",
|
||||
"category": "paraphrase",
|
||||
"query": "if you are able and nobody is coming the job is yours",
|
||||
"returned": [
|
||||
"bl-4476e856-c567-4b49-8ff7-d7dca3e5715e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-e5cc63c0-8701-49d6-855a-e387fe087771",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"mem-75e490d1-f0a9-4b73-8cfc-8daecfaf6f38",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"bfad516b-c306-4c4c-874a-a347c46c05c2",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 725.1,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q19",
|
||||
"category": "paraphrase",
|
||||
"query": "learning is the wealth creditors cannot seize",
|
||||
"returned": [
|
||||
"tag-learning",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"345b6420-e004-4d2e-b55c-6a729393fa99",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-92a7fdc5-9dd0-48cf-a691-506058de3838",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000010",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 562.8,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q20",
|
||||
"category": "paraphrase",
|
||||
"query": "reliability proven by track record not assertion",
|
||||
"returned": [
|
||||
"mem-cde58b77-50d3-4bac-9581-e70a4c02c015",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-8d699e2c-ac2a-4742-bb62-b6da00f4b10e",
|
||||
"knw-f671966c-3387-4848-abca-b5deec122e00",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"mem-53d6adf0-cd08-4707-a237-daa5e65c7298",
|
||||
"a708dd6e-fe73-4f2f-a21e-89daa0985487",
|
||||
"knw-6b48dce2-f21c-452a-9db5-4e6aa61c87ca",
|
||||
"bl-ef2bac68-e119-4139-b529-c7a1404ae3ac"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 686.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q21",
|
||||
"category": "paraphrase",
|
||||
"query": "boundaries that enable instead of confine",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"bl-0e8f4880-7b24-43aa-aed9-ad4d9fc73ff8",
|
||||
"project-Source_kn-6f248a50__Add_containment_rules__convergence__location-independence__failure_modes_",
|
||||
"bl-8848929a-a23a-46bc-a2c7-fe3a3bc1cddf",
|
||||
"bl-205141ad-b2a0-4d93-86d0-89eb0723e1bd",
|
||||
"bl-e93858c4-7cac-4b1a-bb62-490790d4c3f3",
|
||||
"bl-34f51ddb-a840-459f-a248-94214f5febb6",
|
||||
"bl-286b562a-5299-40e0-a32a-afa9cbdfe995"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 584.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q22",
|
||||
"category": "paraphrase",
|
||||
"query": "what shifts tells you where to cut a system apart",
|
||||
"returned": [
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
|
||||
"bl-4c5b385e-135a-4663-8521-96af0b491121",
|
||||
"knw-12b4b913-7a25-4b0d-844c-504c01d6725e",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"knw-9707256e-ed44-4042-bd88-f90fa514e1cf",
|
||||
"34356a36-0df5-4020-8dcc-5e7a423f8d4c",
|
||||
"knw-0087493b-25cd-45b0-bf46-c078c5b49718",
|
||||
"mem-22f5f665-3ad2-4063-88b0-915849a795f5",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 719.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q23",
|
||||
"category": "paraphrase",
|
||||
"query": "a mind that compounds instead of resetting each day",
|
||||
"returned": [
|
||||
"bl-8dd70cac-866d-4ff2-b9fe-b4b3c5f094bb",
|
||||
"mem-b43f6ef4-2f5a-418d-b5ce-3f21520cf6b8",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-ed33e669-0790-44cb-a036-958d605c6fea",
|
||||
"a1000001-0000-0000-0000-000000000001",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-1b58b05c-9305-4f06-a586-a08c96008027",
|
||||
"mem-024598a9-ed2e-4eeb-b1e1-5410856ff132",
|
||||
"ctx-4a41",
|
||||
"mem-5708f4c9-3d61-4182-8543-2843698931e6"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 634.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q24",
|
||||
"category": "paraphrase",
|
||||
"query": "loved for the unedited self and not the polished exterior",
|
||||
"returned": [
|
||||
"077d064f-3489-4c05-9aca-3782f96b51db",
|
||||
"knw-f9ce17a7-17fc-431f-8f23-695b670ec4fa",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"27e1b1a4-ad0b-49d9-812f-fedf43b8aabe",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"bl-87c93185-b2bf-40af-ae23-3c830c007abf",
|
||||
"knw-920c891f-bb8c-48c4-9afc-018ef12dcdc4",
|
||||
"fce2792a-53fc-4d4a-be3b-42bd6ceb1ba7",
|
||||
"a1000001-0000-0000-0000-000000000001"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 674.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q25",
|
||||
"category": "paraphrase",
|
||||
"query": "cheerfulness you arrive at instead of assuming",
|
||||
"returned": [
|
||||
"mem-82158b02-a180-435d-84f0-0b7ce37511b4",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"e4f27651-52c5-43fd-aff3-61d31685b3cd",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"mem-5624ec9d-62ba-4aba-8a3d-6afec6c09dd4",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"mem-a16deccb-16a7-419c-a013-ff824a4daa15",
|
||||
"a1000001-0000-0000-0000-000000000009",
|
||||
"mem-833dbbcd-2400-4594-bb35-93b023049ac0",
|
||||
"kn-13f60407-7b70-4db1-964f-ea1f8196efbd"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 571.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q26",
|
||||
"category": "paraphrase",
|
||||
"query": "a childhood offering no solid foundation to inherit",
|
||||
"returned": [
|
||||
"mem-b99efff0-00e6-40c8-9c5b-730330eef33b",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"knw-2c46cfb4-6d4e-4822-8a1a-7d743c1e4329",
|
||||
"knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"knw-23c27d3b-e0d2-43a8-a80c-0a44477ae18a",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-f230b362-b201-4402-9833-4160c89ab3d4",
|
||||
"tag-childhood",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 603.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.2
|
||||
},
|
||||
{
|
||||
"id": "q27",
|
||||
"category": "associative",
|
||||
"query": "Grandma Lucas stroke February 2006 goodbye window",
|
||||
"returned": [
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"efe53612-6914-4936-8e3b-1e694eb174e5",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 656.2,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q28",
|
||||
"category": "associative",
|
||||
"query": "Marines hernia sepsis medical ward",
|
||||
"returned": [
|
||||
"bl-33ecccc2-e37f-43db-91b3-c2a86f08aaac",
|
||||
"54608b69-78b6-4239-b60f-b8206cfecacc",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"mem-9110798f-d0cb-4446-bc2a-14f09b6a09e2",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"art-79042b8b-6192-440f-90b0-60708f7e6325",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"art-ddfcd045-2c3b-4a1e-9966-fec5ce44e1dd",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 536.9,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.07692307692307693,
|
||||
"recall@10": 0.3076923076923077,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q29",
|
||||
"category": "associative",
|
||||
"query": "Sarah Bishop Dyer trailer performance",
|
||||
"returned": [
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
|
||||
"766de879-f9d0-4a07-b6df-b43ee13763d8",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"tag-trailer-park-paladins",
|
||||
"kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
|
||||
"project-trailer-park-paladins",
|
||||
"kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 562.4,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.18181818181818182,
|
||||
"recall@10": 0.45454545454545453,
|
||||
"precision@5": 0.4,
|
||||
"mrr@10": 0.5
|
||||
},
|
||||
{
|
||||
"id": "q30",
|
||||
"category": "associative",
|
||||
"query": "Swarm Architecture containment lateral worker",
|
||||
"returned": [
|
||||
"bl-0fac287f-f4c0-4f15-bc4d-ff7f8a7af3ae",
|
||||
"bl-9bde67c1-f0ba-4c3a-8fe5-de0deee0ce43",
|
||||
"bl-2515d870-e35e-443b-ba20-5150bbc73fed",
|
||||
"kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
|
||||
"kn-b36902cc-0b05-44ba-9aa7-800e5dea9ca9",
|
||||
"bl-bea7473c-c687-414c-9c0b-00c509a616c1",
|
||||
"bl-fc6fcb0b-9e4b-40bf-8e88-dbfe4e27c31a",
|
||||
"kn-6f248a50-355b-47bb-aec8-e0e646a9b077",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930",
|
||||
"mem-ab34c2f7-3243-424b-affa-25555f6cf9cc"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 556.6,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q31",
|
||||
"category": "associative",
|
||||
"query": "hope won inside the narrative preface",
|
||||
"returned": [
|
||||
"kn-e0423482-cfa5-4796-8689-8495c93b66bc",
|
||||
"knw-e94982a2-358d-4f2f-af31-8ee0fcec07c6",
|
||||
"bl-2b00aeb0-c0fa-4a9f-8f30-4207e98b3d52",
|
||||
"kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
|
||||
"kn-5b606390-a52d-4ca2-8e0e-eba141d13440",
|
||||
"knw-4aebd815-4eaf-49d7-954b-03595f3d48be",
|
||||
"knw-723551f5-1950-42a3-8b89-b6a06913cef0",
|
||||
"tag-hope",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 542.6,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.09090909090909091,
|
||||
"recall@10": 0.18181818181818182,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
{
|
||||
"id": "q32",
|
||||
"category": "associative",
|
||||
"query": "man of the house six years old expectation",
|
||||
"returned": [
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"knw-528dbc37-eabc-4b75-a7a5-65bf38d6018a",
|
||||
"knw-d357b6bb-ad8a-4791-b516-426aea45fa5b",
|
||||
"knw-473f3f24-20f6-4f39-8589-3709538eb6ac",
|
||||
"mem-a0b7cfda-bc9e-4f40-b9a9-1722cf3f8263",
|
||||
"knw-7902acca-604e-409b-8faf-ad85424211d0",
|
||||
"knw-35940684-abc4-42f0-b942-818f66b1f69a",
|
||||
"4f698ae6-c40e-464e-9798-50350991a188",
|
||||
"719aa819-00a9-4f4b-a857-4f9fe5ad44d7",
|
||||
"kn-6061318f-046b-4935-907d-8eafdce14930"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 683.5,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
{
|
||||
"id": "q33",
|
||||
"category": "nonsense",
|
||||
"query": "zqxjvw plimforth grebulon",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 350.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q34",
|
||||
"category": "nonsense",
|
||||
"query": "flarnbistle quommetry",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 258.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q35",
|
||||
"category": "nonsense",
|
||||
"query": "xxqzzt vurblenacht throom",
|
||||
"returned": [],
|
||||
"n_returned": 0,
|
||||
"latency_ms": 348.4,
|
||||
"error": null,
|
||||
"clean": true,
|
||||
"false_positives": 0
|
||||
},
|
||||
{
|
||||
"id": "q36",
|
||||
"category": "superseded",
|
||||
"query": "is the self-improvement architecture called DARMA or DHARMA",
|
||||
"returned": [
|
||||
"mem-f3b37427-b7d1-4f7e-b32c-0241a20ce8da",
|
||||
"mem-80d7416b-20e9-48a0-b176-b215527e2f56",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____architecture__",
|
||||
"kn-b7e98d63-8b83-4911-b4d0-990602a7f575",
|
||||
"tag-__darma____cgi____patents____self-improvement____character-preservation____autonomous____kotlin____architecture__",
|
||||
"knw-e047bb42-dc5b-4383-9e88-e508dc03abe3",
|
||||
"mem-c17aefb1-38b5-4ced-af50-fe524127e1a4",
|
||||
"bl-5b17bd3b-0c41-46cb-a710-6fa4429692ff",
|
||||
"bl-145a0985-2382-400f-a7c5-c335c5e30a72",
|
||||
"mem-82b93b21-a865-410f-9ec1-fc54121d9bb5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 664.5,
|
||||
"error": null,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.2,
|
||||
"mrr@10": 0.5,
|
||||
"outranks": true,
|
||||
"rank_correct": 2,
|
||||
"rank_stale": 8
|
||||
},
|
||||
{
|
||||
"id": "q37",
|
||||
"category": "superseded",
|
||||
"query": "how many provisional patents does Will actually have",
|
||||
"returned": [
|
||||
"mem-6f0b2b45-90c1-4356-ac01-3daac05b09c8",
|
||||
"12082f7e-e320-438b-bd65-083d8259748f",
|
||||
"015644f5-8194-4af0-800d-dd4a0cd71396",
|
||||
"13705072-4515-4124-963d-083af490494f",
|
||||
"527ecb25-2587-47eb-8269-73be2431abd4",
|
||||
"6de314bf-5c4c-4cfc-871f-fa2e422d45e6",
|
||||
"a1000001-0000-0000-0000-000000000002",
|
||||
"3cf706a1-3825-45d8-b0a9-06cae6cdf5b8",
|
||||
"kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
|
||||
"de3b6428-b76c-4c44-90e0-bf1dd6998027"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 747.3,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 1.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.125,
|
||||
"outranks": true,
|
||||
"rank_correct": 8,
|
||||
"rank_stale": null
|
||||
},
|
||||
{
|
||||
"id": "q38",
|
||||
"category": "superseded",
|
||||
"query": "is MCP still the live integration layer",
|
||||
"returned": [
|
||||
"5fcba804-eb5b-48ec-82da-146b1c6bb50d",
|
||||
"bl-7328cbe3-0200-43c2-88e7-0a164e15fca4",
|
||||
"bl-c8c19362-430b-4817-9cf4-9e85e0099c64",
|
||||
"bl-c5c6571e-118f-47c7-8cbb-3ed0ebf64a51",
|
||||
"mem-101e81b4-8097-4749-8d8d-7bb66de34517",
|
||||
"ctx-3a55",
|
||||
"86228228-7adf-41fb-b4c4-9ceea87953ae",
|
||||
"4509ed62-9fb2-48b8-9038-ac569fca9604",
|
||||
"bl-4f7b651b-6b33-449c-8a3b-cfce12ce984b",
|
||||
"mem-3a2cf162-d93b-4f29-86f2-5066fb7fe1f5"
|
||||
],
|
||||
"n_returned": 10,
|
||||
"latency_ms": 504.2,
|
||||
"error": null,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"precision@5": 0.0,
|
||||
"mrr@10": 0.0,
|
||||
"outranks": false,
|
||||
"rank_correct": null,
|
||||
"rank_stale": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,98 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# run_comparison.sh — the whole harness, end to end, from two git refs.
|
||||
#
|
||||
# Builds a soul from each ref, boots each on its own throwaway port with its own
|
||||
# throwaway HOME and its own disposable copy of the corpus, runs the gold set N
|
||||
# times per ref, and prints the comparison with its noise threshold.
|
||||
#
|
||||
# SAFETY: never touches ~/.neuron, /Applications/Neuron*, ~/neuron-dev-stack, or
|
||||
# any running service. Sources are exported with `git archive` into a scratch
|
||||
# dir, so no worktree or branch state is mutated either. Ports are checked
|
||||
# against the live set before anything boots. Every soul this script starts is
|
||||
# killed and confirmed dead by run_eval.py; the sweep at the end is a backstop.
|
||||
#
|
||||
# usage:
|
||||
# run_comparison.sh [--baseline main] [--candidate feat/recall-through-activation]
|
||||
# [--repeats 3] [--corpus <snapshot.json>] [--repo <path>]
|
||||
set -euo pipefail
|
||||
|
||||
BASELINE="main"
|
||||
CANDIDATE="feat/recall-through-activation"
|
||||
REPEATS=3
|
||||
CORPUS="$HOME/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
REPO="$HOME/Development/neuron"
|
||||
BASE_PORT=7893
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--baseline) BASELINE="$2"; shift 2 ;;
|
||||
--candidate) CANDIDATE="$2"; shift 2 ;;
|
||||
--repeats) REPEATS="$2"; shift 2 ;;
|
||||
--corpus) CORPUS="$2"; shift 2 ;;
|
||||
--repo) REPO="$2"; shift 2 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WORK="$(mktemp -d "${TMPDIR:-/tmp}/retrieval-eval.XXXXXX")"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
[ -f "$CORPUS" ] || { echo "no corpus at $CORPUS" >&2; exit 2; }
|
||||
echo "corpus: $CORPUS ($(du -h "$CORPUS" | cut -f1))"
|
||||
|
||||
slug() { printf '%s' "$1" | tr '/' '-'; }
|
||||
|
||||
build_ref() { # ref -> binary path
|
||||
local ref="$1" out="$WORK/soul-$(slug "$1")"
|
||||
local src="$WORK/src-$(slug "$1")"
|
||||
mkdir -p "$src"
|
||||
git -C "$REPO" archive "$ref" | tar -x -C "$src"
|
||||
"$HERE/build-soul.sh" "$src" "$out" >&2
|
||||
printf '%s' "$out"
|
||||
}
|
||||
|
||||
echo "== building $BASELINE =="
|
||||
BIN_A="$(build_ref "$BASELINE")"
|
||||
echo "== building $CANDIDATE =="
|
||||
BIN_B="$(build_ref "$CANDIDATE")"
|
||||
|
||||
echo "== validating the gold set against this corpus =="
|
||||
python3 "$HERE/build_gold_set.py" "$CORPUS" --check
|
||||
|
||||
port=$BASE_PORT
|
||||
run_one() { # binary label out
|
||||
echo "== $2 =="
|
||||
python3 "$HERE/run_eval.py" --soul "$1" --corpus "$CORPUS" --label "$2" \
|
||||
--port "$port" --out "$3"
|
||||
port=$((port + 1))
|
||||
}
|
||||
|
||||
A_MAIN="$WORK/results-a-1.json"; B_MAIN="$WORK/results-b-1.json"
|
||||
A_REP=(); B_REP=()
|
||||
for i in $(seq 1 "$REPEATS"); do
|
||||
a="$WORK/results-a-$i.json"; b="$WORK/results-b-$i.json"
|
||||
run_one "$BIN_A" "$(slug "$BASELINE")-r$i" "$a"
|
||||
run_one "$BIN_B" "$(slug "$CANDIDATE")-r$i" "$b"
|
||||
[ "$i" -gt 1 ] && { A_REP+=("$a"); B_REP+=("$b"); }
|
||||
done
|
||||
|
||||
cp "$A_MAIN" "$HERE/results-$(slug "$BASELINE").json"
|
||||
cp "$B_MAIN" "$HERE/results-$(slug "$CANDIDATE").json"
|
||||
|
||||
echo
|
||||
python3 "$HERE/compare.py" \
|
||||
--baseline "$A_MAIN" --candidate "$B_MAIN" \
|
||||
${A_REP[@]+--repeats-baseline "${A_REP[@]}"} \
|
||||
${B_REP[@]+--repeats-candidate "${B_REP[@]}"} \
|
||||
--out "$HERE/comparison-$(slug "$BASELINE")-vs-$(slug "$CANDIDATE").json"
|
||||
|
||||
# Backstop: run_eval.py kills and confirms its own child, but a crashed run
|
||||
# could leak one. Leaving a soul running is how the live engine got squeezed.
|
||||
STRAY=$(pgrep -f "$WORK/soul-" || true)
|
||||
if [ -n "$STRAY" ]; then
|
||||
echo "!! stray eval souls, killing: $STRAY" >&2
|
||||
kill -9 $STRAY 2>/dev/null || true
|
||||
fi
|
||||
pgrep -f "$WORK/soul-" >/dev/null && { echo "!! STILL RUNNING" >&2; exit 5; }
|
||||
echo "process check: no eval souls running"
|
||||
@@ -1,398 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
run_eval.py — measure one soul build's retrieval against the gold set.
|
||||
|
||||
WHAT IT MEASURES, AND WHY IT BOOTS A REAL SOUL
|
||||
The point is Will's designed retrieval — spreading activation over the
|
||||
weighted directed graph with four-factor multiplicative scoring — not a
|
||||
Python re-implementation of it. A re-implementation would measure my
|
||||
reading of the design; booting the compiled binary measures the design. So
|
||||
this harness compiles the actual `soul.el` amalgam (build-soul.sh) and asks
|
||||
it over HTTP, exactly as the MCP wrapper and the app do.
|
||||
|
||||
SAFETY — read this before changing anything here
|
||||
* Boots on a THROWAWAY port with a THROWAWAY $HOME and a THROWAWAY COPY of
|
||||
the corpus. Refuses to use 7770 / 8742 / 7779 / 17779 / 7771.
|
||||
* ENGRAM_URL / SOUL_ENGRAM_URL are UNSET and SOUL_ISE_URL is pinned to a
|
||||
dead port. This is not belt-and-braces: the periodic engram sync resolves
|
||||
its source as env(SOUL_ISE_URL) -> state -> DEFAULT http://localhost:8742,
|
||||
so leaving it unset makes an "isolated" run silently pull the operator's
|
||||
LIVE brain. (Learned the hard way on 2026-08-03; see the same note in
|
||||
scripts/verify-soul-contract.sh.)
|
||||
* Every process this file starts is tracked and killed in a finally block,
|
||||
then CONFIRMED dead by pid probe, and the confirmation is written into the
|
||||
results file. A run that cannot confirm its child is dead exits non-zero.
|
||||
* Activation is a STATEFUL read by design (patent claim 29: traversal
|
||||
updates last-activation and increments activation counts). The corpus copy
|
||||
is therefore per-run and disposable, and every run starts from a byte-
|
||||
identical copy so two configurations see the same starting graph.
|
||||
|
||||
usage:
|
||||
python3 run_eval.py --soul <binary> --corpus <snapshot.json> --label main \
|
||||
[--port 7893] [--gold gold_set.json] [--limit 10] [--out results-main.json]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
FORBIDDEN_PORTS = {7770, 8742, 7779, 17779, 7771, 8080}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# metrics
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def recall_at_k(returned, relevant, k):
|
||||
if not relevant:
|
||||
return None
|
||||
return len(set(returned[:k]) & set(relevant)) / len(relevant)
|
||||
|
||||
|
||||
def hit_at_k(returned, relevant, k):
|
||||
if not relevant:
|
||||
return None
|
||||
return 1.0 if set(returned[:k]) & set(relevant) else 0.0
|
||||
|
||||
|
||||
def precision_at_k(returned, relevant, k):
|
||||
"""Fixed denominator k, as in docs/research/graphrag_eval/score.py.
|
||||
|
||||
Fixed denominator penalises an empty result and a page of junk equally,
|
||||
which is what we want: a retriever that returns nothing is not 'precise'.
|
||||
"""
|
||||
if not relevant:
|
||||
return None
|
||||
return len(set(returned[:k]) & set(relevant)) / k
|
||||
|
||||
|
||||
def mrr(returned, relevant, k):
|
||||
if not relevant:
|
||||
return None
|
||||
rel = set(relevant)
|
||||
for i, nid in enumerate(returned[:k], start=1):
|
||||
if nid in rel:
|
||||
return 1.0 / i
|
||||
return 0.0
|
||||
|
||||
|
||||
def mean(vals):
|
||||
vals = [v for v in vals if v is not None]
|
||||
return sum(vals) / len(vals) if vals else 0.0
|
||||
|
||||
|
||||
def pct(vals):
|
||||
return f"{100 * mean(vals):.1f}%"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# soul lifecycle
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
class Soul:
|
||||
def __init__(self, binary, corpus, port, verbose=True):
|
||||
if port in FORBIDDEN_PORTS:
|
||||
raise SystemExit(f"REFUSING: port {port} is a live service port.")
|
||||
self.binary = os.path.abspath(binary)
|
||||
self.corpus = os.path.abspath(corpus)
|
||||
self.port = port
|
||||
self.verbose = verbose
|
||||
self.home = None
|
||||
self.proc = None
|
||||
self.pid = None
|
||||
self.log = None
|
||||
self.confirmed_dead = None
|
||||
|
||||
@property
|
||||
def base(self):
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def start(self, boot_timeout=180):
|
||||
self.home = tempfile.mkdtemp(prefix="retrieval-eval-home.")
|
||||
snap = os.path.join(self.home, "corpus.json")
|
||||
t0 = time.time()
|
||||
shutil.copyfile(self.corpus, snap) # per-run disposable copy, never the source
|
||||
self.log = os.path.join(self.home, "soul.log")
|
||||
|
||||
env = {k: v for k, v in os.environ.items()
|
||||
if k not in ("ENGRAM_URL", "ENGRAM_API_KEY", "SOUL_ENGRAM_URL",
|
||||
"ANTHROPIC_API_KEY", "NEURON_LLM_API_KEY", "SOUL_IDENTITY",
|
||||
"SOUL_API_KEY")}
|
||||
env.update({
|
||||
"HOME": self.home,
|
||||
"NEURON_PORT": str(self.port),
|
||||
"SOUL_CGI_ID": f"ntn-retrieval-eval-{os.getpid()}",
|
||||
"SOUL_ENGRAM_PATH": snap,
|
||||
"NEURON_API_URL": "http://127.0.0.1:9", # dead port
|
||||
"SOUL_ISE_URL": "http://127.0.0.1:9", # dead port — see SAFETY above
|
||||
# Park the background loops for an hour so heartbeat/consolidation
|
||||
# cannot mutate the graph between queries and make runs unrepeatable.
|
||||
"SOUL_TICK_MS": "3600000",
|
||||
"SOUL_HEARTBEAT_MS": "3600000",
|
||||
"SOUL_REFRESH_MS": "3600000",
|
||||
})
|
||||
with open(self.log, "wb") as lf:
|
||||
self.proc = subprocess.Popen([self.binary], env=env, stdout=lf, stderr=lf,
|
||||
start_new_session=True)
|
||||
self.pid = self.proc.pid
|
||||
if self.verbose:
|
||||
print(f" booted pid={self.pid} port={self.port} home={self.home}")
|
||||
|
||||
deadline = time.time() + boot_timeout
|
||||
while time.time() < deadline:
|
||||
if self.proc.poll() is not None:
|
||||
raise RuntimeError(f"soul exited during boot: {self._log_tail()}")
|
||||
rss = self._rss_kb()
|
||||
if rss and rss > 6 * 1024 * 1024:
|
||||
self.stop()
|
||||
raise RuntimeError(f"soul RSS {rss}KB > 6GB — aborted")
|
||||
try:
|
||||
with urllib.request.urlopen(f"{self.base}/health", timeout=2) as r:
|
||||
if r.status == 200:
|
||||
if self.verbose:
|
||||
print(f" healthy in {time.time() - t0:.1f}s, RSS={self._rss_kb()}KB")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
self.stop()
|
||||
raise RuntimeError(f"soul never healthy on {self.base}: {self._log_tail()}")
|
||||
|
||||
def _rss_kb(self):
|
||||
try:
|
||||
out = subprocess.run(["ps", "-o", "rss=", "-p", str(self.pid)],
|
||||
capture_output=True, text=True, timeout=5).stdout.strip()
|
||||
return int(out) if out else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _log_tail(self, n=15):
|
||||
try:
|
||||
with open(self.log, encoding="utf-8", errors="replace") as fh:
|
||||
return "\n".join(fh.read().splitlines()[-n:])
|
||||
except Exception:
|
||||
return "(no log)"
|
||||
|
||||
def recall(self, query, limit, timeout=60):
|
||||
url = f"{self.base}/api/neuron/recall?query={urllib.parse.quote(query)}&limit={limit}"
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
raw = r.read().decode("utf-8", "replace")
|
||||
ms = (time.perf_counter() - t0) * 1000
|
||||
except Exception as exc:
|
||||
return [], (time.perf_counter() - t0) * 1000, f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
arr = json.loads(raw)
|
||||
except Exception:
|
||||
return [], ms, f"unparseable response ({len(raw)}B)"
|
||||
if not isinstance(arr, list):
|
||||
return [], ms, f"non-array response: {str(arr)[:120]}"
|
||||
ids = [x.get("id") for x in arr if isinstance(x, dict) and x.get("id")]
|
||||
return ids, ms, None
|
||||
|
||||
def stop(self):
|
||||
"""Kill and CONFIRM. A test process that outlives its test is a bug."""
|
||||
if self.pid is None:
|
||||
self.confirmed_dead = True
|
||||
return True
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
try:
|
||||
os.kill(self.pid, sig)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
for _ in range(20):
|
||||
try:
|
||||
os.kill(self.pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
continue
|
||||
break
|
||||
try:
|
||||
self.proc.wait(timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.kill(self.pid, 0)
|
||||
self.confirmed_dead = False
|
||||
except ProcessLookupError:
|
||||
self.confirmed_dead = True
|
||||
if self.verbose:
|
||||
print(f" pid {self.pid}: {'CONFIRMED DEAD' if self.confirmed_dead else 'STILL ALIVE'}")
|
||||
if self.home and os.path.isdir(self.home):
|
||||
shutil.rmtree(self.home, ignore_errors=True)
|
||||
return self.confirmed_dead
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# eval
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def evaluate(soul, gold, limit):
|
||||
rows = []
|
||||
for q in gold["queries"]:
|
||||
ids, ms, err = soul.recall(q["query"], limit)
|
||||
rel = q.get("relevant") or []
|
||||
row = {
|
||||
"id": q["id"],
|
||||
"category": q["category"],
|
||||
"query": q["query"],
|
||||
"returned": ids,
|
||||
"n_returned": len(ids),
|
||||
"latency_ms": round(ms, 1),
|
||||
"error": err,
|
||||
}
|
||||
if q.get("expect_empty"):
|
||||
row["clean"] = (len(ids) == 0)
|
||||
row["false_positives"] = len(ids)
|
||||
else:
|
||||
row["hit@5"] = hit_at_k(ids, rel, 5)
|
||||
row["recall@5"] = recall_at_k(ids, rel, 5)
|
||||
row["recall@10"] = recall_at_k(ids, rel, 10)
|
||||
row["precision@5"] = precision_at_k(ids, rel, 5)
|
||||
row["mrr@10"] = mrr(ids, rel, 10)
|
||||
if q.get("must_outrank"):
|
||||
correct, stale = q["must_outrank"]
|
||||
ic = ids.index(correct) if correct in ids else None
|
||||
istale = ids.index(stale) if stale in ids else None
|
||||
# Correct must be present AND above the stale node. A run that
|
||||
# returns neither is NOT a pass: the corrected fact is what the
|
||||
# user needed.
|
||||
row["outranks"] = (ic is not None) and (istale is None or ic < istale)
|
||||
row["rank_correct"] = None if ic is None else ic + 1
|
||||
row["rank_stale"] = None if istale is None else istale + 1
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def aggregate(rows):
|
||||
scored = [r for r in rows if "hit@5" in r]
|
||||
nonsense = [r for r in rows if "clean" in r]
|
||||
outrank = [r for r in rows if "outranks" in r]
|
||||
lat = sorted(r["latency_ms"] for r in rows)
|
||||
agg = {
|
||||
"n_queries": len(rows),
|
||||
"n_scored": len(scored),
|
||||
"hit@5": mean([r["hit@5"] for r in scored]),
|
||||
"recall@5": mean([r["recall@5"] for r in scored]),
|
||||
"recall@10": mean([r["recall@10"] for r in scored]),
|
||||
"precision@5": mean([r["precision@5"] for r in scored]),
|
||||
"mrr@10": mean([r["mrr@10"] for r in scored]),
|
||||
"nonsense_clean": f"{sum(1 for r in nonsense if r['clean'])}/{len(nonsense)}",
|
||||
"superseded_outranks": f"{sum(1 for r in outrank if r['outranks'])}/{len(outrank)}",
|
||||
"latency_ms_p50": lat[len(lat) // 2] if lat else 0,
|
||||
"latency_ms_p95": lat[max(0, int(len(lat) * 0.95) - 1)] if lat else 0,
|
||||
"latency_ms_max": lat[-1] if lat else 0,
|
||||
"errors": sum(1 for r in rows if r["error"]),
|
||||
"by_category": {},
|
||||
}
|
||||
cats = sorted({r["category"] for r in rows})
|
||||
for c in cats:
|
||||
cr = [r for r in rows if r["category"] == c]
|
||||
if c == "nonsense":
|
||||
agg["by_category"][c] = {
|
||||
"n": len(cr),
|
||||
"clean": sum(1 for r in cr if r["clean"]),
|
||||
"avg_false_positives": mean([float(r["false_positives"]) for r in cr]),
|
||||
}
|
||||
else:
|
||||
e = {
|
||||
"n": len(cr),
|
||||
"hit@5": mean([r.get("hit@5") for r in cr]),
|
||||
"recall@5": mean([r.get("recall@5") for r in cr]),
|
||||
"recall@10": mean([r.get("recall@10") for r in cr]),
|
||||
"mrr@10": mean([r.get("mrr@10") for r in cr]),
|
||||
}
|
||||
if c == "superseded":
|
||||
e["outranks"] = sum(1 for r in cr if r.get("outranks"))
|
||||
agg["by_category"][c] = e
|
||||
return agg
|
||||
|
||||
|
||||
def print_table(label, agg):
|
||||
print(f"\n=== {label} ===")
|
||||
print(f" queries {agg['n_queries']} ({agg['n_scored']} scored + "
|
||||
f"{agg['n_queries'] - agg['n_scored']} control) · errors {agg['errors']}")
|
||||
print(f" {'hit@5':>12} {'recall@5':>10} {'recall@10':>10} {'prec@5':>9} {'MRR@10':>9}")
|
||||
print(f" {pct([agg['hit@5']]):>12} {pct([agg['recall@5']]):>10} {pct([agg['recall@10']]):>10} "
|
||||
f"{pct([agg['precision@5']]):>9} {agg['mrr@10']:>9.3f}")
|
||||
print(f" nonsense clean {agg['nonsense_clean']} · superseded outranks {agg['superseded_outranks']}")
|
||||
print(f" latency ms p50 {agg['latency_ms_p50']:.0f} · p95 {agg['latency_ms_p95']:.0f} "
|
||||
f"· max {agg['latency_ms_max']:.0f}")
|
||||
print(f"\n {'category':14} {'n':>3} {'hit@5':>8} {'recall@5':>9} {'recall@10':>10} {'MRR@10':>8}")
|
||||
for c, e in agg["by_category"].items():
|
||||
if c == "nonsense":
|
||||
print(f" {c:14} {e['n']:>3} {'clean ' + str(e['clean']) + '/' + str(e['n']):>8}"
|
||||
f"{'':>9} {'':>10} {'avg FP ' + format(e['avg_false_positives'], '.1f'):>8}")
|
||||
else:
|
||||
extra = f" outranks {e['outranks']}/{e['n']}" if "outranks" in e else ""
|
||||
print(f" {c:14} {e['n']:>3} {pct([e['hit@5']]):>8} {pct([e['recall@5']]):>9} "
|
||||
f"{pct([e['recall@10']]):>10} {e['mrr@10']:>8.3f}{extra}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--soul", required=True)
|
||||
ap.add_argument("--corpus", required=True)
|
||||
ap.add_argument("--label", required=True)
|
||||
ap.add_argument("--gold", default=os.path.join(HERE, "gold_set.json"))
|
||||
ap.add_argument("--port", type=int, default=7893)
|
||||
ap.add_argument("--limit", type=int, default=10)
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.gold, encoding="utf-8") as fh:
|
||||
gold = json.load(fh)
|
||||
print(f"[{args.label}] soul={os.path.basename(args.soul)} "
|
||||
f"corpus={os.path.basename(args.corpus)} gold={len(gold['queries'])}q limit={args.limit}")
|
||||
|
||||
soul = Soul(args.soul, args.corpus, args.port)
|
||||
rows = []
|
||||
started = time.time()
|
||||
try:
|
||||
soul.start()
|
||||
rows = evaluate(soul, gold, args.limit)
|
||||
finally:
|
||||
dead = soul.stop()
|
||||
|
||||
agg = aggregate(rows)
|
||||
print_table(args.label, agg)
|
||||
|
||||
out = args.out or os.path.join(HERE, f"results-{args.label}.json")
|
||||
doc = {
|
||||
"label": args.label,
|
||||
"soul_binary": os.path.abspath(args.soul),
|
||||
"soul_md5": subprocess.run(["md5", "-q", args.soul], capture_output=True,
|
||||
text=True).stdout.strip(),
|
||||
"corpus": os.path.abspath(args.corpus),
|
||||
"corpus_nodes": gold.get("corpus_nodes"),
|
||||
"corpus_edges": gold.get("corpus_edges"),
|
||||
"gold_set": os.path.abspath(args.gold),
|
||||
"limit": args.limit,
|
||||
"port": args.port,
|
||||
"wall_clock_s": round(time.time() - started, 1),
|
||||
"child_pid": soul.pid,
|
||||
"child_confirmed_dead": soul.confirmed_dead,
|
||||
"aggregate": agg,
|
||||
"rows": rows,
|
||||
}
|
||||
with open(out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {out}")
|
||||
if not dead:
|
||||
print("FATAL: child process could not be confirmed dead", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,142 +0,0 @@
|
||||
import numpy as np, json, urllib.request, collections, math, re, sys, time
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
EV="/Users/timlingo/Development/neuron-technologies/_wt-semseed/tools/retrieval-eval/"
|
||||
np.seterr(all='ignore')
|
||||
t0=time.time()
|
||||
M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
eidx={k:i for i,k in enumerate(eids)}
|
||||
d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape'))
|
||||
print("loaded corpus %.1fs"%(time.time()-t0),file=sys.stderr)
|
||||
STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"}
|
||||
adj=collections.defaultdict(list)
|
||||
for e in d['edges']:
|
||||
if e.get('relation') not in STRUCT: continue
|
||||
w=float(e.get('weight') or 0.0)
|
||||
adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w))
|
||||
nodes=d['nodes']
|
||||
N={n['id']:n for n in nodes}
|
||||
PRINT=re.compile(r'^[\x20-\x7e]+$')
|
||||
ids=[]; hay=[]; dl=[]; sal=[]; addressable=[]
|
||||
for n in nodes:
|
||||
i=n.get('id') or ''
|
||||
h=((n.get('content') or '')+'\x00'+(n.get('label') or '')+'\x00'+(n.get('tags') or '')).lower()
|
||||
ids.append(i); hay.append(h); dl.append(len(h)); sal.append(float(n.get('salience') or 0.0))
|
||||
addressable.append(bool(PRINT.match(i)))
|
||||
del d
|
||||
NN=len(ids); avgdl=sum(dl)/NN
|
||||
print("nodes=%d avgdl=%.0f addressable=%d %.1fs"%(NN,avgdl,sum(addressable),time.time()-t0),file=sys.stderr)
|
||||
gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']}
|
||||
LEXMAIN={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"),) ['rows']} if False else {r['id']:r['returned'] for r in json.load(open(EV+"results-main.json",encoding='utf-8',errors='surrogateescape'))['rows']}
|
||||
CACHE={}
|
||||
def emb(t):
|
||||
if t in CACHE: return CACHE[t]
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v
|
||||
K1,B=1.2,0.75
|
||||
def lexleg(query, mode, guard, lim=10):
|
||||
toks=[]
|
||||
for w in query.split():
|
||||
wl=w.lower()
|
||||
if wl not in toks: toks.append(wl)
|
||||
nt=len(toks)
|
||||
masks=[]; df=[0]*nt
|
||||
for i in range(NN):
|
||||
if guard and not addressable[i]: continue
|
||||
h=hay[i]; m=0; sc=0
|
||||
for t in range(nt):
|
||||
if toks[t] in h: m|=(1<<t); sc+=1; df[t]+=1
|
||||
if sc: masks.append((i,m,sc))
|
||||
if mode=='tokcount':
|
||||
masks.sort(key=lambda x:(-x[2], -sal[x[0]]))
|
||||
return [ids[i] for i,m,sc in masks[:lim]]
|
||||
idf=[math.log(1.0+(NN-df[t]+0.5)/(df[t]+0.5)) for t in range(nt)]
|
||||
scored=[]
|
||||
for i,m,sc in masks:
|
||||
norm=1.0-B+B*dl[i]/avgdl
|
||||
s=0.0
|
||||
for t in range(nt):
|
||||
if m&(1<<t): s+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
scored.append((s,i))
|
||||
scored.sort(key=lambda x:(-x[0], -sal[x[1]]))
|
||||
return [ids[i] for s,i in scored[:lim]]
|
||||
FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64
|
||||
def assoc(seeds, s):
|
||||
act={x:1.0 for x in seeds}; seen={x:2 for x in seeds}
|
||||
Q=[(x,0) for x in seeds]; h=0
|
||||
while h<len(Q):
|
||||
cur,hop=Q[h]; h+=1
|
||||
if hop>=DEPTH: continue
|
||||
p=act[cur]
|
||||
for oid,w in adj.get(cur,()):
|
||||
n=N.get(oid)
|
||||
if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue
|
||||
na=p*w*DECAY*float(n.get('salience') or 0.0)
|
||||
if na<FIRE: continue
|
||||
if oid in seen and na<=act.get(oid,0): continue
|
||||
act[oid]=na
|
||||
if oid not in seen: seen[oid]=1
|
||||
Q.append((oid,hop+1))
|
||||
out=[]
|
||||
for k,v in seen.items():
|
||||
if v!=1 or k not in eidx: continue
|
||||
c=float(s[eidx[k]])
|
||||
if c<=0: continue
|
||||
out.append((c,k))
|
||||
out.sort(reverse=True)
|
||||
return [k for c,k in out[:ASSOC_MAX]]
|
||||
def inter3(L,S,A,lim=10):
|
||||
out=[]; li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def run(mode, guard, use_main_lex=False):
|
||||
res={}; legs={}
|
||||
for qid,q in gold.items():
|
||||
v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1
|
||||
L = LEXMAIN[qid][:10] if use_main_lex else lexleg(q['query'], mode, guard)
|
||||
ordr=np.argsort(-s)
|
||||
S=[eids[j] for j in ordr[:10] if s[j]>SEED_MIN]
|
||||
if guard: S=[x for x in S if PRINT.match(x or '')]
|
||||
seeds=[x for x in L[:3] if x in N]
|
||||
seeds=seeds+[eids[j] for j in ordr[:8] if eids[j] in N and eids[j] not in seeds and (not guard or PRINT.match(eids[j] or ''))]
|
||||
A=assoc(seeds,s) if seeds else []
|
||||
if guard: A=[x for x in A if PRINT.match(x or '')]
|
||||
res[qid]=inter3(L,S,A); legs[qid]=(L,S,A)
|
||||
return res,legs
|
||||
def score(res,label,verbose=False):
|
||||
det={}
|
||||
for qid,q in gold.items():
|
||||
out=res[qid][:5]
|
||||
if q['category']=='nonsense': ok=(len(res[qid])==0)
|
||||
elif q['category']=='superseded':
|
||||
must=q.get('must_outrank') or {}; ok=False
|
||||
for good,bad in (must.items() if isinstance(must,dict) else []):
|
||||
ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)<res[qid].index(bad))
|
||||
if not must: ok=any(r in out for r in q['relevant'])
|
||||
else: ok=any(r in out for r in q['relevant'])
|
||||
det[qid]=ok
|
||||
print("%-28s outcome-true=%d/38"%(label,sum(det.values())))
|
||||
return det
|
||||
if __name__=="__main__":
|
||||
base,_=run('tokcount',False,use_main_lex=True); b=score(base,'BASELINE semseed(real lex)')
|
||||
variants=[('tokcount',False,'replica: tokcount,noguard'),
|
||||
('tokcount',True ,'A: tokcount + idguard'),
|
||||
('bm25', False,'B: bm25 only'),
|
||||
('bm25', True ,'C: bm25 + idguard')]
|
||||
dets={}
|
||||
for m,g,lab in variants:
|
||||
r,_=run(m,g); dets[lab]=score(r,lab)
|
||||
dd=[q for q in gold if dets[lab][q]!=b[q]]
|
||||
print(" vs BASELINE moved=%d gains=%s losses=%s"%(len(dd),[q for q in dd if dets[lab][q]],[q for q in dd if not dets[lab][q]]))
|
||||
@@ -1,92 +0,0 @@
|
||||
import json,sys,pickle,numpy as np
|
||||
sys.path.insert(0,'.')
|
||||
from legs import *
|
||||
GP='/Users/timlingo/Development/neuron-technologies/_wt-bm25lex/tools/retrieval-eval/'
|
||||
G=json.load(open(GP+'gold_set.json'))
|
||||
def wstart(s,tok):
|
||||
i=s.find(tok)
|
||||
while i!=-1:
|
||||
if i==0 or not s[i-1].isalnum(): return True
|
||||
i=s.find(tok,i+1)
|
||||
return False
|
||||
def legs4(query, wordstart=False, unfloor=False):
|
||||
toks=tokenize(query); lt=[t.lower() for t in toks]
|
||||
hit_idx=[];hit_mask=[];df=[0]*len(toks)
|
||||
for i in range(N):
|
||||
if not OK[i]: continue
|
||||
s=LOW[i];m=0
|
||||
for t,tok in enumerate(lt):
|
||||
if tok in s and (not wordstart or wstart(s,tok)): m|=(1<<t)
|
||||
if m:
|
||||
hit_idx.append(i);hit_mask.append(m)
|
||||
for t in range(len(toks)):
|
||||
if m>>t&1: df[t]+=1
|
||||
dl_n=int(OK.sum());avgdl=float(DL[OK].sum()/max(dl_n,1))
|
||||
idf=[math.log(1.0+((dl_n-d+0.5)/(d+0.5))) for d in df]
|
||||
L=[]
|
||||
for j,i in enumerate(hit_idx):
|
||||
norm=1.0-B+B*(DL[i]/avgdl);w=0.0
|
||||
for t in range(len(toks)):
|
||||
if hit_mask[j]>>t&1: w+=idf[t]*(K1+1.0)/(1.0+K1*norm)
|
||||
L.append((i,w,SAL[i]))
|
||||
L.sort(key=lambda x:(-x[1],-x[2]))
|
||||
if not L: return [],[],[]
|
||||
qv=qemb(query);cos=En@qv;cos=np.where(HAVE&OK,cos,-2.0)
|
||||
order=np.argsort(-cos)[:600]
|
||||
Sl=[int(i) for i in order if cos[i]>(0.0 if unfloor else SEED_MIN)]
|
||||
semseed=[int(i) for i in order[:SEED_K] if cos[i]>0.0]
|
||||
act={};seen={};qq=[]
|
||||
for i,_,_ in L[:ASSOC_SEEDS]:
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
for i in semseed:
|
||||
if i in seen: continue
|
||||
act[i]=1.0;seen[i]=2;qq.append((i,0))
|
||||
qh=0
|
||||
while qh<len(qq):
|
||||
cur,h=qq[qh];qh+=1
|
||||
if h>=DEPTH: continue
|
||||
parent=act[cur]
|
||||
for e,oi in ADJ_F[cur]+ADJ_T[cur]:
|
||||
if e['rel'] not in STRUCT or EXCL[oi]: continue
|
||||
na=parent*e['w']*DECAY*SAL[oi]
|
||||
if na<FIRE: continue
|
||||
if seen.get(oi) and na<=act.get(oi,0): continue
|
||||
act[oi]=na
|
||||
if not seen.get(oi): seen[oi]=1
|
||||
if len(qq)<AMAX*4: qq.append((oi,h+1))
|
||||
A=sorted([(i,float(cos[i])) for i,st in seen.items() if st==1 and OK[i] and HAVE[i] and cos[i]>0.0],key=lambda x:-x[1])[:AMAX]
|
||||
return [i for i,_,_ in L],Sl,[i for i,_ in A]
|
||||
def merge(L,S,A,lim=10):
|
||||
out=[];li=si=ai=0
|
||||
while len(out)<lim and (li<len(L) or si<len(S) or ai<len(A)):
|
||||
if li<len(L):
|
||||
if L[li] not in out: out.append(L[li])
|
||||
li+=1
|
||||
if len(out)>=lim: break
|
||||
if si<len(S):
|
||||
if S[si] not in out: out.append(S[si])
|
||||
si+=1
|
||||
if len(out)>=lim: break
|
||||
if ai<len(A):
|
||||
if A[ai] not in out: out.append(A[ai])
|
||||
ai+=1
|
||||
return out
|
||||
def outcome(q,ids):
|
||||
c=q['category']
|
||||
if c=='nonsense': return len(ids)==0
|
||||
if c=='superseded':
|
||||
a,b=q['must_outrank']
|
||||
if a not in ids: return False
|
||||
if b not in ids: return True
|
||||
return ids.index(a)<ids.index(b)
|
||||
return any(x in ids[:5] for x in q['relevant'])
|
||||
def run(**kw):
|
||||
return {q['id']:outcome(q,[NODES[i]['id'] for i in merge(*legs4(q['query'],**kw),10)]) for q in G['queries']}
|
||||
base=run()
|
||||
print("baseline",sum(base.values()),"misses",[k for k,v in base.items() if not v])
|
||||
for name,kw in [('wordstart',dict(wordstart=True)),
|
||||
('unfloor',dict(unfloor=True)),
|
||||
('wordstart+unfloor',dict(wordstart=True,unfloor=True))]:
|
||||
r=run(**kw)
|
||||
g=sorted(k for k in base if r[k] and not base[k]);l=sorted(k for k in base if base[k] and not r[k])
|
||||
print("%-20s net=%+d gains=%s losses=%s"%(name,len(g)-len(l),g,l))
|
||||
@@ -1,31 +0,0 @@
|
||||
import numpy as np, json, urllib.request
|
||||
SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad"
|
||||
M=np.load(SP+'/emb.npy'); ids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n')
|
||||
np.seterr(all='ignore')
|
||||
bad=~np.isfinite(M).all(axis=1)
|
||||
M[bad]=0.0
|
||||
print("non-finite rows zeroed:",int(bad.sum()))
|
||||
idx={k:i for i,k in enumerate(ids)}
|
||||
gold=json.load(open("/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/gold_set.json"))['queries']
|
||||
def emb(t):
|
||||
b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode()
|
||||
r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"})
|
||||
v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32)
|
||||
return v/(np.linalg.norm(v)+1e-9)
|
||||
out={}
|
||||
for q in gold:
|
||||
v=emb(q['query']); s=M@v
|
||||
s=s[np.isfinite(s)]
|
||||
mu=float(s.mean()); sd=float(s.std())
|
||||
top=np.sort(s)[::-1][:10]
|
||||
z=[(float(t)-mu)/sd for t in top]
|
||||
grank=[]
|
||||
for rel in q['relevant']:
|
||||
if rel in idx:
|
||||
j=idx[rel]; grank.append((int((M@v > (M@v)[j]).sum())+1, round(float((M@v)[j]),3)))
|
||||
grank.sort()
|
||||
out[q['id']]=dict(cat=q['category'],mu=round(mu,3),sd=round(sd,4),top1=round(float(top[0]),3),
|
||||
z1=round(z[0],2),z3=round(z[2],2),z5=round(z[4],2),gold=grank[:1])
|
||||
print("%s %-11s mu=%.3f sd=%.4f top1=%.3f z1=%5.2f z3=%5.2f z5=%5.2f gold=%s"%(
|
||||
q['id'],q['category'],mu,sd,top[0],z[0],z[2],z[4],grank[:1]))
|
||||
json.dump(out,open(SP+'/zprobe.json','w'),indent=1)
|
||||
+18
-618
@@ -6099,12 +6099,6 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) {
|
||||
* propagation loop in engram_activate. 0.25 damps semantically unrelated
|
||||
* branches ~4x without severing them. Unembedded targets are ungated. */
|
||||
#define ENGRAM_QGATE_FLOOR 0.25
|
||||
/* The read-path semantic leg (engram claim 24) reuses ENGRAM_EMBED_SEED_MIN
|
||||
* above as its admission floor: a node joins the embedding ranking only if its
|
||||
* query cosine clears the same bar that lets it join the seed set. No new
|
||||
* tuning constant is introduced, and the floor is load-bearing rather than
|
||||
* cosmetic — it is what keeps a query with no real match (the gold set's
|
||||
* nonsense controls) from being answered with its nearest neighbours. */
|
||||
#define ENGRAM_EMBED_MAX_CHARS 2000
|
||||
#define ENGRAM_EMBED_TIMEOUT_MS 4000L
|
||||
#define ENGRAM_EMBED_BREAKER_LIMIT 3
|
||||
@@ -7175,12 +7169,6 @@ void engram_forget(el_val_t node_id) {
|
||||
if (idx < 0) return;
|
||||
/* Free node strings */
|
||||
EngramNode* n = &g->nodes[idx];
|
||||
if (getenv("EG_DIAG")) {
|
||||
fprintf(stderr, "[EG_DIAG] FORGET id=%s type=%s layer=%u label=%s\n",
|
||||
sid, n->node_type ? n->node_type : "?", n->layer_id,
|
||||
n->label ? n->label : "?");
|
||||
fflush(stderr);
|
||||
}
|
||||
free(n->id); free(n->content); free(n->node_type); free(n->label);
|
||||
free(n->tier); free(n->tags); free(n->metadata);
|
||||
free(n->emb);
|
||||
@@ -7276,8 +7264,6 @@ el_val_t engram_prune_telemetry(el_val_t older_than_ms) {
|
||||
}
|
||||
}
|
||||
g->node_count = w;
|
||||
if (getenv("EG_DIAG"))
|
||||
fprintf(stderr, "[EG_DIAG] PRUNE_TELEMETRY removed=%lld\n", (long long)removed);
|
||||
if (removed == 0) { free(removed_ids); return 0; }
|
||||
|
||||
/* Removed-id hash set (open addressing, power-of-two >= 2*removed). */
|
||||
@@ -7335,39 +7321,6 @@ static int istr_contains(const char* hay, const char* needle) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Word-START-anchored variant of istr_contains.
|
||||
*
|
||||
* WHY. The retrieval match primitive is a raw substring test, so a query token
|
||||
* matches ANYWHERE inside a corpus word: "throom" matches "bathroom", "cat"
|
||||
* matches "concatenate". Measured on this corpus over the 38-query gold set,
|
||||
* that is not a rare accident — it is the bulk of some queries' candidate
|
||||
* sets. q28's lexical leg is 36,954 records of which only 13 contain a query
|
||||
* token at a word start (99.96% mid-word noise); six other queries carry
|
||||
* ~20,500 mid-word-only records each; and the nonsense control q35
|
||||
* ("xxqzzt vurblenacht throom") returns 7 records ALL of which match only
|
||||
* mid-word, which is the entire reason that control has been dirty since main.
|
||||
*
|
||||
* WHAT CHANGES. A token must begin at a word boundary — the preceding
|
||||
* character is not alphanumeric. Suffixes are still matched ("value" still
|
||||
* hits "values", "unjailbreakable" still hits "unjailbreakables"), so this is
|
||||
* strictly a prefix anchor, not whole-word equality; whole-word equality would
|
||||
* break the morphological matching the phrase category depends on.
|
||||
*
|
||||
* PROVENANCE, stated honestly: this restores no engram claim. Will's design
|
||||
* has no lexical leg at all (05-detailed-description l.64 takes "one or more
|
||||
* seed node UUIDs representing the current active context" as its input), so
|
||||
* the lexical leg is the seed-finding step that feeds the designed mechanism.
|
||||
* Cleaner seeds serve that mechanism; they do not replace it. */
|
||||
static int istr_contains_wordstart(const char* hay, const char* needle) {
|
||||
if (!hay || !needle || !*needle) return 0;
|
||||
size_t nl = strlen(needle);
|
||||
for (const char* p = hay; *p; p++) {
|
||||
if (p != hay && isalnum((unsigned char)p[-1])) continue;
|
||||
if (strncasecmp(p, needle, nl) == 0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Tokenized query matching ───────────────────────────────────────────
|
||||
* The engram query surface (search / activate / goal-bias) historically
|
||||
* matched the ENTIRE raw query string as a single case-insensitive
|
||||
@@ -7420,81 +7373,17 @@ static int engram_node_match_score(const EngramNode* n,
|
||||
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||
int score = 0;
|
||||
for (int t = 0; t < ntok; t++) {
|
||||
if (istr_contains_wordstart(n->content, toks[t]) ||
|
||||
istr_contains_wordstart(n->label, toks[t]) ||
|
||||
istr_contains_wordstart(n->tags, toks[t]))
|
||||
if (istr_contains(n->content, toks[t]) ||
|
||||
istr_contains(n->label, toks[t]) ||
|
||||
istr_contains(n->tags, toks[t]))
|
||||
score++;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
/* Same match test as engram_node_match_score, but returns the SET of matched
|
||||
* query tokens as a bitmask instead of only their count. ENGRAM_MAX_QTOKENS is
|
||||
* 32, so one uint32 covers every token the tokenizer can produce. The mask is
|
||||
* what lets the caller accumulate a per-token document frequency in the SAME
|
||||
* pass that finds the hits — no second scan of the corpus. */
|
||||
static uint32_t engram_node_match_mask(const EngramNode* n,
|
||||
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
||||
uint32_t m = 0;
|
||||
for (int t = 0; t < ntok && t < 32; t++) {
|
||||
if (istr_contains_wordstart(n->content, toks[t]) ||
|
||||
istr_contains_wordstart(n->label, toks[t]) ||
|
||||
istr_contains_wordstart(n->tags, toks[t]))
|
||||
m |= (uint32_t)1u << t;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/* Searchable byte length of a node: the same three fields the match test
|
||||
* reads. Used as the BM25 document length so a long node does not out-match a
|
||||
* short one merely by containing more text. */
|
||||
static double engram_node_len(const EngramNode* n) {
|
||||
double l = 0.0;
|
||||
if (n->content) l += (double)strlen(n->content);
|
||||
if (n->label) l += (double)strlen(n->label);
|
||||
if (n->tags) l += (double)strlen(n->tags);
|
||||
return l;
|
||||
}
|
||||
|
||||
/* Addressability guard. Claim 23 stores node records under a key encoding the
|
||||
* node identifier, claim 12 deduplicates merged results by node identifier,
|
||||
* and claim 27's competition map is indexed by node identifier — every one of
|
||||
* those requires the identifier to be a usable string. This corpus contains
|
||||
* records whose id field is binary garbage (a save-side corruption); they are
|
||||
* unfetchable by any caller, so returning one wastes a result slot. Printable
|
||||
* ASCII, non-empty, is the whole test. */
|
||||
static int eg_node_addressable(const EngramNode* n) {
|
||||
const unsigned char* p = (const unsigned char*)n->id;
|
||||
if (!p || !*p) return 0;
|
||||
for (; *p; p++) if (*p < 0x20 || *p > 0x7e) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Semantic leg of the read path (engram claim 24). Returns the query/target
|
||||
* cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0],
|
||||
* and exactly 0.0 when the pair is not comparable (no query embedding, target
|
||||
* unembedded, dim mismatch) or falls at/below the seed floor. Claim 32's
|
||||
* clamp-at-zero is subsumed: nothing below the floor can contribute.
|
||||
* A 0.0 return makes the fused score collapse to the lexical score, which is
|
||||
* why a dead embedder degrades to the historical behaviour exactly. */
|
||||
static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) {
|
||||
if (!qv || qdim <= 0 || !n->emb || n->emb_dim != qdim) return 0.0;
|
||||
double c = eg_cosine(n->emb, qv, qdim);
|
||||
if (c <= ENGRAM_EMBED_SEED_MIN) return 0.0;
|
||||
double t = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN);
|
||||
return t > 1.0 ? 1.0 : t;
|
||||
}
|
||||
|
||||
/* Rank entry: distinct-token match count (primary, desc) then salience
|
||||
* (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was —
|
||||
* the semantic leg is a second ranking merged beside it, never a reweighting
|
||||
* of this one. */
|
||||
typedef struct {
|
||||
int64_t idx; int score; double salience;
|
||||
uint32_t mask; /* which query tokens matched (BM25 leg) */
|
||||
double len; /* searchable byte length (BM25 leg) */
|
||||
double w; /* BM25-shaped weighted score */
|
||||
} EngramRankEntry;
|
||||
* (tiebreak, desc). */
|
||||
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||
static int engram_rank_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||
@@ -7504,269 +7393,6 @@ static int engram_rank_cmp(const void* a, const void* b) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* BM25-shaped ordering for the read path's lexical leg: rare-term weight and
|
||||
* length normalisation instead of a raw distinct-token count. Salience stays
|
||||
* the tiebreak, exactly as in engram_rank_cmp. */
|
||||
#define ENGRAM_BM25_K1 1.2
|
||||
#define ENGRAM_BM25_B 0.75
|
||||
static int engram_rank_w_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||
if (ea->w < eb->w) return 1; /* desc */
|
||||
if (ea->w > eb->w) return -1;
|
||||
if (ea->salience < eb->salience) return 1;
|
||||
if (ea->salience > eb->salience) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Semantic rank entry: node index and its renormalized query similarity,
|
||||
* ordered by similarity desc. This is the claim-24 "embedding search"
|
||||
* ranking, computed independently of the lexical one. */
|
||||
typedef struct { int64_t idx; double sem; } EngramSemEntry;
|
||||
static int engram_sem_cmp(const void* a, const void* b) {
|
||||
const EngramSemEntry* ea = (const EngramSemEntry*)a;
|
||||
const EngramSemEntry* eb = (const EngramSemEntry*)b;
|
||||
if (ea->sem < eb->sem) return 1; /* desc */
|
||||
if (ea->sem > eb->sem) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Merge the two rankings by strict alternation, lexical first:
|
||||
* L1, S1, L2, S2, L3, ... deduplicated by node index, capped at lim.
|
||||
*
|
||||
* Rank fusion, not score fusion. nomic's cosine scale is compressed (real
|
||||
* matches land ~0.55-0.70 while unrelated pairs sit ~0.35-0.50), so any
|
||||
* additive blend of a cosine onto a token-coverage score is dominated by
|
||||
* whichever leg happens to have the wider spread. Alternation is invariant to
|
||||
* both scales: it asks each leg for its next best answer in turn.
|
||||
*
|
||||
* Position 1 is always the top lexical hit, so a query whose answer the
|
||||
* lexical leg already ranks first cannot be displaced — exact-token retrieval
|
||||
* is structurally safe. The cost is bounded and explicit: a lexical hit at
|
||||
* rank r lands at output position 2r-1. */
|
||||
static int64_t engram_interleave(const EngramRankEntry* L, int64_t nL,
|
||||
const EngramSemEntry* S, int64_t nS,
|
||||
int64_t lim, int64_t* out) {
|
||||
int64_t no = 0, li = 0, si = 0;
|
||||
while (no < lim && (li < nL || si < nS)) {
|
||||
if (li < nL) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = L[li].idx;
|
||||
li++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (si < nS) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = S[si].idx;
|
||||
si++;
|
||||
}
|
||||
}
|
||||
return no;
|
||||
}
|
||||
|
||||
/* ── Associative leg (claim 10 typed relations + claim 1 activation) ────────
|
||||
*
|
||||
* WHY THIS EXISTS. engram_search_json has two legs, and neither can reach a
|
||||
* node that shares no token with the query and no embedding neighbourhood
|
||||
* with it. The route the design reserves for that case is the graph: a node
|
||||
* is reachable because it is STRUCTURALLY associated with something the query
|
||||
* did hit. Nothing on the recall path consults an edge today.
|
||||
*
|
||||
* WHY NOT engram_activate. Wiring recall wholesale to activation was measured
|
||||
* (PR #135) and lost 57 points of phrase accuracy. The failure was one of
|
||||
* RANK, not reach: activation seeds on every token-matching node, so a 2-hop
|
||||
* associate at strength ~0.06 never outranks thousands of 1-hop neighbours of
|
||||
* strong lexical seeds. So this is a separate, small, ranked list that is
|
||||
* merged BESIDE the other two, exactly as the semantic leg is.
|
||||
*
|
||||
* TYPED RELATIONS (06-claims.md claim 10). Expansion follows only relations
|
||||
* that assert a structural claim about meaning. The corpus is 4,915 `tagged`
|
||||
* and 3,767 `triggers-safety` edges against 475 `identity` and 11 `contains`;
|
||||
* walking the bulk relations turns any seed into a flood (measured: 1,387
|
||||
* candidates from one seed) while the structural ones stay in the low tens.
|
||||
* This is the first code on the read path to branch on a relation type at all.
|
||||
*
|
||||
* DIRECTION. Edges are walked in BOTH directions. Claim 23 requires the store
|
||||
* to keep reverse edge records keyed by target id, and the adjacency index
|
||||
* already materialises them (adj_to). It has to be both: every value node in
|
||||
* this corpus has exactly ONE inbound edge (hub -> value) and no outbound
|
||||
* structural edge at all, so a forward-only walk from a value node reaches
|
||||
* nothing. Note this is an extension of the traversal as literally specified
|
||||
* (05-detailed-description l.66 says "all outgoing edges"); the reverse index
|
||||
* is designed and stored, but the description does not say the walk reads it.
|
||||
*/
|
||||
#define ENGRAM_ASSOC_SEEDS 3 /* top-N lexical hits form the context */
|
||||
#define ENGRAM_ASSOC_DEPTH 2 /* seed -> hub -> sibling */
|
||||
#define ENGRAM_ASSOC_FIRE 0.02 /* same firing threshold as engram_activate */
|
||||
#define ENGRAM_ASSOC_MAX 64 /* cap on candidates carried forward */
|
||||
|
||||
static int eg_rel_is_structural(const char* r) {
|
||||
if (!r || !*r) return 0;
|
||||
static const char* ok[] = {
|
||||
"identity", "contains", "superseded_by", "references", "embodies",
|
||||
"demonstrated_by", "canonical-self", "depends_on", "currently_holds",
|
||||
"activates", NULL
|
||||
};
|
||||
for (int i = 0; ok[i]; i++) if (strcmp(r, ok[i]) == 0) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Nodes that are index artefacts rather than recallable content. Same
|
||||
* exclusion eg_embed_eligible() already applies when deciding what deserves an
|
||||
* embedding, reused here so the associative leg cannot surface — or relay
|
||||
* through — a Tag. Relaying through them is what makes a graph walk explode:
|
||||
* tag-tier_note alone has 187 members. */
|
||||
static int eg_assoc_excluded(const EngramNode* n) {
|
||||
if (!n->node_type) return 0;
|
||||
return strcmp(n->node_type, "Tag") == 0
|
||||
|| strcmp(n->node_type, "InternalStateEvent") == 0;
|
||||
}
|
||||
|
||||
/* Breadth-first structural expansion from `seeds`, then ORDER BY query
|
||||
* similarity. Activation decides REACHABILITY (the conjunctive prune of
|
||||
* claim 1: parent strength x edge weight x target salience, cut at the firing
|
||||
* threshold); cosine decides ORDER within what was reached. Ranking the
|
||||
* neighbourhood by activation alone does not work and the reason is
|
||||
* structural: every identity edge in this corpus carries weight 0.5 and every
|
||||
* value node salience 0.7, so the activation product degenerates into a
|
||||
* function of hop count and ranks the relay hub above all of its own
|
||||
* children. Composing the two is mine, not Will's — the description ranks the
|
||||
* activation result set by strength (l.78). */
|
||||
/* Semantic seeding of the graph leg — the HippoRAG pass Will documents at
|
||||
* l.6082: "the query is embedded, the top-K nodes by cosine >= SEED_MIN join
|
||||
* the seed set", using his own ENGRAM_EMBED_SEED_K (8). It is "similarity used
|
||||
* twice, coherently": cosine picks where to STAND in the graph, the structural
|
||||
* walk decides what is REACHABLE from there, and cosine then ORDERS what was
|
||||
* reached (iteration-2's finding, kept intact).
|
||||
*
|
||||
* Why the seed list is NOT floored at ENGRAM_EMBED_SEED_MIN here. That
|
||||
* constant is calibrated for a cosine scale this corpus does not have: with
|
||||
* nomic-embed-text every true paraphrase target measures 0.46-0.66, and the
|
||||
* three nonsense controls' own nearest neighbours measure 0.55/0.60/0.62 —
|
||||
* they OVERLAP, so no absolute cosine floor separates signal from gibberish
|
||||
* (measured, all 38 queries). Iteration 2 established the same thing one step
|
||||
* later in the pipeline: applying the floor to graph CANDIDATES removed every
|
||||
* gain, because within a structurally-reached neighbourhood relative cosine
|
||||
* still discriminates below the absolute threshold. The gate that actually
|
||||
* works is reachability — eg_rel_is_structural() plus the firing threshold.
|
||||
* A semantically-near node with no structural attachment expands to nothing
|
||||
* and contributes nothing, which is exactly what happens to gibberish: the
|
||||
* nearest neighbours of q33/q34 are unattached, so their graph leg is empty.
|
||||
*/
|
||||
static int64_t engram_assoc_leg(EngramStore* g,
|
||||
const EngramRankEntry* L, int64_t nL,
|
||||
const int64_t* semseed, int64_t nsemseed,
|
||||
const float* qv, int32_t qdim,
|
||||
EngramSemEntry* out, int64_t out_cap) {
|
||||
if (!g || nL <= 0 || !qv || qdim <= 0 || out_cap <= 0) return 0;
|
||||
if (g->adj_dirty || !g->adj_from || !g->adj_to) engram_adj_rebuild(g);
|
||||
if (!g->adj_from || !g->adj_to) return 0;
|
||||
|
||||
double* act = calloc((size_t)g->node_count, sizeof(double));
|
||||
char* seen = calloc((size_t)g->node_count, sizeof(char));
|
||||
int64_t* q = malloc((size_t)ENGRAM_ASSOC_MAX * 4 * sizeof(int64_t));
|
||||
int64_t* hop = malloc((size_t)ENGRAM_ASSOC_MAX * 4 * sizeof(int64_t));
|
||||
if (!act || !seen || !q || !hop) { free(act); free(seen); free(q); free(hop); return 0; }
|
||||
int64_t qcap = ENGRAM_ASSOC_MAX * 4, qh = 0, qt = 0;
|
||||
|
||||
int64_t nseed = nL < ENGRAM_ASSOC_SEEDS ? nL : ENGRAM_ASSOC_SEEDS;
|
||||
for (int64_t s = 0; s < nseed; s++) {
|
||||
int64_t idx = L[s].idx;
|
||||
if (idx < 0 || idx >= g->node_count) continue;
|
||||
act[idx] = 1.0; seen[idx] = 2; /* 2 = seed: never a result */
|
||||
if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; }
|
||||
}
|
||||
/* ...and the semantic seeds, on the same footing (strength 1.0, hop 0). */
|
||||
for (int64_t s = 0; s < nsemseed; s++) {
|
||||
int64_t idx = semseed[s];
|
||||
if (idx < 0 || idx >= g->node_count) continue;
|
||||
if (seen[idx]) continue;
|
||||
act[idx] = 1.0; seen[idx] = 2;
|
||||
if (qt < qcap) { q[qt] = idx; hop[qt] = 0; qt++; }
|
||||
}
|
||||
|
||||
const double SPREAD_DECAY = 0.7;
|
||||
while (qh < qt) {
|
||||
int64_t cur = q[qh]; int64_t h = hop[qh]; qh++;
|
||||
if (h >= ENGRAM_ASSOC_DEPTH) continue;
|
||||
double parent = act[cur];
|
||||
int from_len = g->adj_from_len[cur];
|
||||
int to_len = g->adj_to_len[cur];
|
||||
for (int scan = 0; scan < from_len + to_len; scan++) {
|
||||
int64_t ei = (scan < from_len) ? g->adj_from[cur][scan]
|
||||
: g->adj_to[cur][scan - from_len];
|
||||
EngramEdge* e = &g->edges[ei];
|
||||
if (!eg_rel_is_structural(e->relation)) continue;
|
||||
int64_t oi = (scan < from_len) ? engram_idmap_get(g, e->to_id)
|
||||
: engram_idmap_get(g, e->from_id);
|
||||
if (oi < 0 || oi >= g->node_count) continue;
|
||||
EngramNode* on = &g->nodes[oi];
|
||||
if (eg_assoc_excluded(on)) continue;
|
||||
double na = parent * e->weight * SPREAD_DECAY * on->salience;
|
||||
if (na < ENGRAM_ASSOC_FIRE) continue;
|
||||
if (seen[oi] && na <= act[oi]) continue;
|
||||
act[oi] = na;
|
||||
if (!seen[oi]) seen[oi] = 1;
|
||||
if (qt < qcap) { q[qt] = oi; hop[qt] = h + 1; qt++; }
|
||||
}
|
||||
}
|
||||
|
||||
int64_t n = 0;
|
||||
for (int64_t i = 0; i < g->node_count && n < out_cap; i++) {
|
||||
if (seen[i] != 1) continue; /* skip unreached and seeds */
|
||||
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
|
||||
EngramNode* nd = &g->nodes[i];
|
||||
if (!eg_node_addressable(nd)) continue; /* unfetchable record */
|
||||
if (!nd->emb || nd->emb_dim != qdim) continue;
|
||||
double c = eg_cosine(nd->emb, qv, qdim);
|
||||
if (c <= 0.0) continue;
|
||||
out[n].idx = i; out[n].sem = c; n++;
|
||||
}
|
||||
qsort(out, (size_t)n, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
free(act); free(seen); free(q); free(hop);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Three-leg merge: lexical, semantic, associative — strict rotation,
|
||||
* L1, S1, A1, L2, S2, A2, ... deduplicated, capped at lim.
|
||||
*
|
||||
* The cost is explicit and worse than the two-leg case: a lexical hit at rank
|
||||
* r lands at output position 3r-2 when both other legs are non-empty. That is
|
||||
* survivable here only because the structural-relation filter leaves the
|
||||
* associative list EMPTY for most queries — a Memory node whose only edges are
|
||||
* `tagged` and `related` expands to nothing, so its ranking is untouched. */
|
||||
static int64_t engram_interleave3(const EngramRankEntry* L, int64_t nL,
|
||||
const EngramSemEntry* S, int64_t nS,
|
||||
const EngramSemEntry* A, int64_t nA,
|
||||
int64_t lim, int64_t* out) {
|
||||
int64_t no = 0, li = 0, si = 0, ai = 0;
|
||||
while (no < lim && (li < nL || si < nS || ai < nA)) {
|
||||
if (li < nL) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = L[li].idx;
|
||||
li++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (si < nS) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = S[si].idx;
|
||||
si++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (ai < nA) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == A[ai].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = A[ai].idx;
|
||||
ai++;
|
||||
}
|
||||
}
|
||||
return no;
|
||||
}
|
||||
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
@@ -7779,12 +7405,6 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
if (ntok == 0) return lst;
|
||||
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||
if (!hits) return lst;
|
||||
/* Claim-24 semantic leg: one query embedding, fetched once per search.
|
||||
* NULL (embedder down / circuit breaker open) => pure lexical, as before. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = eg_embed_fetch(q, &qdim);
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
@@ -7800,24 +7420,14 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
hits[nhits].salience = n->salience;
|
||||
nhits++;
|
||||
}
|
||||
if (sem) {
|
||||
double sv = eg_sem_term(n, qv, qdim);
|
||||
if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; }
|
||||
}
|
||||
}
|
||||
/* Rank each leg independently, then alternate between them. */
|
||||
/* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
if (order) {
|
||||
int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order);
|
||||
for (int64_t k = 0; k < no; k++)
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[order[k]]));
|
||||
free(order);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
||||
}
|
||||
free(hits);
|
||||
free(sem);
|
||||
free(qv);
|
||||
return lst;
|
||||
}
|
||||
|
||||
@@ -9312,26 +8922,6 @@ el_val_t engram_load(el_val_t path) {
|
||||
}
|
||||
}
|
||||
g->adj_dirty = 1;
|
||||
if (getenv("EG_DIAG")) {
|
||||
int64_t we = 0, wrongdim = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (g->nodes[i].emb) { we++; if (g->nodes[i].emb_dim != 768) wrongdim++; }
|
||||
}
|
||||
fprintf(stderr, "[EG_DIAG] loaded nodes=%lld with_emb=%lld wrongdim=%lld\n",
|
||||
(long long)g->node_count, (long long)we, (long long)wrongdim);
|
||||
const char* probe = getenv("EG_DIAG_ID");
|
||||
if (probe) {
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (g->nodes[i].id && strcmp(g->nodes[i].id, probe) == 0) {
|
||||
fprintf(stderr, "[EG_DIAG] probe id=%s idx=%lld emb=%p dim=%d layer=%u addr=%d\n",
|
||||
probe, (long long)i, (void*)g->nodes[i].emb,
|
||||
(int)g->nodes[i].emb_dim, g->nodes[i].layer_id,
|
||||
eg_node_addressable(&g->nodes[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
fflush(stderr);
|
||||
}
|
||||
/* Walk edges array */
|
||||
const char* edges_p = json_find_key(data, "edges");
|
||||
if (edges_p) {
|
||||
@@ -9652,33 +9242,7 @@ el_val_t engram_get_node_by_label(el_val_t label) {
|
||||
return el_wrap_str(el_strdup("{}"));
|
||||
}
|
||||
|
||||
/* ── THE SEARCH / RECALL BOUNDARY (2026-08-07) ───────────────────────────────
|
||||
* engram_search_json is the LEXICAL function ~40 .el call sites already
|
||||
* depend on: they pass a key-shaped string ("soul:boot_count",
|
||||
* "soul-inbox-pending", a session label) and treat every returned record as
|
||||
* a record that CONTAINS that key. Seven of those sites then delete what
|
||||
* comes back (memory.el:176, sessions.el:250/268/444/523, soul.el:359 —
|
||||
* "prune all existing X nodes, keep exactly one").
|
||||
*
|
||||
* The semantic and associative legs must therefore NOT live on this
|
||||
* function. Claim 24 authorises the vector index "to respond to EMBEDDING
|
||||
* SEARCH QUERIES by returning the node records whose embedding vectors have
|
||||
* the highest cosine similarity to a query vector"; a keyed state read is
|
||||
* not an embedding search query, it is the identifier-keyed retrieval of
|
||||
* claim 23 ("node records are stored under a key encoding the node
|
||||
* identifier"). Putting both behind one function erased that boundary, and
|
||||
* a nearest neighbour of the string "soul:boot_count" is not a boot counter.
|
||||
*
|
||||
* MEASURED, on the harness corpus, isolated, read-only, no writes from any
|
||||
* caller: 240 node records destroyed per boot, including 6 Knowledge nodes,
|
||||
* a layer-1 "CORE IDENTITY — GENESIS, LINEAGE" Memory, and the value node
|
||||
* `kn-58874a74` (gold answer for gold-set q15). The deletion list is the
|
||||
* result list of the soul's own mem_boot_count_inc() lookup, in order.
|
||||
*
|
||||
* So: legs OFF here, legs ON in engram_recall_json below, which is what
|
||||
* /api/neuron/recall reaches. Retrieval quality on the recall route is
|
||||
* unchanged; the internal keyed reads get their contract back. */
|
||||
static el_val_t eg_search_json_impl(el_val_t query, el_val_t limit, int with_legs) {
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
int64_t lim = (int64_t)limit;
|
||||
@@ -9695,178 +9259,27 @@ static el_val_t eg_search_json_impl(el_val_t query, el_val_t limit, int with_leg
|
||||
if (ntok > 0) {
|
||||
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
||||
if (hits) {
|
||||
/* Claim-24 semantic leg. This is the function /api/neuron/recall
|
||||
* actually reaches (routes.el -> neuron-api.el handle_api_recall),
|
||||
* so the semantic half of the retrieval surface has to land HERE
|
||||
* to be observable to the MCP wrapper and the app. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = with_legs ? eg_embed_fetch(q, &qdim) : NULL;
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
/* Raw (unfloored) cosine top-K, kept for the graph leg's
|
||||
* semantic seeds. Selected in THIS pass so the cosine is
|
||||
* computed exactly once per node — the seeding costs no extra
|
||||
* pass over the corpus and no extra embed round-trip. */
|
||||
int64_t semseed[ENGRAM_EMBED_SEED_K];
|
||||
double semseedc[ENGRAM_EMBED_SEED_K];
|
||||
int64_t nsemseed = 0;
|
||||
/* BM25 statistics gathered in this same pass: per-token
|
||||
* document frequency, and the corpus mean field length. */
|
||||
int64_t df[ENGRAM_MAX_QTOKENS];
|
||||
for (int t = 0; t < ntok; t++) df[t] = 0;
|
||||
double dl_total = 0.0;
|
||||
int64_t dl_n = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
/* Filter transparent layers — same as engram_search. */
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
/* Unaddressable records cannot be fetched by a caller and
|
||||
* must not consume a result slot (claims 12/23/27). */
|
||||
if (!eg_node_addressable(n)) continue;
|
||||
double dl = engram_node_len(n);
|
||||
dl_total += dl; dl_n++;
|
||||
uint32_t mask = engram_node_match_mask(n, toks, ntok);
|
||||
if (mask) {
|
||||
int sc = 0;
|
||||
for (int t = 0; t < ntok; t++)
|
||||
if (mask & ((uint32_t)1u << t)) { sc++; df[t]++; }
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
if (sc > 0) {
|
||||
hits[nhits].idx = i;
|
||||
hits[nhits].score = sc;
|
||||
hits[nhits].salience = n->salience;
|
||||
hits[nhits].mask = mask;
|
||||
hits[nhits].len = dl;
|
||||
hits[nhits].w = 0.0;
|
||||
nhits++;
|
||||
}
|
||||
if (sem && n->emb && n->emb_dim == qdim) {
|
||||
double c = eg_cosine(n->emb, qv, qdim);
|
||||
/* Claim-24 semantic leg, restored verbatim: "returning
|
||||
* the node records whose embedding vectors have the
|
||||
* HIGHEST COSINE SIMILARITY to a query vector" — a
|
||||
* ranking, with no threshold anywhere in the claim.
|
||||
* ENGRAM_EMBED_SEED_MIN is defined at l.6094 as the
|
||||
* HippoRAG SEED-JOIN threshold; using it as a RESULT
|
||||
* filter here was never authorised, and it is a
|
||||
* per-query lottery rather than a quality gate: the
|
||||
* query's own top-1 cosine ranges 0.56-0.68 across the
|
||||
* held-out gold set, so 0.60 keeps a rank-1 answer for
|
||||
* one query and discards a rank-1 answer for the next.
|
||||
* Measured on the 30 held-out paraphrases: six golds
|
||||
* sit at global cosine rank 1-2 and score 0.564-0.589,
|
||||
* discarded by nothing but this constant.
|
||||
* What holds the nonsense controls is NOT this floor
|
||||
* but the corpus-vocabulary gate below (nhits == 0):
|
||||
* gibberish has no lexical seeds, so no leg reports.
|
||||
* Cosine is clamped to [0,1] per 05-detailed-description
|
||||
* l.69 ("clamped to [0,1] to prevent anti-correlated
|
||||
* embeddings from producing negative activation"). */
|
||||
double sv = c < 0.0 ? 0.0 : (c > 1.0 ? 1.0 : c);
|
||||
if (sv > 0.0) {
|
||||
sem[nsem].idx = i; sem[nsem].sem = sv; nsem++;
|
||||
}
|
||||
/* Graph seeds: top-K by RAW cosine, insertion-ordered. */
|
||||
if (c > 0.0 && (nsemseed < ENGRAM_EMBED_SEED_K
|
||||
|| c > semseedc[nsemseed - 1])) {
|
||||
int64_t p = nsemseed < ENGRAM_EMBED_SEED_K
|
||||
? nsemseed : ENGRAM_EMBED_SEED_K - 1;
|
||||
while (p > 0 && semseedc[p - 1] < c) {
|
||||
semseedc[p] = semseedc[p - 1];
|
||||
semseed[p] = semseed[p - 1];
|
||||
p--;
|
||||
}
|
||||
semseedc[p] = c; semseed[p] = i;
|
||||
if (nsemseed < ENGRAM_EMBED_SEED_K) nsemseed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
/* BM25-shaped lexical score. Binary term frequency (the match
|
||||
* primitive is a substring test, not a count), Lucene-form IDF,
|
||||
* and length normalisation over the corpus mean. A token that
|
||||
* occurs in 30,000 nodes now weighs far less than one that
|
||||
* occurs in 1, and a 1.3 MB record no longer out-matches a
|
||||
* 300-byte one by sheer surface area. */
|
||||
double avgdl = dl_n ? (dl_total / (double)dl_n) : 1.0;
|
||||
if (avgdl <= 0.0) avgdl = 1.0;
|
||||
double idf[ENGRAM_MAX_QTOKENS];
|
||||
for (int t = 0; t < ntok; t++) {
|
||||
double dfx = (double)df[t];
|
||||
idf[t] = log(1.0 + ((double)dl_n - dfx + 0.5) / (dfx + 0.5));
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0);
|
||||
first = 0;
|
||||
}
|
||||
for (int64_t h = 0; h < nhits; h++) {
|
||||
double norm = 1.0 - ENGRAM_BM25_B
|
||||
+ ENGRAM_BM25_B * (hits[h].len / avgdl);
|
||||
double s = 0.0;
|
||||
for (int t = 0; t < ntok; t++)
|
||||
if (hits[h].mask & ((uint32_t)1u << t))
|
||||
s += idf[t] * (ENGRAM_BM25_K1 + 1.0)
|
||||
/ (1.0 + ENGRAM_BM25_K1 * norm);
|
||||
hits[h].w = s;
|
||||
}
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp);
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
if (getenv("EG_DIAG")) {
|
||||
int64_t we = 0, unaddr = 0, found = 0;
|
||||
const char* pid = getenv("EG_DIAG_ID");
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (g->nodes[i].emb) we++;
|
||||
if (!eg_node_addressable(&g->nodes[i])) unaddr++;
|
||||
if (pid && g->nodes[i].id && strcmp(g->nodes[i].id, pid) == 0) found++;
|
||||
}
|
||||
fprintf(stderr, "[EG_DIAG] STORE node_count=%lld with_emb=%lld unaddressable=%lld probe_found=%lld\n",
|
||||
(long long)g->node_count, (long long)we, (long long)unaddr, (long long)found);
|
||||
fprintf(stderr, "[EG_DIAG] q=\"%s\" qdim=%d nhits=%lld nsem=%lld\n",
|
||||
q, (int)qdim, (long long)nhits, (long long)nsem);
|
||||
for (int64_t k = 0; k < 5 && k < nsem; k++)
|
||||
fprintf(stderr, "[EG_DIAG] sem[%lld] cos=%.4f id=%s\n",
|
||||
(long long)k, sem[k].sem, g->nodes[sem[k].idx].id);
|
||||
const char* probe = getenv("EG_DIAG_ID");
|
||||
if (probe) for (int64_t k = 0; k < nsem; k++)
|
||||
if (g->nodes[sem[k].idx].id
|
||||
&& strcmp(g->nodes[sem[k].idx].id, probe) == 0) {
|
||||
fprintf(stderr, "[EG_DIAG] probe at sem rank %lld cos=%.4f\n",
|
||||
(long long)k, sem[k].sem);
|
||||
break;
|
||||
}
|
||||
fflush(stderr);
|
||||
}
|
||||
/* Claim-10 associative leg: expand the top lexical hits along
|
||||
* structural relations only, order the reached set by query
|
||||
* similarity. Empty whenever the seeds have no structural
|
||||
* edges, which is the common case and is what keeps the
|
||||
* lexical ordering safe. */
|
||||
EngramSemEntry* assoc = qv ? malloc((size_t)ENGRAM_ASSOC_MAX * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nassoc = assoc
|
||||
? engram_assoc_leg(g, hits, nhits, semseed, nsemseed,
|
||||
qv, qdim, assoc, ENGRAM_ASSOC_MAX)
|
||||
: 0;
|
||||
/* Corpus-vocabulary gate. If no stored record contains ANY
|
||||
* query token in its content, label or tags, the query is
|
||||
* outside this graph's vocabulary: there are no seeds, and
|
||||
* 05-detailed-description l.64 makes retrieval downstream of
|
||||
* seeds ("the caller provides one or more seed node UUIDs
|
||||
* representing the current active context"). No seeds, no
|
||||
* retrieval — the graph declines rather than confabulating a
|
||||
* nearest neighbour for gibberish. The mechanism is iteration
|
||||
* 6's (feat/claim24-unfloored-semantic); it is required here
|
||||
* because word-start matching empties the lexical leg for
|
||||
* q35-style queries whose only "hits" were mid-word, and the
|
||||
* semantic leg would otherwise answer them anyway. */
|
||||
int64_t* order = (nhits > 0) ? malloc((size_t)lim * sizeof(int64_t)) : NULL;
|
||||
if (order) {
|
||||
int64_t no = engram_interleave3(hits, nhits, sem, nsem,
|
||||
assoc, nassoc, lim, order);
|
||||
for (int64_t k = 0; k < no; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[order[k]], 0);
|
||||
first = 0;
|
||||
}
|
||||
free(order);
|
||||
}
|
||||
free(assoc);
|
||||
free(hits);
|
||||
free(sem);
|
||||
free(qv);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9874,19 +9287,6 @@ static el_val_t eg_search_json_impl(el_val_t query, el_val_t limit, int with_leg
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
/* Lexical keyed read — the historical contract every internal caller relies
|
||||
* on. Every returned record CONTAINS a query token. */
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
return eg_search_json_impl(query, limit, 0);
|
||||
}
|
||||
|
||||
/* The retrieval surface: lexical + claim-24 semantic + claim-10 associative,
|
||||
* rank-fused. Reached from handle_api_recall (/api/neuron/recall) — the route
|
||||
* the MCP wrapper and the app call, and the one the eval harness measures. */
|
||||
el_val_t engram_recall_json(el_val_t query, el_val_t limit) {
|
||||
return eg_search_json_impl(query, limit, 1);
|
||||
}
|
||||
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
|
||||
EngramStore* g = engram_get();
|
||||
int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100;
|
||||
|
||||
@@ -612,7 +612,6 @@ el_val_t engram_load(el_val_t path);
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_recall_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
|
||||
Reference in New Issue
Block a user