neuron-technologies/neuron
3
0
Fork 0
Code Issues 22 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

self-review 2026-07-28: close ISE lifecycle observability gaps

Browse Source 
- session_start now also posted to the HTTP Engram via ise_post: the
  local engram_node_full write never crossed to the observable stream
  (sync flows HTTP->soul only), so boots 5+ were invisible — last
  visible session_start was boot 4, two weeks ago
- graceful shutdown emits a final ISE with boot/pulse/uptime; a boot
  with no shutdown event now reliably signals a crash/SIGKILL
- empty /api/sync responses emit a sync_empty warn ISE instead of
  being silently skipped — unreachable engram no longer looks
  identical to quiet-but-healthy
- sync backflow prune reads ENGRAM_ISE_RETENTION_MS instead of
  duplicating the 48h magic number server.el already honors
This commit is contained in:
Will Anderson
2026-07-28 08:37:39 -05:00
parent 2b612ed5d4
commit 627eb534a2
4 changed files with 105 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
awareness.el
+29 -4
View File
@@ -779,6 +779,17 @@ fn awareness_run() -> Void {
let tick_mark: Any = el_arena_push()
let running: String = state_get("soul.running")
if str_eq(running, "false") {
// Shutdown ISE (2026-07-28 self-review): graceful exit was invisible
// in the ISE stream — indistinguishable from a crash or a frozen
// loop. Emit a final event with uptime/pulse so the stream records
// how this boot ended. (Covers state-driven shutdown only; SIGKILL
// still leaves no trace — that absence is itself the crash signal.)
let sd_boot_raw: String = state_get("soul_boot_count")
let sd_boot: String = if str_eq(sd_boot_raw, "") { "0" } else { sd_boot_raw }
ise_post("{\"event\":\"shutdown\",\"boot\":" + sd_boot
+ ",\"pulse\":" + int_to_str(pulse_count())
+ ",\"uptime_ms\":" + int_to_str(elapsed_ms())
+ ",\"ts\":" + int_to_str(time_now()) + "}")
println("[awareness] exiting")
el_arena_pop(tick_mark)
return ""
@@ -849,15 +860,29 @@ fn awareness_run() -> Void {
let engram_url: String = if str_eq(sync_state_url, "") { "http://localhost:8742" } else { sync_state_url }
if !str_eq(engram_url, "") {
let sync_json: String = http_get(engram_url + "/api/sync")
if !str_eq(sync_json, "") && !str_eq(sync_json, "{}") {
let sync_ok: Bool = !str_eq(sync_json, "") && !str_eq(sync_json, "{}")
// Empty-sync warn ISE (2026-07-28 self-review): an empty/{} sync
// response was silently skipped, making "engram unreachable for
// hours" look identical to "quiet but healthy" — the exact
// failure class the URL-fallback comment above fights. One warn
// event per refresh interval (10 min default) is cheap and makes
// a persistently-failing sync visible in the stream itself.
if !sync_ok {
ise_post("{\"event\":\"sync_empty\",\"ts\":" + int_to_str(time_now()) + "}")
}
if sync_ok {
let cgi_id: String = state_get("soul_cgi_id")
let tmp: String = "/tmp/soul-sync-" + cgi_id + ".json"
fs_write(tmp, sync_json)
let added: Int = engram_load_merge(tmp)
// Backflow control: the merged snapshot carries ISE telemetry
// from the HTTP store. Prune anything older than 48h — same
// horizon the HTTP store itself uses (server.el ISE insert).
let pruned_sync: Int = engram_prune_telemetry(172800000)
// from the HTTP store. Prune on the same horizon the HTTP
// store itself uses — read ENGRAM_ISE_RETENTION_MS (the env
// server.el honors) instead of duplicating the 48h magic
// number, so the two stores can't drift. (2026-07-28)
let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
let pruned_sync: Int = engram_prune_telemetry(ret_ms)
// Running total of merged-in nodes this boot, surfaced in the
// heartbeat ISE as sync_added_total (same state mechanism as
// the pulse counter).