will.anderson
fa47b98d18
Self-contained paged store (lang/runtime/engram_store.{c,h}): 16KiB slotted pages,
u32 TLV self-describing records (forward-compatible), overflow chains, B+-tree
id-index + from/to adjacency, page free-list, tombstones, double superblock + crc
recovery. Not yet wired to activation (M3). 33/33 tests pass (ASan/UBSan clean);
5k nodes/20k edges round-trip bit-exact incl 768xf32 emb + hebb; store 25MB vs 64MB
JSON. Format is final — see design §2.4.
136 lines
5.0 KiB
C
136 lines
5.0 KiB
C
/* engram_store.h — M1 of the engram tiered storage engine.
|
|
*
|
|
* The FINAL on-disk paged store format: superblock (+ mirror), slotted pages,
|
|
* self-describing TLV records, overflow chains, and two B+-tree indexes
|
|
* (primary id->loc, adjacency from_id/to_id->edge-locs) over a free-listed
|
|
* page file. See docs/architecture/design/engram-tiered-storage-engine.md §2.
|
|
*
|
|
* This is a self-contained module (plain C, standard libs only). It defines its
|
|
* own serializable views of a node/edge (StoreNode/StoreEdge) that mirror every
|
|
* persisted field of EngramNode/EngramEdge in el_runtime.c. M3 maps between the
|
|
* live runtime structs and these; M1 does not touch el_runtime.c.
|
|
*
|
|
* Format id: magic "ENGST01", format_version 1. This format is PERMANENT — the
|
|
* TLV record scheme means new fields never force a migration.
|
|
*/
|
|
#ifndef ENGRAM_STORE_H
|
|
#define ENGRAM_STORE_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* Fixed for the life of a store; recorded in the superblock. */
|
|
#define STORE_PAGE_SIZE 16384u
|
|
#define STORE_MAGIC "ENGST01" /* 7 chars + NUL stored in an 8-byte field */
|
|
#define STORE_FORMAT_VERSION 1u
|
|
|
|
/* Ring-buffer length for ACT-R base-level access timestamps.
|
|
* MUST equal ENGRAM_BLL_K in el_runtime.c (currently 10). Static-checked in .c. */
|
|
#define STORE_BLL_K 10
|
|
|
|
/* Page types (page header byte). */
|
|
enum {
|
|
STORE_PT_NODE = 1,
|
|
STORE_PT_EDGE = 2,
|
|
STORE_PT_INDEX = 3,
|
|
STORE_PT_OVERFLOW = 4,
|
|
STORE_PT_FREE = 5
|
|
};
|
|
|
|
/* store_check flags. */
|
|
#define STORE_CHECK_CRC 1u
|
|
|
|
/* ── Serializable node view: every persisted EngramNode field ─────────────── */
|
|
typedef struct StoreNode {
|
|
char* id;
|
|
char* content;
|
|
char* node_type;
|
|
char* label;
|
|
char* tier;
|
|
char* tags;
|
|
char* metadata;
|
|
double salience;
|
|
double importance;
|
|
double confidence;
|
|
double temporal_decay_rate;
|
|
int64_t activation_count;
|
|
int64_t last_activated;
|
|
int64_t created_at;
|
|
int64_t updated_at;
|
|
double background_activation;
|
|
double working_memory_weight;
|
|
int32_t suppression_count;
|
|
uint32_t layer_id;
|
|
int64_t access_ts[STORE_BLL_K];
|
|
int32_t access_head;
|
|
int32_t access_filled;
|
|
double wm_anchor;
|
|
float* emb; /* owned; NULL if not embedded */
|
|
int32_t emb_dim;
|
|
/* Forward-compat: raw bytes of any TLV fields the reader did not recognise,
|
|
* concatenated verbatim ([tag][u32 len][bytes]...). Re-emitted on write so
|
|
* an old reader never drops a newer writer's fields. */
|
|
uint8_t* unknown;
|
|
size_t unknown_len;
|
|
int tombstoned; /* set by store_get_* if the located record is dead */
|
|
/* hebb_elig / hebb_elig_ts are DELIBERATELY NOT persisted (see EngramNode). */
|
|
} StoreNode;
|
|
|
|
/* ── Serializable edge view: every persisted EngramEdge field ─────────────── */
|
|
typedef struct StoreEdge {
|
|
char* id;
|
|
char* from_id;
|
|
char* to_id;
|
|
char* relation;
|
|
char* metadata;
|
|
double weight;
|
|
double hebb;
|
|
double confidence;
|
|
int64_t created_at;
|
|
int64_t updated_at;
|
|
int64_t last_fired;
|
|
int32_t inhibitory;
|
|
uint32_t layer_id;
|
|
uint8_t* unknown;
|
|
size_t unknown_len;
|
|
int tombstoned;
|
|
} StoreEdge;
|
|
|
|
typedef struct EngramPagedStore EngramPagedStore;
|
|
|
|
/* Lifecycle. */
|
|
EngramPagedStore* store_create(const char* path); /* fails if file exists */
|
|
EngramPagedStore* store_open(const char* path); /* recovers via mirror SB */
|
|
int store_close(EngramPagedStore* s); /* syncs + frees */
|
|
int store_sync(EngramPagedStore* s); /* fsync + rewrite both superblocks */
|
|
|
|
/* Nodes. store_get_node returns 1 on hit (fills *out, caller store_node_free),
|
|
* 0 if absent or tombstoned, <0 on error. */
|
|
int store_put_node(EngramPagedStore* s, const StoreNode* n);
|
|
int store_get_node(EngramPagedStore* s, const char* id, StoreNode* out);
|
|
int store_tombstone(EngramPagedStore* s, const char* id);
|
|
|
|
/* Edges. *out is malloc'd (store_edges_free); *n set to count. */
|
|
int store_put_edge(EngramPagedStore* s, const StoreEdge* e);
|
|
int store_get_edges_from(EngramPagedStore* s, const char* from_id, StoreEdge** out, size_t* n);
|
|
int store_get_edges_to(EngramPagedStore* s, const char* to_id, StoreEdge** out, size_t* n);
|
|
|
|
/* Integrity: verify every page's crc (and both superblocks). Returns the number
|
|
* of corrupt pages (0 = clean), or <0 on I/O error. */
|
|
int store_check(EngramPagedStore* s, unsigned flags);
|
|
|
|
/* Ownership helpers. */
|
|
void store_node_free(StoreNode* n);
|
|
void store_edge_free(StoreEdge* e);
|
|
void store_edges_free(StoreEdge* arr, size_t n);
|
|
|
|
/* Test-only hook (NOT a format property — B+-tree nodes are self-describing via
|
|
* their stored key count). Caps entries/keys per index node to force splits on
|
|
* small datasets. 0 = natural full-page fanout. */
|
|
void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max);
|
|
|
|
/* Introspection for tests/tools. */
|
|
uint64_t store_page_count(const EngramPagedStore* s);
|
|
|
|
#endif /* ENGRAM_STORE_H */
|