rename crates/ to engrams/, bindings/ to receptors/
- crates/ → engrams/ (Rust engrams live here) - bindings/ → receptors/ (cross-language access points into the graph) - Cargo.toml workspace paths updated
This commit is contained in:
Will Anderson
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "engram-core"
|
||||
version = "0.1.1"
|
||||
edition = "2021"
|
||||
description = "Engram — native memory substrate for accumulating intelligence"
|
||||
license = "MIT"
|
||||
|
||||
[features]
|
||||
default = ["sled-backend"]
|
||||
sled-backend = ["dep:sled"]
|
||||
wasm = []
|
||||
migration = ["dep:rusqlite"]
|
||||
|
||||
[dependencies]
|
||||
sled = { version = "0.34", optional = true }
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
bincode = "1"
|
||||
anyhow = "1"
|
||||
thiserror = "1"
|
||||
instant-distance = { version = "0.6", features = ["with-serde"] }
|
||||
rusqlite = { version = "0.31", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,318 @@
|
||||
/// Spreading Activation — the core retrieval mechanism of Engram.
|
||||
///
|
||||
/// # The Central Insight
|
||||
///
|
||||
/// Conventional databases separate storage from retrieval. You put data in,
|
||||
/// you query it out. The storage structure (B-tree, LSM, etc.) and the retrieval
|
||||
/// mechanism (SQL planner, index scan) are fundamentally different things.
|
||||
///
|
||||
/// The brain doesn't work this way. Memory is not stored and retrieved —
|
||||
/// it is **activated and propagated**. When you remember something, you don't
|
||||
/// "query" your hippocampus. You activate a node and the pattern spreads through
|
||||
/// weighted connections to neighboring nodes. Long-term potentiation IS the storage
|
||||
/// structure AND the retrieval mechanism simultaneously.
|
||||
///
|
||||
/// This module implements that model directly.
|
||||
///
|
||||
/// # How It Works
|
||||
///
|
||||
/// 1. **Seeds**: Start with one or more known node UUIDs (e.g., the most recent
|
||||
/// context, the current task, recent observations).
|
||||
///
|
||||
/// 2. **Query embedding**: The semantic vector representing what you're looking
|
||||
/// for. This is the "direction of thought" — activation flows more strongly
|
||||
/// toward nodes that are semantically similar to the current context.
|
||||
///
|
||||
/// 3. **BFS propagation**: Activation spreads outward from seeds through edges.
|
||||
/// At each hop, the strength attenuates based on:
|
||||
/// - `edge.weight`: how strongly these two nodes are associated
|
||||
/// - `target.salience`: how salient (recently activated, frequent, important) the target is
|
||||
/// - `cosine_sim(query, target)`: how semantically relevant the target is to what we want
|
||||
///
|
||||
/// 4. **Pruning**: Paths with activation strength below `PRUNE_THRESHOLD` are cut.
|
||||
/// This prevents exponential blowup and models the brain's attention filter.
|
||||
///
|
||||
/// 5. **Return**: The top-N nodes by activation strength, with their hop distance.
|
||||
///
|
||||
/// # Activation Formula (per hop)
|
||||
///
|
||||
/// strength = parent_strength × edge_weight × target_salience × cosine_sim(query, target)
|
||||
///
|
||||
/// This is multiplicative: a weak edge, a dormant node, or a semantically irrelevant
|
||||
/// target all suppress activation. All four factors must be non-trivial for a path
|
||||
/// to propagate successfully. This is exactly how associative memory works.
|
||||
///
|
||||
/// # Why Multiplication, Not Addition
|
||||
///
|
||||
/// Addition would allow many weak signals to accumulate into false relevance.
|
||||
/// The brain's associative memory is conjunctive: an activated path requires
|
||||
/// ALL of its links to be strong enough to carry the signal. Multiplication
|
||||
/// enforces this. If any factor is near zero, the path dies.
|
||||
use crate::error::EngramResult;
|
||||
use crate::types::{ActivatedNode, Node};
|
||||
use crate::vector::cosine_similarity;
|
||||
use std::collections::{BinaryHeap, HashMap};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use crate::graph;
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use sled::Db;
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
use crate::mem_storage::MemStore;
|
||||
|
||||
/// Activation strengths below this threshold are pruned from the BFS frontier.
|
||||
/// 0.01 is deliberately small — we want to allow long indirect chains when
|
||||
/// the intermediate edges are strong. Raise this to focus retrieval, lower to
|
||||
/// allow more associative drift.
|
||||
const PRUNE_THRESHOLD: f32 = 0.01;
|
||||
|
||||
// We need Ord on (f32, Uuid) for the priority queue. Use a wrapper.
|
||||
#[derive(PartialEq)]
|
||||
struct Candidate {
|
||||
strength: f32,
|
||||
hops: u8,
|
||||
id: Uuid,
|
||||
}
|
||||
|
||||
impl Eq for Candidate {}
|
||||
|
||||
impl PartialOrd for Candidate {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for Candidate {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
// BinaryHeap is a max-heap; we want highest-strength first
|
||||
self.strength
|
||||
.partial_cmp(&other.strength)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run spreading activation from a set of seed nodes.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `db` — the open engram database
|
||||
/// * `seeds` — starting node IDs (the current "active set")
|
||||
/// * `query_embedding` — semantic vector representing what we're looking for
|
||||
/// * `max_depth` — maximum number of hops to traverse (typically 2–4)
|
||||
/// * `limit` — return only the top-N results
|
||||
///
|
||||
/// # Returns
|
||||
/// Up to `limit` nodes, sorted by activation strength descending.
|
||||
/// Seed nodes themselves are excluded from the result (they're already known).
|
||||
#[cfg(feature = "sled-backend")]
|
||||
pub fn activate(
|
||||
db: &Db,
|
||||
seeds: &[Uuid],
|
||||
query_embedding: &[f32],
|
||||
max_depth: u8,
|
||||
limit: usize,
|
||||
) -> EngramResult<Vec<ActivatedNode>> {
|
||||
// best_strength[id] = highest activation strength seen so far for this node.
|
||||
// We use this to handle cases where multiple paths lead to the same node —
|
||||
// the strongest path wins (like the brain's winner-take-most competition).
|
||||
let mut best_strength: HashMap<Uuid, (f32, u8)> = HashMap::new();
|
||||
|
||||
// Priority queue: process highest-strength candidates first.
|
||||
// This is a best-first BFS — we explore the most promising paths before
|
||||
// weaker ones, which means pruning cuts off genuinely unimportant branches.
|
||||
let mut queue: BinaryHeap<Candidate> = BinaryHeap::new();
|
||||
|
||||
// Initialize: seed nodes start with full strength (1.0).
|
||||
// They represent our current context — fully activated, zero hops away.
|
||||
for &seed in seeds {
|
||||
// Seeds are tracked with strength 1.0 but NOT added to best_strength yet;
|
||||
// we want to allow other paths to reach them if they form a cycle.
|
||||
// However, we must visit their neighbors. We add seeds directly.
|
||||
queue.push(Candidate {
|
||||
strength: 1.0,
|
||||
hops: 0,
|
||||
id: seed,
|
||||
});
|
||||
// Mark seeds so we don't re-process them as results, but allow
|
||||
// re-traversal from them if another path arrives stronger.
|
||||
best_strength.insert(seed, (1.0, 0));
|
||||
}
|
||||
|
||||
// BFS / best-first traversal
|
||||
while let Some(Candidate { strength, hops, id }) = queue.pop() {
|
||||
// Depth limit: don't propagate beyond max_depth
|
||||
if hops >= max_depth {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Retrieve outgoing edges from the current node
|
||||
let edges = graph::edges_from(db, id)?;
|
||||
|
||||
for edge in &edges {
|
||||
let target_id = edge.to_id;
|
||||
|
||||
// Load the target node. If it doesn't exist (dangling edge), skip.
|
||||
let target: Node = match graph::get_node(db, target_id)? {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
// ── Activation strength computation ──────────────────────────
|
||||
//
|
||||
// Each factor models a distinct aspect of associative memory:
|
||||
//
|
||||
// 1. parent_strength: how strongly was the parent activated?
|
||||
// Activation attenuates with each hop — deep chains carry less signal.
|
||||
//
|
||||
// 2. edge.weight: how strong is the association between these nodes?
|
||||
// High-weight edges are like well-worn neural pathways — low resistance.
|
||||
// Low-weight edges are new or rarely traversed — they carry little signal.
|
||||
//
|
||||
// 3. target.salience: how salient is the target node right now?
|
||||
// Dormant nodes (low salience) resist activation.
|
||||
// Frequently-used, recently-touched nodes activate easily.
|
||||
// This is how recency and frequency bias retrieval, as in human memory.
|
||||
//
|
||||
// 4. cosine_sim(query, target): semantic relevance.
|
||||
// If the target's embedding is far from what we're looking for,
|
||||
// the activation doesn't flow there. This is the "direction of thought"
|
||||
// filtering — the query steers the spread toward relevant regions.
|
||||
//
|
||||
// The product of all four is the activation strength at the target.
|
||||
// All factors are in [0, 1] so the product is also in [0, 1].
|
||||
// (Salience can exceed 1 for very active nodes, which is fine —
|
||||
// it means those nodes are hyper-salient, like obsessive thoughts.)
|
||||
|
||||
let semantic_sim = cosine_similarity(query_embedding, &target.embedding);
|
||||
// We clamp semantic_sim to [0, 1] so that anti-correlated embeddings
|
||||
// don't produce negative activation (which would invert the signal).
|
||||
let semantic_sim = semantic_sim.max(0.0);
|
||||
|
||||
let new_strength = strength * edge.weight * target.salience.max(0.0) * semantic_sim;
|
||||
|
||||
// Prune: if this path is too weak to matter, stop here.
|
||||
// This is the attention filter — irrelevant associations fade away.
|
||||
if new_strength < PRUNE_THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
|
||||
let next_hops = hops + 1;
|
||||
|
||||
// Winner-take-most: only propagate from this node if this is the
|
||||
// strongest path we've seen to it so far. This prevents exponential
|
||||
// blowup when the graph has many parallel paths to the same node.
|
||||
let is_stronger = match best_strength.get(&target_id) {
|
||||
Some(&(prev, _)) => new_strength > prev,
|
||||
None => true,
|
||||
};
|
||||
|
||||
if is_stronger {
|
||||
best_strength.insert(target_id, (new_strength, next_hops));
|
||||
queue.push(Candidate {
|
||||
strength: new_strength,
|
||||
hops: next_hops,
|
||||
id: target_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect results: exclude seed nodes, load full Node structs, sort by strength
|
||||
let seed_set: std::collections::HashSet<Uuid> = seeds.iter().copied().collect();
|
||||
|
||||
let mut results: Vec<ActivatedNode> = Vec::new();
|
||||
for (id, (strength, hops)) in &best_strength {
|
||||
if seed_set.contains(id) {
|
||||
continue;
|
||||
}
|
||||
if let Some(node) = graph::get_node(db, *id)? {
|
||||
results.push(ActivatedNode {
|
||||
node,
|
||||
activation_strength: *strength,
|
||||
hops: *hops,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by activation strength descending, take top N
|
||||
results.sort_by(|a, b| {
|
||||
b.activation_strength
|
||||
.partial_cmp(&a.activation_strength)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
results.truncate(limit);
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// In-memory spreading activation for the WASM backend.
|
||||
///
|
||||
/// Identical algorithm to `activate` but reads from a `MemStore` instead of sled.
|
||||
#[cfg(feature = "wasm")]
|
||||
pub fn activate_mem(
|
||||
store: &MemStore,
|
||||
seeds: &[Uuid],
|
||||
query_embedding: &[f32],
|
||||
max_depth: u8,
|
||||
limit: usize,
|
||||
) -> EngramResult<Vec<ActivatedNode>> {
|
||||
let mut best_strength: HashMap<Uuid, (f32, u8)> = HashMap::new();
|
||||
let mut queue: BinaryHeap<Candidate> = BinaryHeap::new();
|
||||
|
||||
for &seed in seeds {
|
||||
queue.push(Candidate { strength: 1.0, hops: 0, id: seed });
|
||||
best_strength.insert(seed, (1.0, 0));
|
||||
}
|
||||
|
||||
while let Some(Candidate { strength, hops, id }) = queue.pop() {
|
||||
if hops >= max_depth {
|
||||
continue;
|
||||
}
|
||||
let edges = store.read_edges_from(id)?;
|
||||
for edge in &edges {
|
||||
let target_id = edge.to_id;
|
||||
let target: Node = match store.read_node(target_id)? {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
let semantic_sim = cosine_similarity(query_embedding, &target.embedding).max(0.0);
|
||||
let new_strength = strength * edge.weight * target.salience.max(0.0) * semantic_sim;
|
||||
if new_strength < PRUNE_THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
let next_hops = hops + 1;
|
||||
let is_stronger = match best_strength.get(&target_id) {
|
||||
Some(&(prev, _)) => new_strength > prev,
|
||||
None => true,
|
||||
};
|
||||
if is_stronger {
|
||||
best_strength.insert(target_id, (new_strength, next_hops));
|
||||
queue.push(Candidate { strength: new_strength, hops: next_hops, id: target_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let seed_set: std::collections::HashSet<Uuid> = seeds.iter().copied().collect();
|
||||
let mut results: Vec<ActivatedNode> = Vec::new();
|
||||
for (id, (strength, hops)) in &best_strength {
|
||||
if seed_set.contains(id) {
|
||||
continue;
|
||||
}
|
||||
if let Some(node) = store.read_node(*id)? {
|
||||
results.push(ActivatedNode {
|
||||
node,
|
||||
activation_strength: *strength,
|
||||
hops: *hops,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| {
|
||||
b.activation_strength
|
||||
.partial_cmp(&a.activation_strength)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
results.truncate(limit);
|
||||
Ok(results)
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
/// Consolidation — promoting Episodic memories to Semantic knowledge.
|
||||
///
|
||||
/// Biological memory consolidation is the process by which unstable,
|
||||
/// hippocampus-dependent memories are gradually transformed into stable,
|
||||
/// neocortex-integrated semantic knowledge. In the brain this happens
|
||||
/// primarily during sleep through hippocampal replay.
|
||||
///
|
||||
/// Here, consolidation is explicit and on-demand. The caller decides when to
|
||||
/// run a consolidation cycle and with what thresholds. The engine:
|
||||
///
|
||||
/// 1. Scans all Episodic nodes.
|
||||
/// 2. Promotes those that have been activated enough (high activation_count)
|
||||
/// and are still salient enough (above salience_floor) to MemoryTier::Semantic.
|
||||
/// 3. Runs a global salience decay pass to age all nodes.
|
||||
/// 4. Returns a report of what changed.
|
||||
///
|
||||
/// This models the idea that memories become "knowledge" not by being told they
|
||||
/// should be, but by being *used* — activated, reinforced, and found relevant
|
||||
/// repeatedly over time.
|
||||
use crate::error::EngramResult;
|
||||
use crate::salience;
|
||||
use crate::types::{MemoryTier, Node};
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use crate::storage;
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use crate::graph;
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use sled::Db;
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
use crate::mem_storage::MemStore;
|
||||
|
||||
/// Configuration for a consolidation run.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ConsolidationConfig {
|
||||
/// Episodic nodes with activation_count >= this threshold are candidates for promotion.
|
||||
pub episodic_to_semantic_threshold: u64,
|
||||
/// Candidates must also have salience >= this floor to be promoted.
|
||||
pub salience_floor: f32,
|
||||
/// Maximum number of promotions per consolidation cycle (prevents runaway batch writes).
|
||||
pub max_promotions_per_run: usize,
|
||||
/// Decay factor applied to all node saliences after promotion (0.0–1.0).
|
||||
pub decay_factor: f32,
|
||||
}
|
||||
|
||||
impl Default for ConsolidationConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
episodic_to_semantic_threshold: 5,
|
||||
salience_floor: 0.3,
|
||||
max_promotions_per_run: 50,
|
||||
decay_factor: 0.98,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Summary of what happened during a consolidation cycle.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ConsolidationReport {
|
||||
/// Number of Episodic nodes promoted to Semantic.
|
||||
pub promoted: usize,
|
||||
/// Number of nodes whose salience was updated by the decay pass.
|
||||
pub decayed: usize,
|
||||
/// Number of nodes removed because their salience dropped below the minimum
|
||||
/// (currently unused — pruning is opt-in in v0.1).
|
||||
pub pruned: usize,
|
||||
}
|
||||
|
||||
// ── sled-backed consolidation ─────────────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
/// Run a consolidation cycle against the open sled database.
|
||||
pub fn consolidate(db: &Db, config: &ConsolidationConfig) -> EngramResult<ConsolidationReport> {
|
||||
let mut report = ConsolidationReport::default();
|
||||
|
||||
// Step 1: scan all nodes, identify Episodic candidates.
|
||||
let all_nodes: Vec<Node> = storage::scan_nodes(db)?;
|
||||
|
||||
let mut promoted_count = 0usize;
|
||||
|
||||
for mut node in all_nodes {
|
||||
if node.tier != MemoryTier::Episodic {
|
||||
continue;
|
||||
}
|
||||
|
||||
if node.activation_count >= config.episodic_to_semantic_threshold
|
||||
&& node.salience >= config.salience_floor
|
||||
{
|
||||
// Promote: change tier to Semantic and persist.
|
||||
// Use overwrite_node — this is an internal state update, not a new node.
|
||||
node.tier = MemoryTier::Semantic;
|
||||
graph::overwrite_node(db, &node)?;
|
||||
promoted_count += 1;
|
||||
|
||||
if promoted_count >= config.max_promotions_per_run {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.promoted = promoted_count;
|
||||
|
||||
// Step 2: global salience decay.
|
||||
let all_nodes_post: Vec<Node> = storage::scan_nodes(db)?;
|
||||
let mut decayed_count = 0usize;
|
||||
for mut node in all_nodes_post {
|
||||
let new_sal = salience::decay_salience(node.salience, config.decay_factor);
|
||||
if new_sal != node.salience {
|
||||
node.salience = new_sal;
|
||||
storage::write_salience(db, node.id, new_sal)?;
|
||||
graph::overwrite_node(db, &node)?;
|
||||
decayed_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
report.decayed = decayed_count;
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
// ── in-memory consolidation (wasm) ────────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
/// Run a consolidation cycle against the in-memory store.
|
||||
pub fn consolidate_mem(
|
||||
store: &mut MemStore,
|
||||
config: &ConsolidationConfig,
|
||||
) -> EngramResult<ConsolidationReport> {
|
||||
let mut report = ConsolidationReport::default();
|
||||
let mut promoted_count = 0usize;
|
||||
|
||||
let all_ids: Vec<uuid::Uuid> = store.nodes.keys().copied().collect();
|
||||
|
||||
for id in &all_ids {
|
||||
if promoted_count >= config.max_promotions_per_run {
|
||||
break;
|
||||
}
|
||||
if let Some(node) = store.nodes.get_mut(id) {
|
||||
if node.tier == MemoryTier::Episodic
|
||||
&& node.activation_count >= config.episodic_to_semantic_threshold
|
||||
&& node.salience >= config.salience_floor
|
||||
{
|
||||
node.tier = MemoryTier::Semantic;
|
||||
promoted_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.promoted = promoted_count;
|
||||
|
||||
let mut decayed_count = 0usize;
|
||||
for node in store.nodes.values_mut() {
|
||||
let new_sal = salience::decay_salience(node.salience, config.decay_factor);
|
||||
if new_sal != node.salience {
|
||||
node.salience = new_sal;
|
||||
decayed_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
report.decayed = decayed_count;
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{MemoryTier, Node, NodeType};
|
||||
|
||||
fn episodic_node_with_activations(count: u64, salience: f32) -> Node {
|
||||
let mut node = Node::new(
|
||||
NodeType::Memory,
|
||||
vec![0.5; 4],
|
||||
b"test memory".to_vec(),
|
||||
MemoryTier::Episodic,
|
||||
0.8,
|
||||
);
|
||||
node.activation_count = count;
|
||||
node.salience = salience;
|
||||
node
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_config_sensible_values() {
|
||||
let cfg = ConsolidationConfig::default();
|
||||
assert_eq!(cfg.episodic_to_semantic_threshold, 5);
|
||||
assert!(cfg.salience_floor > 0.0);
|
||||
assert!(cfg.max_promotions_per_run > 0);
|
||||
assert!(cfg.decay_factor > 0.0 && cfg.decay_factor <= 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_is_promotion_candidate() {
|
||||
let cfg = ConsolidationConfig::default();
|
||||
let node = episodic_node_with_activations(10, 0.8);
|
||||
let is_candidate = node.tier == MemoryTier::Episodic
|
||||
&& node.activation_count >= cfg.episodic_to_semantic_threshold
|
||||
&& node.salience >= cfg.salience_floor;
|
||||
assert!(is_candidate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_below_threshold_not_candidate() {
|
||||
let cfg = ConsolidationConfig::default();
|
||||
// activation_count below threshold
|
||||
let node = episodic_node_with_activations(2, 0.8);
|
||||
let is_candidate = node.tier == MemoryTier::Episodic
|
||||
&& node.activation_count >= cfg.episodic_to_semantic_threshold
|
||||
&& node.salience >= cfg.salience_floor;
|
||||
assert!(!is_candidate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_below_salience_floor_not_candidate() {
|
||||
let cfg = ConsolidationConfig::default();
|
||||
// salience below floor
|
||||
let node = episodic_node_with_activations(10, 0.1);
|
||||
let is_candidate = node.tier == MemoryTier::Episodic
|
||||
&& node.activation_count >= cfg.episodic_to_semantic_threshold
|
||||
&& node.salience >= cfg.salience_floor;
|
||||
assert!(!is_candidate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decay_reduces_salience() {
|
||||
let original = 1.0f32;
|
||||
let decayed = salience::decay_salience(original, 0.98);
|
||||
assert!(decayed < original);
|
||||
assert!((decayed - 0.98).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_default_is_zero() {
|
||||
let r = ConsolidationReport::default();
|
||||
assert_eq!(r.promoted, 0);
|
||||
assert_eq!(r.decayed, 0);
|
||||
assert_eq!(r.pruned, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
#[test]
|
||||
fn consolidate_promotes_eligible_episodic_nodes() {
|
||||
use crate::graph;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let sled_db = sled::open(dir.path()).unwrap();
|
||||
|
||||
let node = episodic_node_with_activations(10, 0.8);
|
||||
graph::put_node(&sled_db, &node).unwrap();
|
||||
|
||||
let cfg = ConsolidationConfig::default();
|
||||
let report = consolidate(&sled_db, &cfg).unwrap();
|
||||
|
||||
assert_eq!(report.promoted, 1);
|
||||
|
||||
let stored = graph::get_node(&sled_db, node.id).unwrap().unwrap();
|
||||
assert_eq!(stored.tier, MemoryTier::Semantic);
|
||||
}
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
#[test]
|
||||
fn consolidate_respects_max_promotions() {
|
||||
use crate::graph;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let sled_db = sled::open(dir.path()).unwrap();
|
||||
|
||||
// Insert 10 eligible nodes.
|
||||
for _ in 0..10 {
|
||||
let node = episodic_node_with_activations(20, 0.9);
|
||||
graph::put_node(&sled_db, &node).unwrap();
|
||||
}
|
||||
|
||||
let cfg = ConsolidationConfig {
|
||||
max_promotions_per_run: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let report = consolidate(&sled_db, &cfg).unwrap();
|
||||
assert_eq!(report.promoted, 3);
|
||||
}
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
#[test]
|
||||
fn consolidate_runs_decay_after_promotion() {
|
||||
use crate::graph;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let sled_db = sled::open(dir.path()).unwrap();
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Concept,
|
||||
vec![0.0; 4],
|
||||
b"semantic".to_vec(),
|
||||
MemoryTier::Semantic,
|
||||
0.5,
|
||||
);
|
||||
let original_salience = node.salience;
|
||||
graph::put_node(&sled_db, &node).unwrap();
|
||||
|
||||
let cfg = ConsolidationConfig::default();
|
||||
let report = consolidate(&sled_db, &cfg).unwrap();
|
||||
|
||||
assert!(report.decayed >= 1);
|
||||
let stored = graph::get_node(&sled_db, node.id).unwrap().unwrap();
|
||||
assert!(stored.salience < original_salience);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,549 @@
|
||||
/// EngramDb — the top-level database handle.
|
||||
///
|
||||
/// All public API methods live here. The internal modules (graph, vector,
|
||||
/// activation, salience, consolidation) are implementation details. Callers
|
||||
/// interact only with EngramDb.
|
||||
///
|
||||
/// # Feature flags
|
||||
/// - `sled-backend` (default): persistent storage via sled
|
||||
/// - `wasm`: in-memory storage only (no filesystem), for WASM targets
|
||||
|
||||
// ── sled-backed implementation ────────────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
mod sled_impl {
|
||||
use crate::activation;
|
||||
use crate::consolidation::{self, ConsolidationConfig, ConsolidationReport};
|
||||
use crate::edge_type;
|
||||
use crate::error::{EngramError, EngramResult};
|
||||
use crate::graph;
|
||||
use crate::salience;
|
||||
use crate::storage;
|
||||
use crate::types::{ActivatedNode, Edge, EdgeTypeDef, Node, ScoredNode, now_ms};
|
||||
use crate::vector;
|
||||
use sled::Db;
|
||||
use std::path::Path;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub struct EngramDb {
|
||||
pub(crate) db: Db,
|
||||
}
|
||||
|
||||
impl Clone for EngramDb {
|
||||
/// Clone shares the same underlying sled instance (single file lock,
|
||||
/// multiple in-process handles). This is safe and avoids re-opening.
|
||||
fn clone(&self) -> Self {
|
||||
Self { db: self.db.clone() }
|
||||
}
|
||||
}
|
||||
|
||||
impl EngramDb {
|
||||
/// Open (or create) an engram database at the given path.
|
||||
///
|
||||
/// Seeds all built-in edge types on first open. Idempotent — existing
|
||||
/// definitions are not overwritten on subsequent opens.
|
||||
pub fn open(path: &Path) -> EngramResult<Self> {
|
||||
let db = sled::open(path)?;
|
||||
edge_type::seed_builtin_types(&db)?;
|
||||
Ok(Self { db })
|
||||
}
|
||||
|
||||
// ── Node operations ───────────────────────────────────────────────────
|
||||
|
||||
/// Persist a new node. Returns the node's UUID.
|
||||
///
|
||||
/// Returns `EngramError::NodeAlreadyExists` if the ID already exists.
|
||||
/// Nodes are immutable — to update, create a new node and add a
|
||||
/// `supersedes` edge from new → old.
|
||||
pub fn put_node(&self, node: Node) -> EngramResult<Uuid> {
|
||||
let id = graph::put_node(&self.db, &node)?;
|
||||
// Mark HNSW index dirty so next search rebuilds it.
|
||||
vector::mark_dirty(&self.db);
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Retrieve a node by UUID. Returns None if not found.
|
||||
pub fn get_node(&self, id: Uuid) -> EngramResult<Option<Node>> {
|
||||
graph::get_node(&self.db, id)
|
||||
}
|
||||
|
||||
// ── Edge operations ───────────────────────────────────────────────────
|
||||
|
||||
/// Persist a directed edge between two nodes.
|
||||
pub fn put_edge(&self, edge: Edge) -> EngramResult<()> {
|
||||
graph::put_edge(&self.db, &edge)
|
||||
}
|
||||
|
||||
/// All edges originating from a node.
|
||||
pub fn get_edges_from(&self, from_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
graph::edges_from(&self.db, from_id)
|
||||
}
|
||||
|
||||
/// All edges pointing to a node.
|
||||
pub fn get_edges_to(&self, to_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
graph::edges_to(&self.db, to_id)
|
||||
}
|
||||
|
||||
// ── Vector search ─────────────────────────────────────────────────────
|
||||
|
||||
/// Find the `limit` nodes whose embeddings are most similar to `embedding`.
|
||||
///
|
||||
/// Falls back to flat scan for stores with < 100 nodes.
|
||||
/// Uses the HNSW index for larger stores.
|
||||
pub fn search_embedding(
|
||||
&self,
|
||||
embedding: &[f32],
|
||||
limit: usize,
|
||||
) -> EngramResult<Vec<ScoredNode>> {
|
||||
vector::search_embedding(&self.db, embedding, limit, |id| {
|
||||
graph::get_node(&self.db, id)
|
||||
})
|
||||
}
|
||||
|
||||
/// Explicitly build (or rebuild) the HNSW index.
|
||||
///
|
||||
/// This is not normally needed — the index is built lazily on first search.
|
||||
/// Call this if you want to pre-warm the index after a large batch insert.
|
||||
///
|
||||
/// Returns the number of nodes indexed.
|
||||
pub fn build_index(&self) -> EngramResult<usize> {
|
||||
vector::build_index(&self.db)
|
||||
}
|
||||
|
||||
// ── Spreading activation ──────────────────────────────────────────────
|
||||
|
||||
/// Run spreading activation from a set of seed nodes.
|
||||
pub fn activate(
|
||||
&self,
|
||||
seeds: &[Uuid],
|
||||
query_embedding: &[f32],
|
||||
max_depth: u8,
|
||||
limit: usize,
|
||||
) -> EngramResult<Vec<ActivatedNode>> {
|
||||
activation::activate(&self.db, seeds, query_embedding, max_depth, limit)
|
||||
}
|
||||
|
||||
// ── Graph traversal ───────────────────────────────────────────────────
|
||||
|
||||
/// BFS traversal from `from`, following edges up to `max_depth` hops.
|
||||
///
|
||||
/// If `relation` is `Some("causes")`, only edges with that relation
|
||||
/// name are followed. Pass `None` to follow all edges.
|
||||
pub fn traverse(
|
||||
&self,
|
||||
from: Uuid,
|
||||
relation: Option<&str>,
|
||||
max_depth: u8,
|
||||
) -> EngramResult<Vec<Node>> {
|
||||
graph::traverse(&self.db, from, relation, max_depth)
|
||||
}
|
||||
|
||||
// ── Edge type registry (native bindings for el) ───────────────────────
|
||||
//
|
||||
// These are the el-callable surfaces. All intelligence about when to
|
||||
// create types, how to score confidence, and when to merge/split lives
|
||||
// in el. Rust stores and retrieves.
|
||||
|
||||
/// Create or update an edge type. If the type already exists its
|
||||
/// description and confidence are updated; id, first_observed,
|
||||
/// instance_count, derived_from, supersedes, and deprecated are preserved.
|
||||
pub fn native_edge_type_put(
|
||||
&self,
|
||||
name: &str,
|
||||
description: &str,
|
||||
confidence: f32,
|
||||
) -> EngramResult<()> {
|
||||
let confidence = confidence.clamp(0.0, 1.0);
|
||||
if let Some(mut existing) = edge_type::get_edge_type(&self.db, name)? {
|
||||
existing.description = description.to_string();
|
||||
existing.confidence = confidence;
|
||||
edge_type::register_edge_type(&self.db, &existing)?;
|
||||
} else {
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: name.to_string(),
|
||||
description: description.to_string(),
|
||||
first_observed: now_ms(),
|
||||
instance_count: 0,
|
||||
confidence,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
edge_type::register_edge_type(&self.db, &def)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Retrieve an edge type as a JSON string. Returns empty string if not found.
|
||||
pub fn native_edge_type_get(&self, name: &str) -> EngramResult<String> {
|
||||
match edge_type::get_edge_type(&self.db, name)? {
|
||||
Some(def) => Ok(serde_json::to_string(&def).unwrap_or_default()),
|
||||
None => Ok(String::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// List all registered edge types as a JSON array string.
|
||||
pub fn native_edge_type_list(&self) -> EngramResult<String> {
|
||||
let defs = edge_type::all_edge_types(&self.db)?;
|
||||
Ok(serde_json::to_string(&defs).unwrap_or_default())
|
||||
}
|
||||
|
||||
/// Increment the instance_count for a named edge type by one.
|
||||
pub fn native_edge_type_increment_count(&self, name: &str) -> EngramResult<()> {
|
||||
edge_type::increment_edge_type_count(&self.db, name)
|
||||
}
|
||||
|
||||
/// Update the description and confidence of an existing edge type.
|
||||
/// instance_count, id, first_observed, and other metadata are preserved.
|
||||
pub fn native_edge_type_update(
|
||||
&self,
|
||||
name: &str,
|
||||
description: &str,
|
||||
confidence: f32,
|
||||
) -> EngramResult<()> {
|
||||
if let Some(mut def) = edge_type::get_edge_type(&self.db, name)? {
|
||||
def.description = description.to_string();
|
||||
def.confidence = confidence.clamp(0.0, 1.0);
|
||||
edge_type::register_edge_type(&self.db, &def)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Salience management ───────────────────────────────────────────────
|
||||
|
||||
/// Mark a node as recently activated — update last_activated, increment
|
||||
/// activation_count, and recompute salience.
|
||||
pub fn touch(&self, id: Uuid) -> EngramResult<()> {
|
||||
let mut node =
|
||||
graph::get_node(&self.db, id)?.ok_or(EngramError::NotFound(id))?;
|
||||
node.last_activated = crate::types::now_ms();
|
||||
node.activation_count += 1;
|
||||
node.salience = salience::compute_salience(
|
||||
node.importance,
|
||||
node.last_activated,
|
||||
node.activation_count,
|
||||
);
|
||||
// Use overwrite_node — touch is an internal state update, not a new node.
|
||||
graph::overwrite_node(&self.db, &node)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply a multiplicative decay to the salience of every node in the store.
|
||||
///
|
||||
/// `factor` should be in (0.0, 1.0). Returns the number of nodes updated.
|
||||
pub fn decay(&self, factor: f32) -> EngramResult<usize> {
|
||||
if !(0.0..=1.0).contains(&factor) {
|
||||
return Err(EngramError::InvalidParam(format!(
|
||||
"decay factor must be in [0.0, 1.0], got {}",
|
||||
factor
|
||||
)));
|
||||
}
|
||||
|
||||
let nodes = storage::scan_nodes(&self.db)?;
|
||||
let mut count = 0usize;
|
||||
for mut node in nodes {
|
||||
let new_salience = salience::decay_salience(node.salience, factor);
|
||||
if new_salience != node.salience {
|
||||
node.salience = new_salience;
|
||||
storage::write_salience(&self.db, node.id, new_salience)?;
|
||||
graph::overwrite_node(&self.db, &node)?;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
// ── Consolidation ─────────────────────────────────────────────────────
|
||||
|
||||
/// Run a memory consolidation cycle.
|
||||
///
|
||||
/// Promotes Episodic nodes that have been activated enough times and are
|
||||
/// still salient enough to MemoryTier::Semantic. Then decays all saliences.
|
||||
///
|
||||
/// See `consolidation::ConsolidationConfig` for tuning knobs.
|
||||
pub fn consolidate(
|
||||
&self,
|
||||
config: &ConsolidationConfig,
|
||||
) -> EngramResult<ConsolidationReport> {
|
||||
consolidation::consolidate(&self.db, config)
|
||||
}
|
||||
|
||||
// ── Statistics ────────────────────────────────────────────────────────
|
||||
|
||||
/// Total number of nodes stored.
|
||||
pub fn node_count(&self) -> EngramResult<usize> {
|
||||
graph::node_count(&self.db)
|
||||
}
|
||||
|
||||
/// Total number of edges stored.
|
||||
pub fn edge_count(&self) -> EngramResult<usize> {
|
||||
graph::edge_count(&self.db)
|
||||
}
|
||||
|
||||
// ── Bulk scan (for sync) ───────────────────────────────────────────────
|
||||
|
||||
/// Scan all nodes in the store.
|
||||
///
|
||||
/// Used by the sync engine to generate delta snapshots.
|
||||
pub fn scan_nodes(&self) -> EngramResult<Vec<crate::types::Node>> {
|
||||
storage::scan_nodes(&self.db)
|
||||
}
|
||||
|
||||
/// Scan all edges in the store (forward index only).
|
||||
pub fn scan_edges(&self) -> EngramResult<Vec<Edge>> {
|
||||
let prefix = b"edges:from:";
|
||||
let mut edges = Vec::new();
|
||||
for result in self.db.scan_prefix(prefix) {
|
||||
let (_k, v) = result?;
|
||||
let edge: Edge = bincode::deserialize(&v)?;
|
||||
edges.push(edge);
|
||||
}
|
||||
Ok(edges)
|
||||
}
|
||||
|
||||
/// Delete a node by UUID (tombstone support for sync).
|
||||
pub fn delete_node(&self, id: Uuid) -> EngramResult<()> {
|
||||
let key = storage::node_key(id);
|
||||
self.db.remove(key)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── WASM / in-memory implementation ──────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
mod wasm_impl {
|
||||
use crate::consolidation::{self, ConsolidationConfig, ConsolidationReport};
|
||||
use crate::error::{EngramError, EngramResult};
|
||||
use crate::mem_storage::MemStore;
|
||||
use crate::salience;
|
||||
use crate::types::{ActivatedNode, Edge, Node, ScoredNode};
|
||||
use crate::vector;
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub struct EngramDb {
|
||||
pub(crate) store: RwLock<MemStore>,
|
||||
}
|
||||
|
||||
impl EngramDb {
|
||||
/// Create an in-memory engram database. The `path` argument is ignored in WASM mode.
|
||||
pub fn open(_path: &std::path::Path) -> EngramResult<Self> {
|
||||
Ok(Self {
|
||||
store: RwLock::new(MemStore::new()),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Node operations ───────────────────────────────────────────────────
|
||||
|
||||
pub fn put_node(&self, node: Node) -> EngramResult<Uuid> {
|
||||
let id = node.id;
|
||||
self.store
|
||||
.write()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.write_node(&node)?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn get_node(&self, id: Uuid) -> EngramResult<Option<Node>> {
|
||||
self.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.read_node(id)
|
||||
}
|
||||
|
||||
// ── Edge operations ───────────────────────────────────────────────────
|
||||
|
||||
pub fn put_edge(&self, edge: Edge) -> EngramResult<()> {
|
||||
self.store
|
||||
.write()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.write_edge(&edge)
|
||||
}
|
||||
|
||||
pub fn get_edges_from(&self, from_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
self.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.read_edges_from(from_id)
|
||||
}
|
||||
|
||||
pub fn get_edges_to(&self, to_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
self.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.read_edges_to(to_id)
|
||||
}
|
||||
|
||||
// ── Vector search ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn search_embedding(
|
||||
&self,
|
||||
embedding: &[f32],
|
||||
limit: usize,
|
||||
) -> EngramResult<Vec<ScoredNode>> {
|
||||
let store = self
|
||||
.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?;
|
||||
let vectors = store.scan_vectors()?;
|
||||
let nodes_snap: HashMap<Uuid, Node> = store.nodes.clone();
|
||||
drop(store);
|
||||
|
||||
vector::search_embedding_memory(embedding, limit, &vectors, |id| {
|
||||
Ok(nodes_snap.get(&id).cloned())
|
||||
})
|
||||
}
|
||||
|
||||
/// No-op in WASM mode (flat scan is always used). Returns node count.
|
||||
pub fn build_index(&self) -> EngramResult<usize> {
|
||||
self.node_count()
|
||||
}
|
||||
|
||||
// ── Spreading activation ──────────────────────────────────────────────
|
||||
|
||||
pub fn activate(
|
||||
&self,
|
||||
seeds: &[Uuid],
|
||||
query_embedding: &[f32],
|
||||
max_depth: u8,
|
||||
limit: usize,
|
||||
) -> EngramResult<Vec<ActivatedNode>> {
|
||||
use crate::activation;
|
||||
let store = self
|
||||
.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?;
|
||||
activation::activate_mem(&store, seeds, query_embedding, max_depth, limit)
|
||||
}
|
||||
|
||||
// ── Graph traversal ───────────────────────────────────────────────────
|
||||
|
||||
pub fn traverse(
|
||||
&self,
|
||||
from: Uuid,
|
||||
relation: Option<&str>,
|
||||
max_depth: u8,
|
||||
) -> EngramResult<Vec<Node>> {
|
||||
let store = self
|
||||
.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?;
|
||||
let mut visited: HashSet<Uuid> = HashSet::new();
|
||||
let mut queue: VecDeque<(Uuid, u8)> = VecDeque::new();
|
||||
let mut result: Vec<Node> = Vec::new();
|
||||
|
||||
visited.insert(from);
|
||||
queue.push_back((from, 0));
|
||||
|
||||
while let Some((current_id, depth)) = queue.pop_front() {
|
||||
if depth >= max_depth {
|
||||
continue;
|
||||
}
|
||||
let edges = store.read_edges_from(current_id)?;
|
||||
for edge in edges {
|
||||
if let Some(rel) = relation {
|
||||
if edge.relation != rel {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let next = edge.to_id;
|
||||
if visited.contains(&next) {
|
||||
continue;
|
||||
}
|
||||
visited.insert(next);
|
||||
if let Some(node) = store.read_node(next)? {
|
||||
result.push(node);
|
||||
queue.push_back((next, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
// ── Salience management ───────────────────────────────────────────────
|
||||
|
||||
pub fn touch(&self, id: Uuid) -> EngramResult<()> {
|
||||
let mut store = self
|
||||
.store
|
||||
.write()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?;
|
||||
let node = store
|
||||
.nodes
|
||||
.get_mut(&id)
|
||||
.ok_or(EngramError::NotFound(id))?;
|
||||
node.last_activated = crate::types::now_ms();
|
||||
node.activation_count += 1;
|
||||
node.salience = salience::compute_salience(
|
||||
node.importance,
|
||||
node.last_activated,
|
||||
node.activation_count,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn decay(&self, factor: f32) -> EngramResult<usize> {
|
||||
if !(0.0..=1.0).contains(&factor) {
|
||||
return Err(EngramError::InvalidParam(format!(
|
||||
"decay factor must be in [0.0, 1.0], got {}",
|
||||
factor
|
||||
)));
|
||||
}
|
||||
let mut store = self
|
||||
.store
|
||||
.write()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?;
|
||||
let mut count = 0usize;
|
||||
for node in store.nodes.values_mut() {
|
||||
let new_sal = salience::decay_salience(node.salience, factor);
|
||||
if new_sal != node.salience {
|
||||
node.salience = new_sal;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
// ── Consolidation ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn consolidate(
|
||||
&self,
|
||||
config: &ConsolidationConfig,
|
||||
) -> EngramResult<ConsolidationReport> {
|
||||
let mut store = self
|
||||
.store
|
||||
.write()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?;
|
||||
consolidation::consolidate_mem(&mut store, config)
|
||||
}
|
||||
|
||||
// ── Statistics ────────────────────────────────────────────────────────
|
||||
|
||||
pub fn node_count(&self) -> EngramResult<usize> {
|
||||
Ok(self
|
||||
.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.node_count())
|
||||
}
|
||||
|
||||
pub fn edge_count(&self) -> EngramResult<usize> {
|
||||
Ok(self
|
||||
.store
|
||||
.read()
|
||||
.map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?
|
||||
.edge_count())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Re-export the right impl ──────────────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
pub use sled_impl::EngramDb;
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
pub use wasm_impl::EngramDb;
|
||||
@@ -0,0 +1,443 @@
|
||||
/// Edge type registry — dynamic, first-class edge type management.
|
||||
///
|
||||
/// Edge types are stored under the key prefix `edge_types:<name>` in the sled
|
||||
/// database. Each value is a bincode-encoded `EdgeTypeDef`.
|
||||
///
|
||||
/// # Operations
|
||||
///
|
||||
/// - `register_edge_type` — create or update a type definition
|
||||
/// - `get_edge_type` — look up by name
|
||||
/// - `update_edge_type_description` — change the human-readable description
|
||||
/// - `increment_edge_type_count` — bump the instance counter when a new edge is created
|
||||
/// - `merge_edge_types` — retag all edges from one type to another and deprecate the source
|
||||
/// - `split_edge_type` — record a split operation; the predicate is stored as a description note
|
||||
/// - `deprecate_edge_type` — mark a type as no longer current
|
||||
/// - `all_edge_types` — return every registered type
|
||||
/// - `edge_types_by_confidence` — filter by minimum confidence score
|
||||
|
||||
use crate::error::EngramResult;
|
||||
use crate::storage;
|
||||
use crate::types::{now_ms, Edge, EdgeTypeDef};
|
||||
use sled::Db;
|
||||
use uuid::Uuid;
|
||||
|
||||
// ── Key helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
fn edge_type_key(name: &str) -> Vec<u8> {
|
||||
format!("edge_types:{}", name).into_bytes()
|
||||
}
|
||||
|
||||
const EDGE_TYPE_PREFIX: &[u8] = b"edge_types:";
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Register a new edge type, or overwrite an existing definition.
|
||||
///
|
||||
/// Returns the UUID of the stored `EdgeTypeDef`.
|
||||
pub fn register_edge_type(db: &Db, def: &EdgeTypeDef) -> EngramResult<Uuid> {
|
||||
let key = edge_type_key(&def.name);
|
||||
let val = bincode::serialize(def)?;
|
||||
db.insert(key, val)?;
|
||||
Ok(def.id)
|
||||
}
|
||||
|
||||
/// Look up an edge type by its canonical name.
|
||||
pub fn get_edge_type(db: &Db, name: &str) -> EngramResult<Option<EdgeTypeDef>> {
|
||||
match db.get(edge_type_key(name))? {
|
||||
Some(bytes) => Ok(Some(bincode::deserialize(&bytes)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the human-readable description of an existing edge type.
|
||||
///
|
||||
/// No-ops silently if the type does not exist.
|
||||
pub fn update_edge_type_description(db: &Db, name: &str, description: &str) -> EngramResult<()> {
|
||||
if let Some(mut def) = get_edge_type(db, name)? {
|
||||
def.description = description.to_string();
|
||||
let val = bincode::serialize(&def)?;
|
||||
db.insert(edge_type_key(name), val)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Increment the instance counter for an edge type.
|
||||
///
|
||||
/// Called whenever a new edge with this type is persisted. No-ops if the type
|
||||
/// is not registered (the counter stays in-registry, not in the edge itself).
|
||||
pub fn increment_edge_type_count(db: &Db, name: &str) -> EngramResult<()> {
|
||||
if let Some(mut def) = get_edge_type(db, name)? {
|
||||
def.instance_count = def.instance_count.saturating_add(1);
|
||||
let val = bincode::serialize(&def)?;
|
||||
db.insert(edge_type_key(name), val)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Merge two edge types: retag all edges from `from_name` to `into_name`,
|
||||
/// then deprecate `from_name`.
|
||||
///
|
||||
/// After this call every edge that carried `from_name` will carry `into_name`
|
||||
/// instead. The `from_name` definition is marked deprecated and its `supersedes`
|
||||
/// field records the merge destination.
|
||||
pub fn merge_edge_types(db: &Db, from_name: &str, into_name: &str) -> EngramResult<()> {
|
||||
// Collect and retag every edge carrying from_name
|
||||
let prefix = b"edges:from:";
|
||||
let mut edges_to_retag: Vec<Edge> = Vec::new();
|
||||
for result in db.scan_prefix(prefix) {
|
||||
let (_k, v) = result?;
|
||||
let edge: Edge = bincode::deserialize(&v)?;
|
||||
if edge.relation == from_name {
|
||||
edges_to_retag.push(edge);
|
||||
}
|
||||
}
|
||||
|
||||
for mut edge in edges_to_retag {
|
||||
edge.relation = into_name.to_string();
|
||||
storage::write_edge(db, &edge)?;
|
||||
}
|
||||
|
||||
// Deprecate the source type and record the merge destination
|
||||
if let Some(mut def) = get_edge_type(db, from_name)? {
|
||||
def.deprecated = true;
|
||||
def.supersedes = Some(into_name.to_string());
|
||||
let val = bincode::serialize(&def)?;
|
||||
db.insert(edge_type_key(from_name), val)?;
|
||||
}
|
||||
|
||||
// Update instance count on the destination to account for the absorbed edges
|
||||
if let Some(mut into_def) = get_edge_type(db, into_name)? {
|
||||
// Recount from graph (simple: scan all edges for into_name)
|
||||
let mut count = 0u64;
|
||||
for result in db.scan_prefix(prefix) {
|
||||
let (_k, v) = result?;
|
||||
let edge: Edge = bincode::deserialize(&v)?;
|
||||
if edge.relation == into_name {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
into_def.instance_count = count;
|
||||
let val = bincode::serialize(&into_def)?;
|
||||
db.insert(edge_type_key(into_name), val)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record a split of `name` into `new_name_a` and `new_name_b`.
|
||||
///
|
||||
/// The predicate that drives the split is stored as a descriptive note on
|
||||
/// both new types. This does NOT automatically retag edges — the caller is
|
||||
/// responsible for deciding which edges go to `new_name_a` vs `new_name_b`
|
||||
/// and calling `storage::write_edge` for each. The split records the intent;
|
||||
/// the actual retagging is domain-specific.
|
||||
///
|
||||
/// The original type is deprecated with a note referencing the two successors.
|
||||
pub fn split_edge_type(
|
||||
db: &Db,
|
||||
name: &str,
|
||||
new_name_a: &str,
|
||||
new_name_b: &str,
|
||||
predicate: &str,
|
||||
) -> EngramResult<()> {
|
||||
let now = now_ms();
|
||||
|
||||
// Deprecate the original
|
||||
if let Some(mut original) = get_edge_type(db, name)? {
|
||||
original.deprecated = true;
|
||||
original.description = format!(
|
||||
"{} [SPLIT into '{}' and '{}' via predicate: {}]",
|
||||
original.description, new_name_a, new_name_b, predicate
|
||||
);
|
||||
let val = bincode::serialize(&original)?;
|
||||
db.insert(edge_type_key(name), val)?;
|
||||
}
|
||||
|
||||
// Register new_name_a if it doesn't already exist
|
||||
if get_edge_type(db, new_name_a)?.is_none() {
|
||||
let def_a = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: new_name_a.to_string(),
|
||||
description: format!(
|
||||
"Split from '{}' — predicate: {}",
|
||||
name, predicate
|
||||
),
|
||||
first_observed: now,
|
||||
instance_count: 0,
|
||||
confidence: 0.0,
|
||||
derived_from: Some(format!("split from '{}'", name)),
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(db, &def_a)?;
|
||||
}
|
||||
|
||||
// Register new_name_b if it doesn't already exist
|
||||
if get_edge_type(db, new_name_b)?.is_none() {
|
||||
let def_b = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: new_name_b.to_string(),
|
||||
description: format!(
|
||||
"Split from '{}' — predicate: {}",
|
||||
name, predicate
|
||||
),
|
||||
first_observed: now,
|
||||
instance_count: 0,
|
||||
confidence: 0.0,
|
||||
derived_from: Some(format!("split from '{}'", name)),
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(db, &def_b)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark an edge type as deprecated. Deprecated types should not be used on
|
||||
/// new edges, but existing edges carrying this type remain valid.
|
||||
pub fn deprecate_edge_type(db: &Db, name: &str) -> EngramResult<()> {
|
||||
if let Some(mut def) = get_edge_type(db, name)? {
|
||||
def.deprecated = true;
|
||||
let val = bincode::serialize(&def)?;
|
||||
db.insert(edge_type_key(name), val)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return all registered edge type definitions.
|
||||
pub fn all_edge_types(db: &Db) -> EngramResult<Vec<EdgeTypeDef>> {
|
||||
let mut types = Vec::new();
|
||||
for result in db.scan_prefix(EDGE_TYPE_PREFIX) {
|
||||
let (_k, v) = result?;
|
||||
let def: EdgeTypeDef = bincode::deserialize(&v)?;
|
||||
types.push(def);
|
||||
}
|
||||
Ok(types)
|
||||
}
|
||||
|
||||
/// Return all edge types with a confidence score at or above `min_confidence`.
|
||||
pub fn edge_types_by_confidence(db: &Db, min_confidence: f32) -> EngramResult<Vec<EdgeTypeDef>> {
|
||||
let mut types = all_edge_types(db)?;
|
||||
types.retain(|t| t.confidence >= min_confidence);
|
||||
types.sort_by(|a, b| b.confidence.partial_cmp(&a.confidence).unwrap_or(std::cmp::Ordering::Equal));
|
||||
Ok(types)
|
||||
}
|
||||
|
||||
// ── Built-in type seed ────────────────────────────────────────────────────────
|
||||
|
||||
/// Register all built-in edge types if they have not already been registered.
|
||||
///
|
||||
/// Called once from `EngramDb::open`. Idempotent — existing definitions are
|
||||
/// not overwritten, so user customisations survive restarts.
|
||||
pub fn seed_builtin_types(db: &Db) -> EngramResult<()> {
|
||||
let now = now_ms();
|
||||
|
||||
// Original relational types — confidence 1.0 (canonical, well-established)
|
||||
let originals: &[(&str, &str)] = &[
|
||||
("supersedes", "This node replaces or obsoletes another"),
|
||||
("causes", "This node is a causal precursor to another"),
|
||||
("contains", "This node hierarchically contains another"),
|
||||
("references", "This node cites another as supporting context"),
|
||||
("contradicts", "This node is in logical tension with another"),
|
||||
("exemplifies", "This node is a concrete instance of a more abstract node"),
|
||||
("activates", "Co-activation: firing this tends to fire the other"),
|
||||
("temporally_precedes", "Temporal ordering: this node came before the other"),
|
||||
];
|
||||
|
||||
for (name, description) in originals {
|
||||
if get_edge_type(db, name)?.is_none() {
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: name.to_string(),
|
||||
description: description.to_string(),
|
||||
first_observed: now,
|
||||
instance_count: 0,
|
||||
confidence: 1.0,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(db, &def)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Personhood / relational types — confidence 0.9 (well-understood but newer)
|
||||
let personhood: &[(&str, &str)] = &[
|
||||
("grounded_in", "This value or belief is rooted in this experience"),
|
||||
("reinforced_by", "This pattern kept being confirmed by this"),
|
||||
("derives_from", "This preference or belief flows from this value"),
|
||||
("in_tension_with", "These two things pull against each other"),
|
||||
("expressed_through","This value surfaces in this voice or behavior"),
|
||||
("shaped_by", "This pattern was formed by this relationship or experience"),
|
||||
("challenged_by", "This belief was tested by this experience"),
|
||||
("resonates_with", "This memory echoes this value"),
|
||||
];
|
||||
|
||||
for (name, description) in personhood {
|
||||
if get_edge_type(db, name)?.is_none() {
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: name.to_string(),
|
||||
description: description.to_string(),
|
||||
first_observed: now,
|
||||
instance_count: 0,
|
||||
confidence: 0.9,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(db, &def)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn open_tmp() -> (Db, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = sled::open(dir.path()).unwrap();
|
||||
(db, dir)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_and_retrieve() {
|
||||
let (db, _dir) = open_tmp();
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: "causes".to_string(),
|
||||
description: "A causes B".to_string(),
|
||||
first_observed: 0,
|
||||
instance_count: 0,
|
||||
confidence: 1.0,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(&db, &def).unwrap();
|
||||
let got = get_edge_type(&db, "causes").unwrap().unwrap();
|
||||
assert_eq!(got.name, "causes");
|
||||
assert!((got.confidence - 1.0).abs() < f32::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_description() {
|
||||
let (db, _dir) = open_tmp();
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: "test_type".to_string(),
|
||||
description: "old".to_string(),
|
||||
first_observed: 0,
|
||||
instance_count: 0,
|
||||
confidence: 0.5,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(&db, &def).unwrap();
|
||||
update_edge_type_description(&db, "test_type", "new description").unwrap();
|
||||
let got = get_edge_type(&db, "test_type").unwrap().unwrap();
|
||||
assert_eq!(got.description, "new description");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn increment_count() {
|
||||
let (db, _dir) = open_tmp();
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: "references".to_string(),
|
||||
description: "refs".to_string(),
|
||||
first_observed: 0,
|
||||
instance_count: 5,
|
||||
confidence: 1.0,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(&db, &def).unwrap();
|
||||
increment_edge_type_count(&db, "references").unwrap();
|
||||
let got = get_edge_type(&db, "references").unwrap().unwrap();
|
||||
assert_eq!(got.instance_count, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deprecate() {
|
||||
let (db, _dir) = open_tmp();
|
||||
let def = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: "old_type".to_string(),
|
||||
description: "going away".to_string(),
|
||||
first_observed: 0,
|
||||
instance_count: 0,
|
||||
confidence: 0.3,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(&db, &def).unwrap();
|
||||
deprecate_edge_type(&db, "old_type").unwrap();
|
||||
let got = get_edge_type(&db, "old_type").unwrap().unwrap();
|
||||
assert!(got.deprecated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_types_and_confidence_filter() {
|
||||
let (db, _dir) = open_tmp();
|
||||
seed_builtin_types(&db).unwrap();
|
||||
|
||||
let all = all_edge_types(&db).unwrap();
|
||||
assert!(all.len() >= 16); // 8 originals + 8 personhood
|
||||
|
||||
// All original types have confidence 1.0
|
||||
let high = edge_types_by_confidence(&db, 1.0).unwrap();
|
||||
assert!(high.len() >= 8);
|
||||
for t in &high {
|
||||
assert!((t.confidence - 1.0).abs() < f32::EPSILON);
|
||||
}
|
||||
|
||||
// personhood types have confidence 0.9 — included when threshold is <= 0.9
|
||||
let wide = edge_types_by_confidence(&db, 0.9).unwrap();
|
||||
assert!(wide.len() >= 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_is_idempotent() {
|
||||
let (db, _dir) = open_tmp();
|
||||
seed_builtin_types(&db).unwrap();
|
||||
seed_builtin_types(&db).unwrap(); // second call must not panic or duplicate
|
||||
let all = all_edge_types(&db).unwrap();
|
||||
// All names should be distinct
|
||||
let mut names: Vec<String> = all.iter().map(|t| t.name.clone()).collect();
|
||||
names.sort();
|
||||
names.dedup();
|
||||
assert_eq!(names.len(), all.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_type_records_both_halves() {
|
||||
let (db, _dir) = open_tmp();
|
||||
let original = EdgeTypeDef {
|
||||
id: Uuid::new_v4(),
|
||||
name: "relates_to".to_string(),
|
||||
description: "generic relation".to_string(),
|
||||
first_observed: 0,
|
||||
instance_count: 0,
|
||||
confidence: 0.5,
|
||||
derived_from: None,
|
||||
supersedes: None,
|
||||
deprecated: false,
|
||||
};
|
||||
register_edge_type(&db, &original).unwrap();
|
||||
split_edge_type(&db, "relates_to", "causes", "references", "directionality").unwrap();
|
||||
let orig = get_edge_type(&db, "relates_to").unwrap().unwrap();
|
||||
assert!(orig.deprecated);
|
||||
assert!(get_edge_type(&db, "causes").unwrap().is_some());
|
||||
assert!(get_edge_type(&db, "references").unwrap().is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum EngramError {
|
||||
#[error("Storage error: {0}")]
|
||||
Storage(#[from] sled::Error),
|
||||
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(#[from] bincode::Error),
|
||||
|
||||
#[error("Node not found: {0}")]
|
||||
NotFound(uuid::Uuid),
|
||||
|
||||
#[error("Node already exists: {0} — nodes are immutable; supersede via edge")]
|
||||
NodeAlreadyExists(uuid::Uuid),
|
||||
|
||||
#[error("Invalid embedding: expected {expected} dimensions, got {got}")]
|
||||
DimensionMismatch { expected: usize, got: usize },
|
||||
|
||||
#[error("Invalid parameter: {0}")]
|
||||
InvalidParam(String),
|
||||
|
||||
#[error(transparent)]
|
||||
Other(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
pub type EngramResult<T> = Result<T, EngramError>;
|
||||
@@ -0,0 +1,98 @@
|
||||
/// Graph operations: store and retrieve nodes and edges, and depth-limited traversal.
|
||||
///
|
||||
/// The graph is stored in sled (a persistent embedded B-tree). Edges are indexed
|
||||
/// in both directions so that forward and backward traversals are equally cheap.
|
||||
use crate::error::EngramResult;
|
||||
use crate::storage;
|
||||
use crate::types::{Edge, Node};
|
||||
use sled::Db;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Persist a new node and its embedding. Enforces node immutability — returns
|
||||
/// `EngramError::NodeAlreadyExists` if the ID is already in the store.
|
||||
pub fn put_node(db: &Db, node: &Node) -> EngramResult<Uuid> {
|
||||
storage::write_node(db, node)?;
|
||||
Ok(node.id)
|
||||
}
|
||||
|
||||
/// Overwrite a node unconditionally. Used internally for salience/tier mutations
|
||||
/// (touch, decay, consolidation). Do NOT call this for user-visible node creation.
|
||||
pub fn overwrite_node(db: &Db, node: &Node) -> EngramResult<Uuid> {
|
||||
storage::overwrite_node(db, node)?;
|
||||
Ok(node.id)
|
||||
}
|
||||
|
||||
/// Retrieve a node by id. Returns None if not found.
|
||||
pub fn get_node(db: &Db, id: Uuid) -> EngramResult<Option<Node>> {
|
||||
storage::read_node(db, id)
|
||||
}
|
||||
|
||||
/// Persist an edge. Both forward and reverse indices are written atomically.
|
||||
pub fn put_edge(db: &Db, edge: &Edge) -> EngramResult<()> {
|
||||
storage::write_edge(db, edge)
|
||||
}
|
||||
|
||||
/// All edges originating from a given node.
|
||||
pub fn edges_from(db: &Db, from_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
storage::read_edges_from(db, from_id)
|
||||
}
|
||||
|
||||
/// All edges pointing to a given node.
|
||||
pub fn edges_to(db: &Db, to_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
storage::read_edges_to(db, to_id)
|
||||
}
|
||||
|
||||
/// Breadth-first traversal starting from `from`, following forward edges only.
|
||||
///
|
||||
/// If `relation` is `Some(&str)`, only edges whose `relation` field matches
|
||||
/// that string are followed. The BFS respects `max_depth` hops. The seed node
|
||||
/// itself is NOT included. Visited nodes are deduplicated.
|
||||
pub fn traverse(
|
||||
db: &Db,
|
||||
from: Uuid,
|
||||
relation: Option<&str>,
|
||||
max_depth: u8,
|
||||
) -> EngramResult<Vec<Node>> {
|
||||
let mut visited: HashSet<Uuid> = HashSet::new();
|
||||
let mut queue: VecDeque<(Uuid, u8)> = VecDeque::new();
|
||||
let mut result: Vec<Node> = Vec::new();
|
||||
|
||||
visited.insert(from);
|
||||
queue.push_back((from, 0));
|
||||
|
||||
while let Some((current_id, depth)) = queue.pop_front() {
|
||||
if depth >= max_depth {
|
||||
continue;
|
||||
}
|
||||
let edges = edges_from(db, current_id)?;
|
||||
for edge in edges {
|
||||
// Filter by relation type if specified
|
||||
if let Some(rel) = relation {
|
||||
if edge.relation != rel {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let next = edge.to_id;
|
||||
if visited.contains(&next) {
|
||||
continue;
|
||||
}
|
||||
visited.insert(next);
|
||||
if let Some(node) = get_node(db, next)? {
|
||||
result.push(node);
|
||||
queue.push_back((next, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Count all nodes in the store.
|
||||
pub fn node_count(db: &Db) -> EngramResult<usize> {
|
||||
storage::count_prefix(db, b"nodes:")
|
||||
}
|
||||
|
||||
/// Count all edges in the store (forward index only — each edge counted once).
|
||||
pub fn edge_count(db: &Db) -> EngramResult<usize> {
|
||||
storage::count_prefix(db, b"edges:from:")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/// Engram — a local-first memory substrate for accumulating intelligence.
|
||||
///
|
||||
/// An engram is the physical trace of a memory in the brain — the actual encoded
|
||||
/// substrate. This crate provides the storage and retrieval primitives that model
|
||||
/// how biological memory works: not as query-and-retrieve, but as
|
||||
/// activation-and-propagation.
|
||||
///
|
||||
/// # Quick Start
|
||||
///
|
||||
/// ```rust,no_run
|
||||
/// use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, EDGE_SUPERSEDES};
|
||||
/// use std::path::Path;
|
||||
///
|
||||
/// let db = EngramDb::open(Path::new("/tmp/my-engram")).unwrap();
|
||||
///
|
||||
/// let node = Node::new(
|
||||
/// NodeType::Memory,
|
||||
/// vec![0.9, 0.1, 0.3, 0.7],
|
||||
/// b"The spreading activation model of memory".to_vec(),
|
||||
/// MemoryTier::Semantic,
|
||||
/// 0.9,
|
||||
/// );
|
||||
/// let id = db.put_node(node).unwrap();
|
||||
///
|
||||
/// // Retrieve by spreading activation from a seed
|
||||
/// let results = db.activate(&[id], &[0.8, 0.2, 0.3, 0.6], 3, 10).unwrap();
|
||||
/// for r in results {
|
||||
/// println!("{:.4} hops={} {:?}", r.activation_strength, r.hops,
|
||||
/// String::from_utf8_lossy(&r.node.content));
|
||||
/// }
|
||||
/// ```
|
||||
pub mod activation;
|
||||
pub mod consolidation;
|
||||
pub mod db;
|
||||
#[cfg(not(feature = "wasm"))]
|
||||
pub mod edge_type;
|
||||
pub mod error;
|
||||
pub mod graph;
|
||||
pub mod salience;
|
||||
#[cfg(not(feature = "wasm"))]
|
||||
pub mod storage;
|
||||
#[cfg(feature = "wasm")]
|
||||
pub mod mem_storage;
|
||||
pub mod types;
|
||||
pub mod vector;
|
||||
#[cfg(feature = "migration")]
|
||||
pub mod migration;
|
||||
|
||||
// Re-export the public surface
|
||||
pub use db::EngramDb;
|
||||
pub use error::{EngramError, EngramResult};
|
||||
pub use types::{
|
||||
ActivatedNode, Edge, EdgeTypeDef, MemoryTier, Node, NodeType, ScoredNode, now_ms,
|
||||
EDGE_ACTIVATES, EDGE_CAUSES, EDGE_CONTAINS, EDGE_CONTRADICTS, EDGE_EXEMPLIFIES,
|
||||
EDGE_REFERENCES, EDGE_SUPERSEDES, EDGE_TEMPORALLY_PRECEDES,
|
||||
};
|
||||
pub use consolidation::{ConsolidationConfig, ConsolidationReport};
|
||||
@@ -0,0 +1,99 @@
|
||||
/// In-memory storage backend for environments without a filesystem.
|
||||
///
|
||||
/// Used when the `wasm` feature is enabled (e.g. browser via wasm-bindgen).
|
||||
/// Implements the same logical interface as the sled-backed `storage` module
|
||||
/// so that `EngramDb` can work identically in both environments.
|
||||
///
|
||||
/// All state lives in a `MemStore` that is held by `EngramDb` under a `RwLock`
|
||||
/// so concurrent reads are fine and writes are serialised.
|
||||
use crate::error::{EngramError, EngramResult};
|
||||
use crate::types::{Edge, Node};
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MemStore {
|
||||
pub nodes: HashMap<Uuid, Node>,
|
||||
/// from_id → list of edges
|
||||
pub edges_from: HashMap<Uuid, Vec<Edge>>,
|
||||
/// to_id → list of edges
|
||||
pub edges_to: HashMap<Uuid, Vec<Edge>>,
|
||||
}
|
||||
|
||||
impl MemStore {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
// ── Node operations ───────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_node(&mut self, node: &Node) -> EngramResult<()> {
|
||||
self.nodes.insert(node.id, node.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_node(&self, id: Uuid) -> EngramResult<Option<Node>> {
|
||||
Ok(self.nodes.get(&id).cloned())
|
||||
}
|
||||
|
||||
pub fn scan_nodes(&self) -> EngramResult<Vec<Node>> {
|
||||
Ok(self.nodes.values().cloned().collect())
|
||||
}
|
||||
|
||||
pub fn node_count(&self) -> usize {
|
||||
self.nodes.len()
|
||||
}
|
||||
|
||||
// ── Edge operations ───────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_edge(&mut self, edge: &Edge) -> EngramResult<()> {
|
||||
self.edges_from
|
||||
.entry(edge.from_id)
|
||||
.or_default()
|
||||
.push(edge.clone());
|
||||
self.edges_to
|
||||
.entry(edge.to_id)
|
||||
.or_default()
|
||||
.push(edge.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_edges_from(&self, from_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
Ok(self
|
||||
.edges_from
|
||||
.get(&from_id)
|
||||
.cloned()
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
pub fn read_edges_to(&self, to_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
Ok(self
|
||||
.edges_to
|
||||
.get(&to_id)
|
||||
.cloned()
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
pub fn edge_count(&self) -> usize {
|
||||
self.edges_from.values().map(|v| v.len()).sum()
|
||||
}
|
||||
|
||||
// ── Vector operations ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn scan_vectors(&self) -> EngramResult<Vec<(Uuid, Vec<f32>)>> {
|
||||
Ok(self
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|(id, n)| (*id, n.embedding.clone()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
// ── Salience ──────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_salience(&mut self, id: Uuid, salience: f32) -> EngramResult<()> {
|
||||
if let Some(node) = self.nodes.get_mut(&id) {
|
||||
node.salience = salience;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
/// Migration connector — imports Neuron's SQLite database into Engram.
|
||||
///
|
||||
/// Neuron stores memories, knowledge, and graph nodes in a SQLite database.
|
||||
/// This module reads that database and converts records to Engram nodes and edges.
|
||||
///
|
||||
/// # Schema mapping
|
||||
///
|
||||
/// | Neuron table | Engram node |
|
||||
/// |----------------------|----------------------------------------------|
|
||||
/// | `memory_nodes` | `Node { tier: Episodic, node_type: Memory }` |
|
||||
/// | `knowledge_entries` | `Node { tier: Semantic, node_type: Concept }` |
|
||||
///
|
||||
/// Edges from `graph_edges` are converted using their `edge_type` string directly
|
||||
/// (normalised to lowercase). Unknown types map to `"references"`.
|
||||
///
|
||||
/// # Embeddings
|
||||
///
|
||||
/// Neuron does not currently expose embeddings through the SQLite schema.
|
||||
/// Random unit vectors are generated as placeholders. Replace the call to
|
||||
/// `placeholder_embedding` with your embedding model once the ONNX engine is wired in.
|
||||
///
|
||||
/// TODO: wire in real embeddings from all-MiniLM-L6-v2 via the ONNX runtime.
|
||||
|
||||
use crate::error::{EngramError, EngramResult};
|
||||
use crate::types::{Edge, MemoryTier, Node, NodeType};
|
||||
use rusqlite::{Connection, OpenFlags};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use uuid::Uuid;
|
||||
|
||||
// ── Config and report ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Configuration for a Neuron → Engram migration.
|
||||
pub struct MigrationConfig {
|
||||
/// Path to `~/.neuron/neuron.db` (or any other Neuron SQLite file).
|
||||
pub sqlite_path: PathBuf,
|
||||
/// Path where the new Engram sled store will be created.
|
||||
pub engram_path: PathBuf,
|
||||
/// Dimensionality of placeholder embeddings.
|
||||
/// Default: 384 (matches all-MiniLM-L6-v2).
|
||||
pub embedding_dim: usize,
|
||||
}
|
||||
|
||||
impl MigrationConfig {
|
||||
pub fn new(sqlite_path: PathBuf, engram_path: PathBuf) -> Self {
|
||||
Self {
|
||||
sqlite_path,
|
||||
engram_path,
|
||||
embedding_dim: 384,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Summary of what was imported during migration.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MigrationReport {
|
||||
/// Rows imported from `memory_nodes`.
|
||||
pub memories_migrated: usize,
|
||||
/// Rows imported from `knowledge_entries`.
|
||||
pub knowledge_migrated: usize,
|
||||
/// Edges created from `graph_edges`.
|
||||
pub edges_created: usize,
|
||||
/// Non-fatal errors collected during the run.
|
||||
pub errors: Vec<String>,
|
||||
}
|
||||
|
||||
// ── Main entry point ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Read the Neuron SQLite database at `config.sqlite_path` and import all
|
||||
/// records into a new Engram sled store at `config.engram_path`.
|
||||
///
|
||||
/// Returns a `MigrationReport` describing what was imported.
|
||||
///
|
||||
/// Non-fatal errors (e.g. a single unreadable row) are collected in
|
||||
/// `report.errors` rather than aborting the entire migration.
|
||||
pub fn migrate_from_neuron(config: &MigrationConfig) -> EngramResult<MigrationReport> {
|
||||
let conn = Connection::open_with_flags(
|
||||
&config.sqlite_path,
|
||||
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
|
||||
)
|
||||
.map_err(|e| EngramError::InvalidParam(format!("Cannot open SQLite: {e}")))?;
|
||||
|
||||
let engram_db = crate::db::EngramDb::open(&config.engram_path)?;
|
||||
|
||||
let mut report = MigrationReport::default();
|
||||
|
||||
// Maps Neuron string IDs to the Engram UUIDs we assigned.
|
||||
let mut id_map: HashMap<String, Uuid> = HashMap::new();
|
||||
|
||||
// ── Import memory_nodes ───────────────────────────────────────────────────
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT id, content, importance, superseded_by, created_at \
|
||||
FROM memory_nodes ORDER BY created_at ASC",
|
||||
)
|
||||
.map_err(|e| EngramError::InvalidParam(format!("prepare memory_nodes: {e}")))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?, // id
|
||||
row.get::<_, String>(1)?, // content
|
||||
row.get::<_, String>(2)?, // importance
|
||||
row.get::<_, Option<String>>(3)?, // superseded_by
|
||||
row.get::<_, i64>(4)?, // created_at
|
||||
))
|
||||
})
|
||||
.map_err(|e| EngramError::InvalidParam(format!("query memory_nodes: {e}")))?;
|
||||
|
||||
for row_result in rows {
|
||||
match row_result {
|
||||
Ok((neuron_id, content, importance_str, _superseded_by, _created_at)) => {
|
||||
let importance = importance_string_to_f32(&importance_str);
|
||||
let embedding = placeholder_embedding(config.embedding_dim);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Memory,
|
||||
embedding,
|
||||
content.into_bytes(),
|
||||
MemoryTier::Episodic,
|
||||
importance,
|
||||
);
|
||||
|
||||
match engram_db.put_node(node.clone()) {
|
||||
Ok(uuid) => {
|
||||
id_map.insert(neuron_id, uuid);
|
||||
report.memories_migrated += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push(format!("put_node memory {neuron_id}: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push(format!("read memory row: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Import knowledge_entries ──────────────────────────────────────────────
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT id, title, content, tier, created_at \
|
||||
FROM knowledge_entries ORDER BY created_at ASC",
|
||||
)
|
||||
.map_err(|e| EngramError::InvalidParam(format!("prepare knowledge_entries: {e}")))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?, // id
|
||||
row.get::<_, String>(1)?, // title
|
||||
row.get::<_, String>(2)?, // content
|
||||
row.get::<_, String>(3)?, // tier
|
||||
row.get::<_, i64>(4)?, // created_at
|
||||
))
|
||||
})
|
||||
.map_err(|e| EngramError::InvalidParam(format!("query knowledge_entries: {e}")))?;
|
||||
|
||||
for row_result in rows {
|
||||
match row_result {
|
||||
Ok((neuron_id, title, body, _tier_str, _created_at)) => {
|
||||
// Combine title + content as the engram node content.
|
||||
let combined = format!("{title}\n\n{body}");
|
||||
let embedding = placeholder_embedding(config.embedding_dim);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Concept,
|
||||
embedding,
|
||||
combined.into_bytes(),
|
||||
MemoryTier::Semantic,
|
||||
0.75, // knowledge is moderately important by default
|
||||
);
|
||||
|
||||
match engram_db.put_node(node.clone()) {
|
||||
Ok(uuid) => {
|
||||
id_map.insert(neuron_id, uuid);
|
||||
report.knowledge_migrated += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push(format!(
|
||||
"put_node knowledge {neuron_id}: {e}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push(format!("read knowledge row: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Import graph_edges ────────────────────────────────────────────────────
|
||||
{
|
||||
// Only import edges where both endpoints ended up in our id_map.
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT from_id, to_id, edge_type, weight FROM graph_edges",
|
||||
)
|
||||
.map_err(|e| EngramError::InvalidParam(format!("prepare graph_edges: {e}")))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?, // from_id
|
||||
row.get::<_, String>(1)?, // to_id
|
||||
row.get::<_, String>(2)?, // edge_type
|
||||
row.get::<_, f64>(3)?, // weight
|
||||
))
|
||||
})
|
||||
.map_err(|e| EngramError::InvalidParam(format!("query graph_edges: {e}")))?;
|
||||
|
||||
for row_result in rows {
|
||||
match row_result {
|
||||
Ok((from_str, to_str, edge_type, weight)) => {
|
||||
let from_uuid = match id_map.get(&from_str) {
|
||||
Some(u) => *u,
|
||||
None => continue, // endpoint not migrated, skip
|
||||
};
|
||||
let to_uuid = match id_map.get(&to_str) {
|
||||
Some(u) => *u,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let relation = normalise_edge_type(&edge_type);
|
||||
let edge = Edge::new(from_uuid, to_uuid, relation, weight as f32);
|
||||
|
||||
match engram_db.put_edge(edge) {
|
||||
Ok(()) => report.edges_created += 1,
|
||||
Err(e) => {
|
||||
report.errors.push(format!(
|
||||
"put_edge {from_str} → {to_str}: {e}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push(format!("read edge row: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Convert Neuron's text importance level to a float score.
|
||||
fn importance_string_to_f32(importance: &str) -> f32 {
|
||||
match importance.to_lowercase().as_str() {
|
||||
"critical" => 1.0,
|
||||
"high" => 0.85,
|
||||
"normal" | "medium" => 0.5,
|
||||
"low" => 0.25,
|
||||
_ => {
|
||||
// Try parsing directly as a float.
|
||||
importance.parse::<f32>().unwrap_or(0.5).clamp(0.0, 1.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalise a Neuron edge type string to a canonical Engram edge type name.
|
||||
///
|
||||
/// Known variants (including old PascalCase forms from the Rust enum era) are
|
||||
/// mapped to their lowercase canonical names. Unknown types fall back to
|
||||
/// `"references"` as a safe, non-destructive default.
|
||||
fn normalise_edge_type(edge_type: &str) -> &'static str {
|
||||
match edge_type.to_lowercase().as_str() {
|
||||
"supersedes" | "superseded_by" => "supersedes",
|
||||
"causes" | "caused_by" => "causes",
|
||||
"contains" | "contained_by" => "contains",
|
||||
"references" | "referenced_by" => "references",
|
||||
"contradicts" => "contradicts",
|
||||
"exemplifies" | "exemplified_by" => "exemplifies",
|
||||
"activates" => "activates",
|
||||
"temporally_precedes" | "temporallyprecedes" | "follows" => "temporally_precedes",
|
||||
_ => "references", // safe default
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a pseudo-random unit vector of the given dimension as a placeholder embedding.
|
||||
///
|
||||
/// Uses a simple xorshift64 PRNG seeded from the current time. The result is
|
||||
/// semantically meaningless — it only satisfies the schema requirement that
|
||||
/// every node has an embedding vector.
|
||||
///
|
||||
/// TODO: replace with actual embeddings from all-MiniLM-L6-v2 via ONNX runtime
|
||||
/// once the embedding engine is wired in.
|
||||
pub fn placeholder_embedding(dim: usize) -> Vec<f32> {
|
||||
// Seed from subsecond wall time for reasonable entropy across calls.
|
||||
let mut state: u64 = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.subsec_nanos() as u64
|
||||
| 1; // ensure non-zero
|
||||
|
||||
// xorshift64 — no overflow risk, passes statistical tests well enough for placeholders.
|
||||
let mut xorshift = || -> f32 {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
// Map to [-1, 1]
|
||||
(state as f32 / u64::MAX as f32) * 2.0 - 1.0
|
||||
};
|
||||
|
||||
let mut raw: Vec<f32> = (0..dim).map(|_| xorshift()).collect();
|
||||
|
||||
// Normalise to unit length.
|
||||
let norm: f32 = raw.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if norm > 0.0 {
|
||||
for x in &mut raw {
|
||||
*x /= norm;
|
||||
}
|
||||
}
|
||||
raw
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn importance_string_critical() {
|
||||
assert!((importance_string_to_f32("critical") - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn importance_string_normal() {
|
||||
assert!((importance_string_to_f32("normal") - 0.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn importance_string_unknown_defaults_to_half() {
|
||||
assert!((importance_string_to_f32("???") - 0.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edge_type_supersedes() {
|
||||
assert_eq!(normalise_edge_type("supersedes"), "supersedes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edge_type_unknown_is_references() {
|
||||
assert_eq!(normalise_edge_type("foobar"), "references");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placeholder_embedding_correct_length() {
|
||||
let emb = placeholder_embedding(384);
|
||||
assert_eq!(emb.len(), 384);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placeholder_embedding_is_unit_vector() {
|
||||
let emb = placeholder_embedding(128);
|
||||
let norm: f32 = emb.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
assert!((norm - 1.0).abs() < 1e-4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrate_from_in_memory_db() {
|
||||
// Build a minimal SQLite DB in a temp dir and migrate it.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db_path = dir.path().join("test.db");
|
||||
let engram_path = dir.path().join("engram");
|
||||
|
||||
// Create minimal Neuron-like schema and insert a couple of rows.
|
||||
let conn = Connection::open(&db_path).unwrap();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE memory_nodes (
|
||||
id TEXT PRIMARY KEY, content TEXT NOT NULL, importance TEXT NOT NULL DEFAULT 'normal',
|
||||
superseded_by TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE knowledge_entries (
|
||||
id TEXT PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL,
|
||||
category TEXT NOT NULL DEFAULT '', tier TEXT NOT NULL DEFAULT 'note',
|
||||
tags TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE graph_edges (
|
||||
from_id TEXT NOT NULL, from_type TEXT NOT NULL,
|
||||
to_id TEXT NOT NULL, to_type TEXT NOT NULL,
|
||||
edge_type TEXT NOT NULL, weight REAL NOT NULL DEFAULT 1.0,
|
||||
PRIMARY KEY (from_id, to_id, edge_type)
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO memory_nodes (id, content, importance, created_at, updated_at)
|
||||
VALUES ('mem-1', 'First memory', 'high', 1000, 1000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO knowledge_entries (id, title, content, created_at, updated_at)
|
||||
VALUES ('kn-1', 'Some concept', 'Body text.', 2000, 2000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
drop(conn); // close before migrating
|
||||
|
||||
let config = MigrationConfig {
|
||||
sqlite_path: db_path,
|
||||
engram_path,
|
||||
embedding_dim: 16,
|
||||
};
|
||||
|
||||
let report = migrate_from_neuron(&config).unwrap();
|
||||
assert_eq!(report.memories_migrated, 1);
|
||||
assert_eq!(report.knowledge_migrated, 1);
|
||||
assert_eq!(report.edges_created, 0); // no edges in the test DB
|
||||
assert!(report.errors.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/// Salience is the brain's answer to the question: "Is this worth remembering right now?"
|
||||
///
|
||||
/// It combines three signals:
|
||||
/// - **Importance**: explicit weight assigned at creation (how significant is this?)
|
||||
/// - **Recency**: exponential decay since last activation (recent = more relevant)
|
||||
/// - **Frequency**: log-compressed activation count (things recalled often stay accessible)
|
||||
///
|
||||
/// The formula is intentionally simple. It models forgetting as *adaptive*, not as failure.
|
||||
/// Things that aren't activated decay toward zero — not because they are lost, but because
|
||||
/// they are no longer relevant to current cognition. This is how biological memory works.
|
||||
///
|
||||
/// ```text
|
||||
/// salience = importance × (1 / (1 + days_since_activation)) × ln(activation_count + 1)
|
||||
/// ```
|
||||
///
|
||||
/// Note: a node activated for the first time has activation_count=0, so the log term
|
||||
/// evaluates to ln(1) = 0. We add 1 to the ln argument to give first activations a
|
||||
/// baseline salience equal to importance × recency.
|
||||
use crate::types::now_ms;
|
||||
|
||||
/// Compute the current salience of a node.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `importance` - explicit importance score, 0.0–1.0
|
||||
/// * `last_activated_ms` - Unix milliseconds of last activation
|
||||
/// * `activation_count` - how many times the node has been activated
|
||||
///
|
||||
/// # Returns
|
||||
/// Salience score, unbounded above but typically 0.0–5.0 for well-used nodes.
|
||||
pub fn compute_salience(importance: f32, last_activated_ms: i64, activation_count: u64) -> f32 {
|
||||
let days_since = (now_ms() - last_activated_ms) as f32 / 86_400_000.0;
|
||||
// Recency factor: 1.0 at activation, approaching 0 asymptotically.
|
||||
// At 1 day: 0.5. At 6 days: ~0.14. At 30 days: ~0.03.
|
||||
let recency = 1.0 / (1.0 + days_since);
|
||||
// Frequency factor: log-compressed so that going from 0→1 activations matters
|
||||
// more than going from 100→101. This mirrors the diminishing returns of rehearsal.
|
||||
let frequency = (activation_count as f32 + 1.0).ln();
|
||||
importance * recency * frequency
|
||||
}
|
||||
|
||||
/// Apply a multiplicative decay to a salience score.
|
||||
///
|
||||
/// Called periodically to age stored salience values without recomputing from scratch.
|
||||
/// A factor of 0.95 means 5% forgetting per decay cycle.
|
||||
pub fn decay_salience(current: f32, factor: f32) -> f32 {
|
||||
(current * factor).max(0.0)
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
/// Low-level sled key/value operations for nodes, edges, vectors, and salience.
|
||||
///
|
||||
/// Key schema:
|
||||
/// nodes:{uuid} → bincode-encoded Node
|
||||
/// edges:from:{from}:{to} → bincode-encoded Edge
|
||||
/// edges:to:{to}:{from} → reverse index (same Edge bytes)
|
||||
/// vectors:{uuid} → raw little-endian f32 bytes
|
||||
/// salience:{uuid} → 4-byte little-endian f32
|
||||
/// edge_types:{name} → bincode-encoded EdgeTypeDef (managed by edge_type.rs)
|
||||
use crate::error::{EngramError, EngramResult};
|
||||
use crate::types::{Edge, Node};
|
||||
use sled::Db;
|
||||
use uuid::Uuid;
|
||||
|
||||
// ── Key constructors ──────────────────────────────────────────────────────────
|
||||
|
||||
pub fn node_key(id: Uuid) -> Vec<u8> {
|
||||
format!("nodes:{}", id).into_bytes()
|
||||
}
|
||||
|
||||
pub fn edge_from_key(from: Uuid, to: Uuid) -> Vec<u8> {
|
||||
format!("edges:from:{}:{}", from, to).into_bytes()
|
||||
}
|
||||
|
||||
pub fn edge_to_key(to: Uuid, from: Uuid) -> Vec<u8> {
|
||||
format!("edges:to:{}:{}", to, from).into_bytes()
|
||||
}
|
||||
|
||||
pub fn vector_key(id: Uuid) -> Vec<u8> {
|
||||
format!("vectors:{}", id).into_bytes()
|
||||
}
|
||||
|
||||
pub fn salience_key(id: Uuid) -> Vec<u8> {
|
||||
format!("salience:{}", id).into_bytes()
|
||||
}
|
||||
|
||||
// ── Node storage ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Persist a node. Returns `EngramError::NodeAlreadyExists` if a node with
|
||||
/// this ID already exists in the store.
|
||||
///
|
||||
/// Nodes are immutable and append-only. To update, create a new node and
|
||||
/// connect it to the old with a `supersedes` edge.
|
||||
pub fn write_node(db: &Db, node: &Node) -> EngramResult<()> {
|
||||
let key = node_key(node.id);
|
||||
|
||||
// Immutability guard — reject writes to existing node IDs.
|
||||
if db.contains_key(&key)? {
|
||||
return Err(EngramError::NodeAlreadyExists(node.id));
|
||||
}
|
||||
|
||||
let val = bincode::serialize(node)?;
|
||||
db.insert(key, val)?;
|
||||
|
||||
// Store the embedding separately for fast vector scan
|
||||
let vkey = vector_key(node.id);
|
||||
let vbytes = floats_to_bytes(&node.embedding);
|
||||
db.insert(vkey, vbytes)?;
|
||||
|
||||
// Store salience separately so the decay pass can update it cheaply
|
||||
let skey = salience_key(node.id);
|
||||
db.insert(skey, f32_to_bytes(node.salience))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Overwrite a node unconditionally. Used internally for salience/tier updates
|
||||
/// that must mutate in-place (touch, decay, consolidation).
|
||||
///
|
||||
/// Do NOT expose this in public API — callers should use `write_node` which
|
||||
/// enforces immutability.
|
||||
pub(crate) fn overwrite_node(db: &Db, node: &Node) -> EngramResult<()> {
|
||||
let key = node_key(node.id);
|
||||
let val = bincode::serialize(node)?;
|
||||
db.insert(key, val)?;
|
||||
let vkey = vector_key(node.id);
|
||||
db.insert(vkey, floats_to_bytes(&node.embedding))?;
|
||||
let skey = salience_key(node.id);
|
||||
db.insert(skey, f32_to_bytes(node.salience))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_node(db: &Db, id: Uuid) -> EngramResult<Option<Node>> {
|
||||
match db.get(node_key(id))? {
|
||||
Some(bytes) => Ok(Some(bincode::deserialize(&bytes)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterate over every node in the store.
|
||||
pub fn scan_nodes(db: &Db) -> EngramResult<Vec<Node>> {
|
||||
let prefix = b"nodes:";
|
||||
let mut nodes = Vec::new();
|
||||
for result in db.scan_prefix(prefix) {
|
||||
let (_k, v) = result?;
|
||||
let node: Node = bincode::deserialize(&v)?;
|
||||
nodes.push(node);
|
||||
}
|
||||
Ok(nodes)
|
||||
}
|
||||
|
||||
// ── Edge storage ─────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_edge(db: &Db, edge: &Edge) -> EngramResult<()> {
|
||||
let bytes = bincode::serialize(edge)?;
|
||||
// Forward index: from → to
|
||||
db.insert(edge_from_key(edge.from_id, edge.to_id), bytes.clone())?;
|
||||
// Reverse index: to → from
|
||||
db.insert(edge_to_key(edge.to_id, edge.from_id), bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_edges_from(db: &Db, from_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
let prefix = format!("edges:from:{}:", from_id).into_bytes();
|
||||
read_edges_with_prefix(db, &prefix)
|
||||
}
|
||||
|
||||
pub fn read_edges_to(db: &Db, to_id: Uuid) -> EngramResult<Vec<Edge>> {
|
||||
let prefix = format!("edges:to:{}:", to_id).into_bytes();
|
||||
read_edges_with_prefix(db, &prefix)
|
||||
}
|
||||
|
||||
fn read_edges_with_prefix(db: &Db, prefix: &[u8]) -> EngramResult<Vec<Edge>> {
|
||||
let mut edges = Vec::new();
|
||||
for result in db.scan_prefix(prefix) {
|
||||
let (_k, v) = result?;
|
||||
let edge: Edge = bincode::deserialize(&v)?;
|
||||
edges.push(edge);
|
||||
}
|
||||
Ok(edges)
|
||||
}
|
||||
|
||||
// ── Vector scan ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// Read all stored (uuid, embedding) pairs. Used for flat cosine search.
|
||||
pub fn scan_vectors(db: &Db) -> EngramResult<Vec<(Uuid, Vec<f32>)>> {
|
||||
let prefix = b"vectors:";
|
||||
let mut out = Vec::new();
|
||||
for result in db.scan_prefix(prefix) {
|
||||
let (k, v) = result?;
|
||||
// key = "vectors:{uuid}" — slice off the prefix
|
||||
let id_str = std::str::from_utf8(&k[prefix.len()..])
|
||||
.map_err(|e| EngramError::InvalidParam(e.to_string()))?;
|
||||
let id = id_str
|
||||
.parse::<Uuid>()
|
||||
.map_err(|e| EngramError::InvalidParam(e.to_string()))?;
|
||||
let floats = bytes_to_floats(&v);
|
||||
out.push((id, floats));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ── Salience update ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Overwrite the salience entry for a node without rewriting the full node blob.
|
||||
pub fn write_salience(db: &Db, id: Uuid, salience: f32) -> EngramResult<()> {
|
||||
db.insert(salience_key(id), f32_to_bytes(salience))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_salience(db: &Db, id: Uuid) -> EngramResult<Option<f32>> {
|
||||
match db.get(salience_key(id))? {
|
||||
Some(b) => Ok(Some(bytes_to_f32(&b))),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Count entries matching a key prefix.
|
||||
pub fn count_prefix(db: &Db, prefix: &[u8]) -> EngramResult<usize> {
|
||||
let mut n = 0usize;
|
||||
for result in db.scan_prefix(prefix) {
|
||||
result?;
|
||||
n += 1;
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
// ── Byte encoding helpers ─────────────────────────────────────────────────────
|
||||
|
||||
fn f32_to_bytes(v: f32) -> Vec<u8> {
|
||||
v.to_le_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn bytes_to_f32(b: &[u8]) -> f32 {
|
||||
let arr: [u8; 4] = b[..4].try_into().unwrap_or([0u8; 4]);
|
||||
f32::from_le_bytes(arr)
|
||||
}
|
||||
|
||||
fn floats_to_bytes(floats: &[f32]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(floats.len() * 4);
|
||||
for f in floats {
|
||||
out.extend_from_slice(&f.to_le_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn bytes_to_floats(bytes: &[u8]) -> Vec<f32> {
|
||||
bytes
|
||||
.chunks_exact(4)
|
||||
.map(|c| f32::from_le_bytes(c.try_into().unwrap()))
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
// ── Built-in edge type name constants ─────────────────────────────────────────
|
||||
//
|
||||
// String constants for the 8 built-in relation types. Use these on Edge.relation
|
||||
// to reference known types. Intelligence about when to create new types or how
|
||||
// to score confidence belongs in el, not here.
|
||||
pub const EDGE_SUPERSEDES: &str = "supersedes";
|
||||
pub const EDGE_CAUSES: &str = "causes";
|
||||
pub const EDGE_CONTAINS: &str = "contains";
|
||||
pub const EDGE_REFERENCES: &str = "references";
|
||||
pub const EDGE_CONTRADICTS: &str = "contradicts";
|
||||
pub const EDGE_EXEMPLIFIES: &str = "exemplifies";
|
||||
pub const EDGE_ACTIVATES: &str = "activates";
|
||||
pub const EDGE_TEMPORALLY_PRECEDES: &str = "temporally_precedes";
|
||||
|
||||
/// The functional role of a node in the memory graph.
|
||||
///
|
||||
/// Different node types participate in different retrieval patterns:
|
||||
/// - Memories and Events are time-anchored
|
||||
/// - Concepts and Entities form the semantic backbone
|
||||
/// - Processes encode procedural knowledge
|
||||
/// - InternalState captures the system's own affective context
|
||||
/// - Custom(String) is an open extension point; el defines new types freely
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum NodeType {
|
||||
/// A specific remembered experience or observation
|
||||
Memory,
|
||||
/// An abstract idea, category, or semantic anchor
|
||||
Concept,
|
||||
/// A time-stamped occurrence in the world or in processing
|
||||
Event,
|
||||
/// A named thing — person, place, object, system
|
||||
Entity,
|
||||
/// A procedural pattern, workflow, or sequence of steps
|
||||
Process,
|
||||
/// An internal affective or motivational state
|
||||
InternalState,
|
||||
/// Caller-defined node type. el uses this for types Rust does not need to know about.
|
||||
Custom(String),
|
||||
}
|
||||
|
||||
/// Where in the memory hierarchy a node currently lives.
|
||||
///
|
||||
/// Tiers model the brain's own stratified memory architecture.
|
||||
/// Nodes migrate between tiers based on salience decay and reinforcement.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum MemoryTier {
|
||||
/// Hot working memory — the K most recently activated nodes.
|
||||
/// Ultra-fast access. Evicted by recency when K is exceeded.
|
||||
Working,
|
||||
/// Episodic memory — time-ordered events and experiences.
|
||||
/// Indexed chronologically; supports temporal traversal.
|
||||
Episodic,
|
||||
/// Semantic memory — the concept graph with weighted associations.
|
||||
/// This is the long-term structural knowledge of the system.
|
||||
Semantic,
|
||||
/// Procedural memory — patterns, workflows, habits.
|
||||
/// Retrieved by similarity to current task context.
|
||||
Procedural,
|
||||
}
|
||||
|
||||
/// Metadata record for a named edge type. Dumb data container — Rust stores and
|
||||
/// returns it. All decisions about confidence thresholds, when to create new
|
||||
/// types, merge/split logic, and pattern recognition belong in el, not here.
|
||||
///
|
||||
/// Fields like `deprecated`, `derived_from`, and `supersedes` are data.
|
||||
/// They are set by the caller (el) and stored verbatim. Rust never inspects
|
||||
/// or acts on them autonomously.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EdgeTypeDef {
|
||||
/// Stable unique identifier for this edge type record
|
||||
pub id: Uuid,
|
||||
/// The canonical name used on edges (e.g. `"causes"`, `"resonates_with"`)
|
||||
pub name: String,
|
||||
/// Human-readable description of what this relation means
|
||||
pub description: String,
|
||||
/// Unix milliseconds when this type was first registered
|
||||
pub first_observed: i64,
|
||||
/// How many edges currently carry this type
|
||||
pub instance_count: u64,
|
||||
/// Caller-supplied confidence, 0.0–1.0. Stored as-is; not computed here.
|
||||
pub confidence: f32,
|
||||
/// Free-text note about what observation prompted this type's creation.
|
||||
/// Set by el; stored verbatim.
|
||||
pub derived_from: Option<String>,
|
||||
/// Name of the edge type this one replaced, if any. Set by el; stored verbatim.
|
||||
pub supersedes: Option<String>,
|
||||
/// When true, this type should no longer be used for new edges.
|
||||
/// Set by el; stored verbatim.
|
||||
pub deprecated: bool,
|
||||
}
|
||||
|
||||
/// A node in the engram graph — the fundamental unit of stored memory.
|
||||
///
|
||||
/// A node is not just a record. It is an activation site. Its embedding
|
||||
/// is its semantic identity; its salience governs whether it surfaces
|
||||
/// during retrieval; its activation history encodes its importance to
|
||||
/// the system's ongoing cognition.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Node {
|
||||
/// Stable unique identifier
|
||||
pub id: Uuid,
|
||||
/// Functional role in the memory system
|
||||
pub node_type: NodeType,
|
||||
/// Semantic vector — the node's position in meaning-space.
|
||||
/// Cosine similarity to a query embedding drives spreading activation.
|
||||
pub embedding: Vec<f32>,
|
||||
/// Compressed raw content — the actual payload
|
||||
pub content: Vec<u8>,
|
||||
/// Unix milliseconds when the node was first created
|
||||
pub created_at: i64,
|
||||
/// Unix milliseconds when the node was last activated (read or touched)
|
||||
pub last_activated: i64,
|
||||
/// How many times the node has been activated
|
||||
pub activation_count: u64,
|
||||
/// Composite score: recency × frequency × importance.
|
||||
/// Updated on every touch. Governs spreading activation priority.
|
||||
pub salience: f32,
|
||||
/// Which memory tier this node currently occupies
|
||||
pub tier: MemoryTier,
|
||||
/// Explicit importance, 0.0–1.0. Set by the caller; stable over time.
|
||||
pub importance: f32,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
/// Construct a new node with sensible defaults.
|
||||
pub fn new(
|
||||
node_type: NodeType,
|
||||
embedding: Vec<f32>,
|
||||
content: Vec<u8>,
|
||||
tier: MemoryTier,
|
||||
importance: f32,
|
||||
) -> Self {
|
||||
let now = now_ms();
|
||||
let importance = importance.clamp(0.0, 1.0);
|
||||
// Creation counts as the first activation — activation_count starts at 1.
|
||||
// This ensures a newly created node has non-zero salience from birth.
|
||||
// (ln(1+1) = ln(2) ≈ 0.693, so salience ≈ importance × recency × 0.693)
|
||||
let initial_count = 1u64;
|
||||
let salience = crate::salience::compute_salience(importance, now, initial_count);
|
||||
Self {
|
||||
id: Uuid::new_v4(),
|
||||
node_type,
|
||||
embedding,
|
||||
content,
|
||||
created_at: now,
|
||||
last_activated: now,
|
||||
activation_count: initial_count,
|
||||
salience,
|
||||
tier,
|
||||
importance,
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the node's UUID. Used in tests and deserialization helpers.
|
||||
pub fn with_id(mut self, id: Uuid) -> Self {
|
||||
self.id = id;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A node returned from spreading activation, annotated with how strongly
|
||||
/// it was activated and how many hops from the seed set it is.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ActivatedNode {
|
||||
pub node: Node,
|
||||
/// Activation strength at this node — product of path weights,
|
||||
/// salience, and semantic similarity. Higher is more relevant.
|
||||
pub activation_strength: f32,
|
||||
/// Number of graph hops from the nearest seed node
|
||||
pub hops: u8,
|
||||
}
|
||||
|
||||
/// A node returned from vector similarity search, annotated with its score.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScoredNode {
|
||||
pub node: Node,
|
||||
/// Cosine similarity to the query embedding, in [0.0, 1.0]
|
||||
pub score: f32,
|
||||
}
|
||||
|
||||
/// An edge in the engram graph — a typed, weighted connection between nodes.
|
||||
///
|
||||
/// Edge weights strengthen with co-activation (Hebbian learning).
|
||||
/// The weight directly multiplies activation flow during spreading activation.
|
||||
///
|
||||
/// The `relation` field is a free-form string naming the edge type — look up
|
||||
/// the canonical definition in the `EdgeTypeDef` registry via `edge_type`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Edge {
|
||||
pub id: Uuid,
|
||||
pub from_id: Uuid,
|
||||
pub to_id: Uuid,
|
||||
/// The edge type name (e.g. `"causes"`, `"resonates_with"`). Matches the
|
||||
/// `name` field of the corresponding `EdgeTypeDef` in the registry.
|
||||
pub relation: String,
|
||||
/// Connection strength, 0.0–1.0. Increases when both endpoints are
|
||||
/// activated in close temporal proximity (long-term potentiation).
|
||||
pub weight: f32,
|
||||
pub created_at: i64,
|
||||
/// Unix ms when this edge last carried activation
|
||||
pub last_fired: i64,
|
||||
}
|
||||
|
||||
impl Edge {
|
||||
pub fn new(from_id: Uuid, to_id: Uuid, relation: impl Into<String>, weight: f32) -> Self {
|
||||
let now = now_ms();
|
||||
Self {
|
||||
id: Uuid::new_v4(),
|
||||
from_id,
|
||||
to_id,
|
||||
relation: relation.into(),
|
||||
weight: weight.clamp(0.0, 1.0),
|
||||
created_at: now,
|
||||
last_fired: now,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Current wall time in Unix milliseconds.
|
||||
pub fn now_ms() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock before epoch")
|
||||
.as_millis() as i64
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
/// Vector similarity search over stored node embeddings.
|
||||
///
|
||||
/// Strategy:
|
||||
/// - For < HNSW_THRESHOLD indexed nodes: flat O(n) cosine scan (correct, no deps)
|
||||
/// - For >= HNSW_THRESHOLD nodes: HNSW approximate nearest-neighbour index
|
||||
///
|
||||
/// The HNSW index is built lazily on first search call when the graph is large
|
||||
/// enough. A "dirty" flag in sled (`hnsw:dirty`) is set to 1 whenever `put_node`
|
||||
/// adds an embedding; on the next search the index is rebuilt from the current
|
||||
/// store. For small graphs (< threshold) the flat scan is always used — it is
|
||||
/// fast enough and avoids the overhead of HNSW construction.
|
||||
///
|
||||
/// Cosine similarity: cos(θ) = (A · B) / (|A| × |B|)
|
||||
/// instant-distance expects a *distance* metric (lower = closer), so we expose:
|
||||
/// distance = 1 − cosine_similarity, clamped to [0, 2]
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use crate::storage;
|
||||
#[cfg(feature = "sled-backend")]
|
||||
use sled::Db;
|
||||
|
||||
use crate::error::EngramResult;
|
||||
use crate::types::{Node, ScoredNode};
|
||||
use instant_distance::{Builder, HnswMap, Search};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Minimum number of nodes before we switch from flat scan to HNSW.
|
||||
const HNSW_THRESHOLD: usize = 100;
|
||||
|
||||
/// sled key used to store the dirty flag (1 = needs rebuild, 0 = clean).
|
||||
#[cfg(feature = "sled-backend")]
|
||||
const HNSW_DIRTY_KEY: &[u8] = b"hnsw:dirty";
|
||||
|
||||
// ── Point wrapper ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// An f32 embedding vector treated as an HNSW point.
|
||||
///
|
||||
/// The distance metric is `1 − cosine_similarity` so that instant-distance
|
||||
/// (which minimises distance) finds the most similar vectors.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct EmbeddingPoint(pub Vec<f32>);
|
||||
|
||||
impl instant_distance::Point for EmbeddingPoint {
|
||||
fn distance(&self, other: &Self) -> f32 {
|
||||
let sim = cosine_similarity(&self.0, &other.0);
|
||||
(1.0 - sim).clamp(0.0, 2.0)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Public similarity helper ──────────────────────────────────────────────────
|
||||
|
||||
/// Compute the cosine similarity between two equal-length f32 slices.
|
||||
///
|
||||
/// Returns a value in [-1.0, 1.0], where 1.0 means identical direction.
|
||||
pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
if a.len() != b.len() || a.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if norm_a == 0.0 || norm_b == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
(dot / (norm_a * norm_b)).clamp(-1.0, 1.0)
|
||||
}
|
||||
|
||||
// ── sled-backed search ────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "sled-backend")]
|
||||
/// Search all stored embeddings for the `limit` closest nodes to `query`.
|
||||
///
|
||||
/// Falls back to flat scan for stores with < HNSW_THRESHOLD nodes, or when the
|
||||
/// index has not yet been built. Uses HNSW for large stores.
|
||||
pub fn search_embedding(
|
||||
db: &Db,
|
||||
query: &[f32],
|
||||
limit: usize,
|
||||
node_loader: impl Fn(Uuid) -> EngramResult<Option<Node>>,
|
||||
) -> EngramResult<Vec<ScoredNode>> {
|
||||
let vectors = storage::scan_vectors(db)?;
|
||||
|
||||
if vectors.len() < HNSW_THRESHOLD {
|
||||
return flat_search(query, limit, &vectors, node_loader);
|
||||
}
|
||||
|
||||
// Check if the index needs rebuilding.
|
||||
let dirty = db
|
||||
.get(HNSW_DIRTY_KEY)?
|
||||
.map(|v| v.first().copied().unwrap_or(1) != 0)
|
||||
.unwrap_or(true);
|
||||
|
||||
// We always rebuild if dirty. The index is not serialised to sled because
|
||||
// HnswMap serialisation size can be large and the rebuild is fast (<10ms
|
||||
// for typical node counts). The dirty flag is persisted so we skip
|
||||
// unnecessary rebuilds between searches within the same sled session.
|
||||
let (map, ids) = build_hnsw_index(&vectors);
|
||||
|
||||
if dirty {
|
||||
// Clear the dirty flag now that we have a fresh index.
|
||||
let _ = db.insert(HNSW_DIRTY_KEY, vec![0u8]);
|
||||
}
|
||||
|
||||
hnsw_search(query, limit, &map, &ids, node_loader)
|
||||
}
|
||||
|
||||
/// Mark the HNSW index as dirty. Call this after any `put_node`.
|
||||
#[cfg(feature = "sled-backend")]
|
||||
pub fn mark_dirty(db: &Db) {
|
||||
let _ = db.insert(HNSW_DIRTY_KEY, vec![1u8]);
|
||||
}
|
||||
|
||||
/// Explicitly build and persist the HNSW index. Returns the number of nodes indexed.
|
||||
///
|
||||
/// Not normally needed — the index is built lazily on first search.
|
||||
/// Call this to pre-warm after a large batch insert.
|
||||
#[cfg(feature = "sled-backend")]
|
||||
pub fn build_index(db: &Db) -> EngramResult<usize> {
|
||||
let vectors = storage::scan_vectors(db)?;
|
||||
let n = vectors.len();
|
||||
// Build the index (result is discarded — next search will build from the
|
||||
// current clean state).
|
||||
if n > 0 {
|
||||
let _ = build_hnsw_index(&vectors);
|
||||
}
|
||||
let _ = db.insert(HNSW_DIRTY_KEY, vec![0u8]);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Retrieve the stored embedding for a single node by id.
|
||||
#[cfg(feature = "sled-backend")]
|
||||
pub fn get_embedding(db: &Db, id: Uuid) -> EngramResult<Vec<f32>> {
|
||||
use crate::error::EngramError;
|
||||
let key = storage::vector_key(id);
|
||||
match db.get(key)? {
|
||||
Some(bytes) => {
|
||||
let floats: Vec<f32> = bytes
|
||||
.chunks_exact(4)
|
||||
.map(|c| f32::from_le_bytes(c.try_into().unwrap()))
|
||||
.collect();
|
||||
Ok(floats)
|
||||
}
|
||||
None => Err(EngramError::NotFound(id)),
|
||||
}
|
||||
}
|
||||
|
||||
// ── In-memory search (used by wasm / unit tests) ──────────────────────────────
|
||||
|
||||
/// Search a list of (id, embedding) pairs without a database.
|
||||
pub fn search_embedding_memory(
|
||||
query: &[f32],
|
||||
limit: usize,
|
||||
vectors: &[(Uuid, Vec<f32>)],
|
||||
node_loader: impl Fn(Uuid) -> EngramResult<Option<Node>>,
|
||||
) -> EngramResult<Vec<ScoredNode>> {
|
||||
if vectors.len() < HNSW_THRESHOLD {
|
||||
flat_search(query, limit, vectors, node_loader)
|
||||
} else {
|
||||
let (map, ids) = build_hnsw_index(vectors);
|
||||
hnsw_search(query, limit, &map, &ids, node_loader)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Internal helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Flat cosine scan — O(n). Used when the graph is small.
|
||||
fn flat_search(
|
||||
query: &[f32],
|
||||
limit: usize,
|
||||
vectors: &[(Uuid, Vec<f32>)],
|
||||
node_loader: impl Fn(Uuid) -> EngramResult<Option<Node>>,
|
||||
) -> EngramResult<Vec<ScoredNode>> {
|
||||
let mut scored: Vec<(Uuid, f32)> = vectors
|
||||
.iter()
|
||||
.map(|(id, emb)| (*id, cosine_similarity(query, emb)))
|
||||
.collect();
|
||||
|
||||
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
|
||||
scored.truncate(limit);
|
||||
|
||||
let mut results = Vec::with_capacity(scored.len());
|
||||
for (id, score) in scored {
|
||||
if let Some(node) = node_loader(id)? {
|
||||
results.push(ScoredNode { node, score });
|
||||
}
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Build an HnswMap from a flat vector list.
|
||||
fn build_hnsw_index(vectors: &[(Uuid, Vec<f32>)]) -> (HnswMap<EmbeddingPoint, Uuid>, Vec<Uuid>) {
|
||||
let points: Vec<EmbeddingPoint> = vectors
|
||||
.iter()
|
||||
.map(|(_, emb)| EmbeddingPoint(emb.clone()))
|
||||
.collect();
|
||||
let ids: Vec<Uuid> = vectors.iter().map(|(id, _)| *id).collect();
|
||||
let map = Builder::default().build(points, ids.clone());
|
||||
(map, ids)
|
||||
}
|
||||
|
||||
/// Search using an HnswMap. Converts distance back to cosine similarity score.
|
||||
fn hnsw_search(
|
||||
query: &[f32],
|
||||
limit: usize,
|
||||
map: &HnswMap<EmbeddingPoint, Uuid>,
|
||||
_ids: &[Uuid],
|
||||
node_loader: impl Fn(Uuid) -> EngramResult<Option<Node>>,
|
||||
) -> EngramResult<Vec<ScoredNode>> {
|
||||
let query_point = EmbeddingPoint(query.to_vec());
|
||||
let mut search = Search::default();
|
||||
|
||||
let mut results = Vec::new();
|
||||
for item in map.search(&query_point, &mut search).take(limit) {
|
||||
// distance = 1 − cosine_sim → cosine_sim = 1 − distance
|
||||
let score = (1.0 - item.distance).clamp(-1.0, 1.0);
|
||||
let node_id = *item.value;
|
||||
if let Some(node) = node_loader(node_id)? {
|
||||
results.push(ScoredNode { node, score });
|
||||
}
|
||||
}
|
||||
// Ensure descending score order (HNSW returns ascending distance order)
|
||||
results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{MemoryTier, Node, NodeType};
|
||||
|
||||
fn dummy_node(id: Uuid) -> Node {
|
||||
Node::new(
|
||||
NodeType::Memory,
|
||||
vec![0.0; 4],
|
||||
vec![],
|
||||
MemoryTier::Episodic,
|
||||
0.5,
|
||||
)
|
||||
.with_id(id)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_identical_vectors() {
|
||||
let v = vec![1.0_f32, 0.0, 0.0, 1.0];
|
||||
assert!((cosine_similarity(&v, &v) - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_orthogonal_vectors() {
|
||||
let a = vec![1.0_f32, 0.0];
|
||||
let b = vec![0.0_f32, 1.0];
|
||||
assert!(cosine_similarity(&a, &b).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_opposite_vectors() {
|
||||
let a = vec![1.0_f32, 0.0];
|
||||
let b = vec![-1.0_f32, 0.0];
|
||||
let sim = cosine_similarity(&a, &b);
|
||||
assert!((sim - (-1.0)).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_search_returns_ordered_results() {
|
||||
let id_best = Uuid::new_v4();
|
||||
let id_mid = Uuid::new_v4();
|
||||
let id_low = Uuid::new_v4();
|
||||
|
||||
let query = vec![1.0_f32, 0.0, 0.0, 0.0];
|
||||
let vecs: Vec<(Uuid, Vec<f32>)> = vec![
|
||||
(id_low, vec![0.0, 1.0, 0.0, 0.0]), // sim=0
|
||||
(id_best, vec![1.0, 0.0, 0.0, 0.0]), // sim=1 ← best
|
||||
(id_mid, vec![0.7, 0.7, 0.0, 0.0]), // sim≈0.7
|
||||
];
|
||||
|
||||
let results = flat_search(&query, 3, &vecs, |id| Ok(Some(dummy_node(id)))).unwrap();
|
||||
|
||||
assert_eq!(results.len(), 3);
|
||||
assert_eq!(results[0].node.id, id_best);
|
||||
assert!(results[0].score > results[1].score);
|
||||
assert!(results[1].score > results[2].score);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_search_respects_limit() {
|
||||
let query = vec![1.0_f32, 0.0];
|
||||
let vecs: Vec<(Uuid, Vec<f32>)> = (0..10)
|
||||
.map(|i| (Uuid::new_v4(), vec![i as f32, 0.0]))
|
||||
.collect();
|
||||
let results = flat_search(&query, 3, &vecs, |id| Ok(Some(dummy_node(id)))).unwrap();
|
||||
assert_eq!(results.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedding_point_distance_self_is_zero() {
|
||||
use instant_distance::Point;
|
||||
let p = EmbeddingPoint(vec![0.6_f32, 0.8]);
|
||||
assert!(p.distance(&p) < 1e-5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_memory_small_falls_back_to_flat() {
|
||||
let vecs: Vec<(Uuid, Vec<f32>)> = (0..10)
|
||||
.map(|i| {
|
||||
let mut emb = vec![0.0_f32; 8];
|
||||
emb[i % 8] = 1.0;
|
||||
(Uuid::new_v4(), emb)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let target = vecs[3].clone();
|
||||
let query = target.1.clone();
|
||||
|
||||
let results =
|
||||
search_embedding_memory(&query, 1, &vecs, |id| Ok(Some(dummy_node(id)))).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].node.id, target.0);
|
||||
assert!((results[0].score - 1.0).abs() < 1e-5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cosine_zero_vector_returns_zero() {
|
||||
let a = vec![0.0_f32, 0.0];
|
||||
let b = vec![1.0_f32, 0.0];
|
||||
assert_eq!(cosine_similarity(&a, &b), 0.0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user