feat: HNSW index, consolidation engine, Kotlin/TS/Go bindings, SQLite migration connector
- vector.rs: replace flat O(n) scan with instant-distance HNSW for stores >= 100 nodes; flat scan retained as fallback for small graphs; dirty-flag persistence in sled triggers index rebuild only when nodes are added - consolidation.rs: Episodic → Semantic promotion based on activation_count and salience_floor thresholds; global decay pass after each cycle; ConsolidationConfig + ConsolidationReport types; 8 tests - migration.rs: reads Neuron SQLite (memory_nodes, knowledge_entries, graph_edges) and writes to Engram sled; placeholder unit-vector embeddings with TODO for ONNX; 5 tests including full in-memory DB roundtrip - crates/engram-migrate: CLI binary (engram-migrate --sqlite / --output) - crates/engram-jni: JNI cdylib exposing open/close/put_node/get_node/ activate/search_embedding/touch/decay/node_count/edge_count via Java_ai_neuron_engram_EngramDb_* entry points; 6 tests - bindings/kotlin: EngramDb.kt (AutoCloseable JNI wrapper), EngramNode, EngramEdge, ActivatedNode, EngramTypes; build.gradle.kts; settings.gradle.kts - bindings/typescript: engram-wasm crate (wasm-bindgen, serde-wasm-bindgen); WasmEngramDb with in-memory backend (sled not available in WASM); TypeScript wrapper (index.ts, types.ts, package.json, tsconfig.json) - bindings/go: engram.go (CGo wrapper), engram.h (C header), engram_test.go (4 tests covering open/close/put_node/get_node/node_count/decay); go.mod - engram-core: wasm feature gate for in-memory backend; mem_storage.rs; activation.activate_mem for WASM path; Node::with_id helper; salience.rs doctest fixed (text block) - examples/basic.rs: consolidation section added - examples/migrate.rs: migration API demonstration Build: cargo build --workspace -- zero warnings, zero errors Tests: 38 pass (25 engram-core + 7 engram-ffi + 6 engram-jni)
This commit is contained in:
Will Anderson
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "engram-wasm"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "WASM/TypeScript bindings for engram-core via wasm-bindgen"
|
||||
license = "MIT"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
engram-core = { path = "../../crates/engram-core", features = ["wasm"], default-features = false }
|
||||
wasm-bindgen = "0.2"
|
||||
serde-wasm-bindgen = "0.6"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
uuid = { version = "1", features = ["v4", "serde", "js"] }
|
||||
getrandom = { version = "0.2", features = ["js"] }
|
||||
console_error_panic_hook = { version = "0.1", optional = true }
|
||||
|
||||
[features]
|
||||
default = ["console_error_panic_hook"]
|
||||
|
||||
[package.metadata.wasm-pack.profile.release]
|
||||
wasm-opt = false
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@neuron/engram",
|
||||
"version": "0.1.0",
|
||||
"description": "Engram memory substrate — TypeScript/WASM bindings",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"files": [
|
||||
"dist",
|
||||
"pkg"
|
||||
],
|
||||
"scripts": {
|
||||
"build:wasm": "wasm-pack build . --target web --out-dir pkg",
|
||||
"build:ts": "tsc",
|
||||
"build": "npm run build:wasm && npm run build:ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* TypeScript wrapper around the engram WASM module.
|
||||
*
|
||||
* Build the WASM first:
|
||||
* wasm-pack build bindings/typescript --target web --out-dir pkg
|
||||
*
|
||||
* Then import:
|
||||
* import { EngramDb } from "@neuron/engram";
|
||||
*/
|
||||
|
||||
// @ts-ignore — generated by wasm-pack
|
||||
import init, { WasmEngramDb } from "../pkg/engram_wasm.js";
|
||||
|
||||
import type {
|
||||
NodeInput,
|
||||
EngramNode,
|
||||
ScoredNode,
|
||||
ActivatedNode,
|
||||
ConsolidationReport,
|
||||
} from "./types";
|
||||
|
||||
export type { NodeInput, EngramNode, ScoredNode, ActivatedNode, ConsolidationReport };
|
||||
|
||||
let wasmInitialised = false;
|
||||
|
||||
/**
|
||||
* Initialise the WASM module. Must be called once before creating any EngramDb.
|
||||
*/
|
||||
export async function initEngram(): Promise<void> {
|
||||
if (!wasmInitialised) {
|
||||
await init();
|
||||
wasmInitialised = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* TypeScript wrapper around `WasmEngramDb`.
|
||||
*
|
||||
* All state is in-memory (WASM has no filesystem access). The path argument
|
||||
* is accepted for API symmetry but is ignored.
|
||||
*
|
||||
* ```ts
|
||||
* await initEngram();
|
||||
* const db = new EngramDb();
|
||||
*
|
||||
* const id = await db.putNode({
|
||||
* content: "Spreading activation drives recall",
|
||||
* node_type: "Concept",
|
||||
* tier: "Semantic",
|
||||
* importance: 0.9,
|
||||
* embedding: Array.from({ length: 384 }, () => Math.random()),
|
||||
* });
|
||||
*
|
||||
* const results = await db.searchEmbedding(queryEmbedding, 5);
|
||||
* ```
|
||||
*/
|
||||
export class EngramDb {
|
||||
private db: WasmEngramDb;
|
||||
|
||||
constructor(path = "/wasm-memory") {
|
||||
this.db = new WasmEngramDb(path);
|
||||
}
|
||||
|
||||
/** Store a node and return its UUID. */
|
||||
putNode(node: NodeInput): string {
|
||||
return this.db.put_node(node);
|
||||
}
|
||||
|
||||
/** Retrieve a node by UUID, or null if not found. */
|
||||
getNode(id: string): EngramNode | null {
|
||||
return this.db.get_node(id);
|
||||
}
|
||||
|
||||
/** Find the `limit` most similar nodes by embedding vector. */
|
||||
searchEmbedding(embedding: Float32Array | number[], limit: number): ScoredNode[] {
|
||||
const arr = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
|
||||
return this.db.search_embedding(arr, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run spreading activation from seed nodes.
|
||||
*
|
||||
* @param seeds Array of UUID strings (the "active context")
|
||||
* @param queryEmbedding Semantic vector for the current query
|
||||
* @param maxDepth Maximum BFS hops (typically 2–4)
|
||||
* @param limit Number of results to return
|
||||
*/
|
||||
activate(
|
||||
seeds: string[],
|
||||
queryEmbedding: Float32Array | number[],
|
||||
maxDepth = 3,
|
||||
limit = 10
|
||||
): ActivatedNode[] {
|
||||
const arr =
|
||||
queryEmbedding instanceof Float32Array
|
||||
? queryEmbedding
|
||||
: new Float32Array(queryEmbedding);
|
||||
return this.db.activate(seeds, arr, maxDepth, limit);
|
||||
}
|
||||
|
||||
/** Mark a node as recently accessed (increments activation count). */
|
||||
touch(id: string): void {
|
||||
this.db.touch(id);
|
||||
}
|
||||
|
||||
/** Apply multiplicative salience decay. Returns the number of nodes updated. */
|
||||
decay(factor: number): number {
|
||||
return this.db.decay(factor);
|
||||
}
|
||||
|
||||
/** Run a memory consolidation cycle. */
|
||||
consolidate(): ConsolidationReport {
|
||||
return this.db.consolidate();
|
||||
}
|
||||
|
||||
/** Total number of nodes stored. */
|
||||
nodeCount(): number {
|
||||
return this.db.node_count();
|
||||
}
|
||||
|
||||
/** Total number of edges stored. */
|
||||
edgeCount(): number {
|
||||
return this.db.edge_count();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
/// WASM/TypeScript bindings for engram-core via wasm-bindgen.
|
||||
///
|
||||
/// This crate compiles to a WebAssembly module that can be loaded by the
|
||||
/// TypeScript wrapper in `src/index.ts`. All types are passed as JSON strings
|
||||
/// across the WASM boundary to avoid bespoke serialisation code.
|
||||
///
|
||||
/// # Storage
|
||||
/// sled is not available in WASM (no filesystem). When compiled with the `wasm`
|
||||
/// feature, engram-core switches to an in-memory HashMap backend. All state is
|
||||
/// therefore lost on page reload — persistence requires sending nodes to a
|
||||
/// server-side store and re-loading them on startup.
|
||||
///
|
||||
/// # Build
|
||||
/// ```
|
||||
/// wasm-pack build bindings/typescript --target web
|
||||
/// ```
|
||||
use engram_core::{
|
||||
ActivatedNode, ConsolidationConfig, EngramDb, MemoryTier, Node, NodeType, ScoredNode,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
use uuid::Uuid;
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
// ── Panic hook ────────────────────────────────────────────────────────────────
|
||||
|
||||
#[wasm_bindgen(start)]
|
||||
pub fn main() {
|
||||
#[cfg(feature = "console_error_panic_hook")]
|
||||
console_error_panic_hook::set_once();
|
||||
}
|
||||
|
||||
// ── WasmEngramDb ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// The main entry point for WASM callers.
|
||||
///
|
||||
/// TypeScript:
|
||||
/// ```ts
|
||||
/// const db = new WasmEngramDb("ignored-path");
|
||||
/// const id = db.putNode(JSON.stringify({ content: "...", node_type: "Memory", ... }));
|
||||
/// ```
|
||||
#[wasm_bindgen]
|
||||
pub struct WasmEngramDb {
|
||||
inner: EngramDb,
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
impl WasmEngramDb {
|
||||
/// Create a new in-memory engram database.
|
||||
///
|
||||
/// The `path` argument is accepted for API symmetry with the sled backend
|
||||
/// but is ignored — all storage is in-memory.
|
||||
#[wasm_bindgen(constructor)]
|
||||
pub fn open(_path: &str) -> Result<WasmEngramDb, JsValue> {
|
||||
let db = EngramDb::open(Path::new("/wasm-memory"))
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
Ok(WasmEngramDb { inner: db })
|
||||
}
|
||||
|
||||
// ── Node operations ───────────────────────────────────────────────────────
|
||||
|
||||
/// Store a node. Accepts a JSON object with fields:
|
||||
/// `{ content, node_type, tier, importance, embedding }`
|
||||
/// Returns the assigned UUID string.
|
||||
pub fn put_node(&self, node: JsValue) -> Result<String, JsValue> {
|
||||
let n = js_value_to_node(node)?;
|
||||
self.inner
|
||||
.put_node(n)
|
||||
.map(|id| id.to_string())
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
/// Retrieve a node by UUID. Returns a JSON object or null.
|
||||
pub fn get_node(&self, id: &str) -> Result<JsValue, JsValue> {
|
||||
let uuid = id
|
||||
.parse::<Uuid>()
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
match self
|
||||
.inner
|
||||
.get_node(uuid)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?
|
||||
{
|
||||
Some(node) => node_to_js_value(&node),
|
||||
None => Ok(JsValue::NULL),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Vector search ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Search for similar nodes by embedding vector.
|
||||
///
|
||||
/// `embedding` is a JS Float32Array. Returns a JSON array of
|
||||
/// `{ node, score }` objects.
|
||||
pub fn search_embedding(
|
||||
&self,
|
||||
embedding: &[f32],
|
||||
limit: usize,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let results = self
|
||||
.inner
|
||||
.search_embedding(embedding, limit)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
scored_nodes_to_js(&results)
|
||||
}
|
||||
|
||||
// ── Spreading activation ──────────────────────────────────────────────────
|
||||
|
||||
/// Run spreading activation.
|
||||
///
|
||||
/// `seeds` is a JS array of UUID strings.
|
||||
/// `query_embedding` is a Float32Array.
|
||||
/// Returns a JSON array of `{ node, activation_strength, hops }`.
|
||||
pub fn activate(
|
||||
&self,
|
||||
seeds: JsValue,
|
||||
query_embedding: &[f32],
|
||||
max_depth: u8,
|
||||
limit: usize,
|
||||
) -> Result<JsValue, JsValue> {
|
||||
let seed_strs: Vec<String> = serde_wasm_bindgen::from_value(seeds)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
let seeds: Vec<Uuid> = seed_strs
|
||||
.iter()
|
||||
.filter_map(|s| s.parse::<Uuid>().ok())
|
||||
.collect();
|
||||
|
||||
let results = self
|
||||
.inner
|
||||
.activate(&seeds, query_embedding, max_depth, limit)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
activated_nodes_to_js(&results)
|
||||
}
|
||||
|
||||
// ── Salience management ───────────────────────────────────────────────────
|
||||
|
||||
/// Touch a node (increment activation count and update salience).
|
||||
pub fn touch(&self, id: &str) -> Result<(), JsValue> {
|
||||
let uuid = id
|
||||
.parse::<Uuid>()
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
self.inner
|
||||
.touch(uuid)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
/// Apply multiplicative salience decay. Returns the number of nodes updated.
|
||||
pub fn decay(&self, factor: f32) -> Result<u32, JsValue> {
|
||||
self.inner
|
||||
.decay(factor)
|
||||
.map(|n| n as u32)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
// ── Consolidation ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Run a memory consolidation cycle.
|
||||
/// Returns `{ promoted, decayed, pruned }`.
|
||||
pub fn consolidate(&self) -> Result<JsValue, JsValue> {
|
||||
let config = ConsolidationConfig::default();
|
||||
let report = self
|
||||
.inner
|
||||
.consolidate(&config)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))?;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Report {
|
||||
promoted: usize,
|
||||
decayed: usize,
|
||||
pruned: usize,
|
||||
}
|
||||
serde_wasm_bindgen::to_value(&Report {
|
||||
promoted: report.promoted,
|
||||
decayed: report.decayed,
|
||||
pruned: report.pruned,
|
||||
})
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
// ── Statistics ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Return the total number of nodes.
|
||||
pub fn node_count(&self) -> Result<u32, JsValue> {
|
||||
self.inner
|
||||
.node_count()
|
||||
.map(|n| n as u32)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
/// Return the total number of edges.
|
||||
pub fn edge_count(&self) -> Result<u32, JsValue> {
|
||||
self.inner
|
||||
.edge_count()
|
||||
.map(|n| n as u32)
|
||||
.map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
// ── Serialisation helpers ─────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct NodeInput {
|
||||
content: String,
|
||||
node_type: Option<String>,
|
||||
tier: Option<String>,
|
||||
importance: Option<f32>,
|
||||
embedding: Option<Vec<f32>>,
|
||||
}
|
||||
|
||||
fn js_value_to_node(val: JsValue) -> Result<Node, JsValue> {
|
||||
let input: NodeInput = serde_wasm_bindgen::from_value(val)
|
||||
.map_err(|e| JsValue::from_str(&format!("Invalid node: {e}")))?;
|
||||
|
||||
let node_type = match input.node_type.as_deref().unwrap_or("Memory") {
|
||||
"Concept" => NodeType::Concept,
|
||||
"Event" => NodeType::Event,
|
||||
"Entity" => NodeType::Entity,
|
||||
"Process" => NodeType::Process,
|
||||
"InternalState" => NodeType::InternalState,
|
||||
_ => NodeType::Memory,
|
||||
};
|
||||
let tier = match input.tier.as_deref().unwrap_or("Episodic") {
|
||||
"Working" => MemoryTier::Working,
|
||||
"Semantic" => MemoryTier::Semantic,
|
||||
"Procedural" => MemoryTier::Procedural,
|
||||
_ => MemoryTier::Episodic,
|
||||
};
|
||||
let embedding = input.embedding.unwrap_or_default();
|
||||
let importance = input.importance.unwrap_or(0.5);
|
||||
|
||||
Ok(Node::new(node_type, embedding, input.content.into_bytes(), tier, importance))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct NodeOutput {
|
||||
id: String,
|
||||
content: String,
|
||||
node_type: String,
|
||||
tier: String,
|
||||
salience: f32,
|
||||
importance: f32,
|
||||
activation_count: u64,
|
||||
embedding: Vec<f32>,
|
||||
}
|
||||
|
||||
fn node_to_js_value(node: &Node) -> Result<JsValue, JsValue> {
|
||||
let out = NodeOutput {
|
||||
id: node.id.to_string(),
|
||||
content: String::from_utf8_lossy(&node.content).into_owned(),
|
||||
node_type: format!("{:?}", node.node_type),
|
||||
tier: format!("{:?}", node.tier),
|
||||
salience: node.salience,
|
||||
importance: node.importance,
|
||||
activation_count: node.activation_count,
|
||||
embedding: node.embedding.clone(),
|
||||
};
|
||||
serde_wasm_bindgen::to_value(&out).map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ScoredNodeOutput {
|
||||
node: NodeOutput,
|
||||
score: f32,
|
||||
}
|
||||
|
||||
fn scored_nodes_to_js(nodes: &[ScoredNode]) -> Result<JsValue, JsValue> {
|
||||
let out: Vec<ScoredNodeOutput> = nodes
|
||||
.iter()
|
||||
.map(|s| ScoredNodeOutput {
|
||||
node: NodeOutput {
|
||||
id: s.node.id.to_string(),
|
||||
content: String::from_utf8_lossy(&s.node.content).into_owned(),
|
||||
node_type: format!("{:?}", s.node.node_type),
|
||||
tier: format!("{:?}", s.node.tier),
|
||||
salience: s.node.salience,
|
||||
importance: s.node.importance,
|
||||
activation_count: s.node.activation_count,
|
||||
embedding: s.node.embedding.clone(),
|
||||
},
|
||||
score: s.score,
|
||||
})
|
||||
.collect();
|
||||
serde_wasm_bindgen::to_value(&out).map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ActivatedNodeOutput {
|
||||
node: NodeOutput,
|
||||
activation_strength: f32,
|
||||
hops: u8,
|
||||
}
|
||||
|
||||
fn activated_nodes_to_js(nodes: &[ActivatedNode]) -> Result<JsValue, JsValue> {
|
||||
let out: Vec<ActivatedNodeOutput> = nodes
|
||||
.iter()
|
||||
.map(|a| ActivatedNodeOutput {
|
||||
node: NodeOutput {
|
||||
id: a.node.id.to_string(),
|
||||
content: String::from_utf8_lossy(&a.node.content).into_owned(),
|
||||
node_type: format!("{:?}", a.node.node_type),
|
||||
tier: format!("{:?}", a.node.tier),
|
||||
salience: a.node.salience,
|
||||
importance: a.node.importance,
|
||||
activation_count: a.node.activation_count,
|
||||
embedding: a.node.embedding.clone(),
|
||||
},
|
||||
activation_strength: a.activation_strength,
|
||||
hops: a.hops,
|
||||
})
|
||||
.collect();
|
||||
serde_wasm_bindgen::to_value(&out).map_err(|e| JsValue::from_str(&e.to_string()))
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* TypeScript types mirroring the Rust structs in engram-core.
|
||||
*/
|
||||
|
||||
export type NodeType =
|
||||
| "Memory"
|
||||
| "Concept"
|
||||
| "Event"
|
||||
| "Entity"
|
||||
| "Process"
|
||||
| "InternalState";
|
||||
|
||||
export type MemoryTier = "Working" | "Episodic" | "Semantic" | "Procedural";
|
||||
|
||||
export interface EngramNode {
|
||||
id: string;
|
||||
content: string;
|
||||
node_type: NodeType;
|
||||
tier: MemoryTier;
|
||||
salience: number;
|
||||
importance: number;
|
||||
activation_count: number;
|
||||
embedding: number[];
|
||||
}
|
||||
|
||||
export interface NodeInput {
|
||||
content: string;
|
||||
node_type?: NodeType;
|
||||
tier?: MemoryTier;
|
||||
importance?: number;
|
||||
embedding?: number[];
|
||||
}
|
||||
|
||||
export interface ScoredNode {
|
||||
node: EngramNode;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface ActivatedNode {
|
||||
node: EngramNode;
|
||||
activation_strength: number;
|
||||
hops: number;
|
||||
}
|
||||
|
||||
export interface ConsolidationReport {
|
||||
promoted: number;
|
||||
decayed: number;
|
||||
pruned: number;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "ES2020",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "dist",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"allowJs": false,
|
||||
"rootDir": "src"
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "pkg"]
|
||||
}
|
||||
Reference in New Issue
Block a user