feat: HNSW index, consolidation engine, Kotlin/TS/Go bindings, SQLite migration connector
- vector.rs: replace flat O(n) scan with instant-distance HNSW for stores >= 100 nodes; flat scan retained as fallback for small graphs; dirty-flag persistence in sled triggers index rebuild only when nodes are added - consolidation.rs: Episodic → Semantic promotion based on activation_count and salience_floor thresholds; global decay pass after each cycle; ConsolidationConfig + ConsolidationReport types; 8 tests - migration.rs: reads Neuron SQLite (memory_nodes, knowledge_entries, graph_edges) and writes to Engram sled; placeholder unit-vector embeddings with TODO for ONNX; 5 tests including full in-memory DB roundtrip - crates/engram-migrate: CLI binary (engram-migrate --sqlite / --output) - crates/engram-jni: JNI cdylib exposing open/close/put_node/get_node/ activate/search_embedding/touch/decay/node_count/edge_count via Java_ai_neuron_engram_EngramDb_* entry points; 6 tests - bindings/kotlin: EngramDb.kt (AutoCloseable JNI wrapper), EngramNode, EngramEdge, ActivatedNode, EngramTypes; build.gradle.kts; settings.gradle.kts - bindings/typescript: engram-wasm crate (wasm-bindgen, serde-wasm-bindgen); WasmEngramDb with in-memory backend (sled not available in WASM); TypeScript wrapper (index.ts, types.ts, package.json, tsconfig.json) - bindings/go: engram.go (CGo wrapper), engram.h (C header), engram_test.go (4 tests covering open/close/put_node/get_node/node_count/decay); go.mod - engram-core: wasm feature gate for in-memory backend; mem_storage.rs; activation.activate_mem for WASM path; Node::with_id helper; salience.rs doctest fixed (text block) - examples/basic.rs: consolidation section added - examples/migrate.rs: migration API demonstration Build: cargo build --workspace -- zero warnings, zero errors Tests: 38 pass (25 engram-core + 7 engram-ffi + 6 engram-jni)
This commit is contained in:
Will Anderson
@@ -0,0 +1,197 @@
|
||||
// Package engram provides Go bindings for the Engram memory substrate via CGo.
|
||||
//
|
||||
// Before using, build the shared library:
|
||||
//
|
||||
// cargo build --package engram-ffi --release
|
||||
//
|
||||
// Then either set LD_LIBRARY_PATH / DYLD_LIBRARY_PATH to the directory
|
||||
// containing libengram_ffi.so/.dylib, or copy the library to a standard path.
|
||||
//
|
||||
// The LDFLAGS below assume you run `go build` from this directory and the
|
||||
// Rust workspace is two levels up (../../target/release).
|
||||
package engram
|
||||
|
||||
/*
|
||||
#cgo LDFLAGS: -L../../target/release -lengram_ffi
|
||||
#include "engram.h"
|
||||
#include <stdlib.h>
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
// DB is a handle to an open Engram database.
|
||||
type DB struct {
|
||||
ptr *C.EngramHandle
|
||||
}
|
||||
|
||||
// Node mirrors the Rust Node struct.
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
NodeType string `json:"node_type"`
|
||||
Tier string `json:"tier"`
|
||||
Salience float32 `json:"salience"`
|
||||
Importance float32 `json:"importance"`
|
||||
ActivationCount uint64 `json:"activation_count"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
}
|
||||
|
||||
// NodeInput is used when creating a new node.
|
||||
type NodeInput struct {
|
||||
Content string `json:"content"`
|
||||
NodeType string `json:"node_type,omitempty"`
|
||||
Tier string `json:"tier,omitempty"`
|
||||
Importance float32 `json:"importance,omitempty"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
}
|
||||
|
||||
// ActivatedNode is a node returned from spreading activation.
|
||||
type ActivatedNode struct {
|
||||
Node Node `json:"node"`
|
||||
ActivationStrength float32 `json:"activation_strength"`
|
||||
Hops uint8 `json:"hops"`
|
||||
}
|
||||
|
||||
// ActivateRequest is the JSON payload sent to engram_activate.
|
||||
type activateRequest struct {
|
||||
Seeds []string `json:"seeds"`
|
||||
QueryEmbedding []float32 `json:"query_embedding"`
|
||||
MaxDepth uint8 `json:"max_depth"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
// ── Lifecycle ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// Open opens or creates an Engram database at the given path.
|
||||
func Open(path string) (*DB, error) {
|
||||
cpath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cpath))
|
||||
|
||||
ptr := C.engram_open(cpath)
|
||||
if ptr == nil {
|
||||
return nil, fmt.Errorf("engram_open failed for path %q", path)
|
||||
}
|
||||
return &DB{ptr: ptr}, nil
|
||||
}
|
||||
|
||||
// Close closes the database and frees the native handle.
|
||||
func (db *DB) Close() {
|
||||
if db.ptr != nil {
|
||||
C.engram_close(db.ptr)
|
||||
db.ptr = nil
|
||||
}
|
||||
}
|
||||
|
||||
// ── Statistics ────────────────────────────────────────────────────────────────
|
||||
|
||||
// NodeCount returns the total number of nodes.
|
||||
func (db *DB) NodeCount() (uint64, error) {
|
||||
n := C.engram_node_count(db.ptr)
|
||||
if n < 0 {
|
||||
return 0, errors.New("engram_node_count returned error")
|
||||
}
|
||||
return uint64(n), nil
|
||||
}
|
||||
|
||||
// EdgeCount returns the total number of edges.
|
||||
func (db *DB) EdgeCount() (uint64, error) {
|
||||
n := C.engram_edge_count(db.ptr)
|
||||
if n < 0 {
|
||||
return 0, errors.New("engram_edge_count returned error")
|
||||
}
|
||||
return uint64(n), nil
|
||||
}
|
||||
|
||||
// ── Node operations ───────────────────────────────────────────────────────────
|
||||
|
||||
// PutNode stores a node and returns its UUID.
|
||||
func (db *DB) PutNode(node *NodeInput) (string, error) {
|
||||
jsonBytes, err := json.Marshal(node)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal node: %w", err)
|
||||
}
|
||||
|
||||
cjson := C.CString(string(jsonBytes))
|
||||
defer C.free(unsafe.Pointer(cjson))
|
||||
|
||||
result := C.engram_put_node(db.ptr, cjson)
|
||||
if result == nil {
|
||||
return "", errors.New("engram_put_node returned null")
|
||||
}
|
||||
defer C.engram_free_string(result)
|
||||
|
||||
return C.GoString(result), nil
|
||||
}
|
||||
|
||||
// GetNode retrieves a node by UUID. Returns nil if not found.
|
||||
func (db *DB) GetNode(id string) (*Node, error) {
|
||||
cid := C.CString(id)
|
||||
defer C.free(unsafe.Pointer(cid))
|
||||
|
||||
result := C.engram_get_node(db.ptr, cid)
|
||||
if result == nil {
|
||||
return nil, nil
|
||||
}
|
||||
defer C.engram_free_string(result)
|
||||
|
||||
var node Node
|
||||
if err := json.Unmarshal([]byte(C.GoString(result)), &node); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal node: %w", err)
|
||||
}
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// ── Spreading activation ──────────────────────────────────────────────────────
|
||||
|
||||
// Activate runs spreading activation from seed node UUIDs.
|
||||
func (db *DB) Activate(
|
||||
seeds []string,
|
||||
queryEmbedding []float32,
|
||||
maxDepth uint8,
|
||||
limit int,
|
||||
) ([]*ActivatedNode, error) {
|
||||
req := activateRequest{
|
||||
Seeds: seeds,
|
||||
QueryEmbedding: queryEmbedding,
|
||||
MaxDepth: maxDepth,
|
||||
Limit: limit,
|
||||
}
|
||||
jsonBytes, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal activate request: %w", err)
|
||||
}
|
||||
|
||||
cjson := C.CString(string(jsonBytes))
|
||||
defer C.free(unsafe.Pointer(cjson))
|
||||
|
||||
result := C.engram_activate(db.ptr, cjson)
|
||||
if result == nil {
|
||||
return nil, errors.New("engram_activate returned null")
|
||||
}
|
||||
defer C.engram_free_string(result)
|
||||
|
||||
var nodes []*ActivatedNode
|
||||
if err := json.Unmarshal([]byte(C.GoString(result)), &nodes); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal activate result: %w", err)
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// ── Salience management ───────────────────────────────────────────────────────
|
||||
|
||||
// Decay applies multiplicative salience decay to all nodes.
|
||||
// factor should be in (0.0, 1.0). Returns the number of nodes updated.
|
||||
func (db *DB) Decay(factor float32) (uint64, error) {
|
||||
n := C.engram_decay(db.ptr, C.float(factor))
|
||||
if n < 0 {
|
||||
return 0, errors.New("engram_decay returned error")
|
||||
}
|
||||
return uint64(n), nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* engram.h — C header for the engram FFI.
|
||||
*
|
||||
* Generated from crates/engram-ffi/src/lib.rs.
|
||||
* To regenerate: cargo install cbindgen && cbindgen --crate engram-ffi -o bindings/go/engram.h
|
||||
*
|
||||
* Build the shared library:
|
||||
* cargo build --package engram-ffi --release
|
||||
* # macOS: target/release/libengram_ffi.dylib
|
||||
* # Linux: target/release/libengram_ffi.so
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/** Opaque handle to an open EngramDb. Obtained via engram_open. */
|
||||
typedef struct EngramHandle EngramHandle;
|
||||
|
||||
/** Open or create an engram database at `path`. Returns null on error. */
|
||||
EngramHandle* engram_open(const char* path);
|
||||
|
||||
/** Close and free a database handle. The pointer must not be used afterwards. */
|
||||
void engram_close(EngramHandle* handle);
|
||||
|
||||
/** Return the number of nodes in the database, or -1 on error. */
|
||||
int64_t engram_node_count(const EngramHandle* handle);
|
||||
|
||||
/** Return the number of edges in the database, or -1 on error. */
|
||||
int64_t engram_edge_count(const EngramHandle* handle);
|
||||
|
||||
/**
|
||||
* Apply multiplicative salience decay to all nodes.
|
||||
* `factor` should be in (0.0, 1.0). Returns nodes updated, or -1 on error.
|
||||
*/
|
||||
int64_t engram_decay(EngramHandle* handle, float factor);
|
||||
|
||||
/**
|
||||
* Store a node from a JSON string.
|
||||
* JSON: { "content": "...", "node_type": "Memory", "tier": "Episodic",
|
||||
* "importance": 0.8, "embedding": [f32, ...] }
|
||||
* Returns a heap-allocated UUID string on success, null on error.
|
||||
* Free with engram_free_string.
|
||||
*/
|
||||
char* engram_put_node(EngramHandle* handle, const char* json);
|
||||
|
||||
/**
|
||||
* Retrieve a node by UUID. Returns a heap-allocated JSON string, or null.
|
||||
* Free with engram_free_string.
|
||||
*/
|
||||
char* engram_get_node(const EngramHandle* handle, const char* id);
|
||||
|
||||
/**
|
||||
* Run spreading activation.
|
||||
* `req_json`: { "seeds": ["uuid", ...], "query_embedding": [f32, ...],
|
||||
* "max_depth": 3, "limit": 10 }
|
||||
* Returns a heap-allocated JSON array, or null on error.
|
||||
* Free with engram_free_string.
|
||||
*/
|
||||
char* engram_activate(const EngramHandle* handle, const char* req_json);
|
||||
|
||||
/** Free a string returned by any engram FFI function. */
|
||||
void engram_free_string(char* s);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
@@ -0,0 +1,130 @@
|
||||
// Package engram basic test.
|
||||
//
|
||||
// NOTE: This test requires the FFI shared library to be compiled first:
|
||||
//
|
||||
// cargo build --package engram-ffi --release
|
||||
//
|
||||
// Then run:
|
||||
//
|
||||
// DYLD_LIBRARY_PATH=../../target/release go test ./...
|
||||
// # or on Linux:
|
||||
// LD_LIBRARY_PATH=../../target/release go test ./...
|
||||
package engram
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOpenClose verifies that a database can be opened and closed without errors.
|
||||
func TestOpenClose(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(filepath.Join(dir, "test-engram"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
n, err := db.NodeCount()
|
||||
if err != nil {
|
||||
t.Fatalf("NodeCount: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("expected 0 nodes, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPutGetNode verifies node storage and retrieval roundtrip.
|
||||
func TestPutGetNode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(filepath.Join(dir, "test-engram"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
input := &NodeInput{
|
||||
Content: "Spreading activation is the core retrieval mechanism",
|
||||
NodeType: "Concept",
|
||||
Tier: "Semantic",
|
||||
Importance: 0.9,
|
||||
Embedding: []float32{0.1, 0.2, 0.3, 0.4},
|
||||
}
|
||||
|
||||
id, err := db.PutNode(input)
|
||||
if err != nil {
|
||||
t.Fatalf("PutNode: %v", err)
|
||||
}
|
||||
if id == "" {
|
||||
t.Fatal("expected non-empty UUID")
|
||||
}
|
||||
|
||||
node, err := db.GetNode(id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetNode: %v", err)
|
||||
}
|
||||
if node == nil {
|
||||
t.Fatal("expected node, got nil")
|
||||
}
|
||||
if node.Content != input.Content {
|
||||
t.Errorf("content mismatch: got %q, want %q", node.Content, input.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeCount verifies node count increments.
|
||||
func TestNodeCount(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(filepath.Join(dir, "test-engram"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err := db.PutNode(&NodeInput{
|
||||
Content: "node content",
|
||||
Embedding: []float32{float32(i), 0.0, 0.0},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PutNode[%d]: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
n, err := db.NodeCount()
|
||||
if err != nil {
|
||||
t.Fatalf("NodeCount: %v", err)
|
||||
}
|
||||
if n != 3 {
|
||||
t.Errorf("expected 3 nodes, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecay verifies salience decay runs without error.
|
||||
func TestDecay(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(filepath.Join(dir, "test-engram"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
_, err = db.PutNode(&NodeInput{
|
||||
Content: "test node",
|
||||
Embedding: []float32{1.0, 0.0},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PutNode: %v", err)
|
||||
}
|
||||
|
||||
updated, err := db.Decay(0.95)
|
||||
if err != nil {
|
||||
t.Fatalf("Decay: %v", err)
|
||||
}
|
||||
if updated == 0 {
|
||||
t.Error("expected at least one node to be decayed")
|
||||
}
|
||||
}
|
||||
|
||||
// ensure test file exists (compile guard)
|
||||
var _ = os.DevNull
|
||||
@@ -0,0 +1,3 @@
|
||||
module github.com/neuron-technologies/engram/bindings/go
|
||||
|
||||
go 1.21
|
||||
Reference in New Issue
Block a user