From 192528543f1ceae0575f637e87b536fd5ad89d2f Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Mon, 27 Apr 2026 18:26:46 -0500 Subject: [PATCH] feat: schema projections, command transactions, quantum-secure encryption --- engram/Cargo.lock | 350 ++- engram/Cargo.toml | 3 + engram/crates/engram-crypto/Cargo.toml | 34 + engram/crates/engram-crypto/src/algorithm.rs | 87 + engram/crates/engram-crypto/src/engine.rs | 340 +++ engram/crates/engram-crypto/src/error.rs | 30 + engram/crates/engram-crypto/src/lib.rs | 43 + engram/crates/engram-crypto/src/registry.rs | 95 + engram/crates/engram-projection/Cargo.toml | 17 + engram/crates/engram-projection/src/engine.rs | 462 ++++ engram/crates/engram-projection/src/error.rs | 24 + engram/crates/engram-projection/src/lib.rs | 33 + .../crates/engram-projection/src/registry.rs | 133 + engram/crates/engram-projection/src/schema.rs | 133 + engram/crates/engram-server/Cargo.toml | 4 + engram/crates/engram-server/src/main.rs | 38 + engram/crates/engram-server/src/routes/mod.rs | 2 + .../engram-server/src/routes/projection.rs | 121 + engram/crates/engram-server/src/routes/tx.rs | 109 + engram/crates/engram-server/src/state.rs | 6 + engram/crates/engram-tx/Cargo.toml | 17 + engram/crates/engram-tx/src/command.rs | 166 ++ engram/crates/engram-tx/src/engine.rs | 710 +++++ engram/crates/engram-tx/src/error.rs | 31 + engram/crates/engram-tx/src/lib.rs | 29 + engram/crates/engram-tx/src/log.rs | 151 ++ engram/engram-explainer.html | 2296 +++++++++++++++++ 27 files changed, 5460 insertions(+), 4 deletions(-) create mode 100644 engram/crates/engram-crypto/Cargo.toml create mode 100644 engram/crates/engram-crypto/src/algorithm.rs create mode 100644 engram/crates/engram-crypto/src/engine.rs create mode 100644 engram/crates/engram-crypto/src/error.rs create mode 100644 engram/crates/engram-crypto/src/lib.rs create mode 100644 engram/crates/engram-crypto/src/registry.rs create mode 100644 engram/crates/engram-projection/Cargo.toml create mode 100644 engram/crates/engram-projection/src/engine.rs create mode 100644 engram/crates/engram-projection/src/error.rs create mode 100644 engram/crates/engram-projection/src/lib.rs create mode 100644 engram/crates/engram-projection/src/registry.rs create mode 100644 engram/crates/engram-projection/src/schema.rs create mode 100644 engram/crates/engram-server/src/routes/projection.rs create mode 100644 engram/crates/engram-server/src/routes/tx.rs create mode 100644 engram/crates/engram-tx/Cargo.toml create mode 100644 engram/crates/engram-tx/src/command.rs create mode 100644 engram/crates/engram-tx/src/engine.rs create mode 100644 engram/crates/engram-tx/src/error.rs create mode 100644 engram/crates/engram-tx/src/lib.rs create mode 100644 engram/crates/engram-tx/src/log.rs create mode 100644 engram/engram-explainer.html diff --git a/engram/Cargo.lock b/engram/Cargo.lock index 5c26306..8e19c4c 100644 --- a/engram/Cargo.lock +++ b/engram/Cargo.lock @@ -2,6 +2,41 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -29,6 +64,18 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-trait" version = "0.1.89" @@ -65,7 +112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "bytes", "futures-util", "http 1.4.0", @@ -74,7 +121,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -92,6 +139,34 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http 1.4.0", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tokio", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.5" @@ -113,6 +188,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-test" version = "14.10.0" @@ -122,7 +215,7 @@ dependencies = [ "anyhow", "async-trait", "auto-future", - "axum", + "axum 0.7.9", "bytes", "cookie", "http 1.4.0", @@ -169,6 +262,29 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -209,6 +325,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "combine" version = "4.6.7" @@ -219,6 +345,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "cookie" version = "0.18.1" @@ -255,6 +387,24 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -289,6 +439,26 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "deranged" version = "0.5.8" @@ -304,6 +474,16 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -352,6 +532,21 @@ dependencies = [ "uuid", ] +[[package]] +name = "engram-crypto" +version = "0.1.0" +dependencies = [ + "aes-gcm", + "base64", + "blake3", + "rand", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "uuid", +] + [[package]] name = "engram-ffi" version = "0.1.0" @@ -379,17 +574,36 @@ dependencies = [ "engram-core", ] +[[package]] +name = "engram-projection" +version = "0.1.0" +dependencies = [ + "base64", + "engram-core", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "uuid", +] + [[package]] name = "engram-server" version = "0.1.0" dependencies = [ "anyhow", - "axum", + "axum 0.7.9", "axum-test", "engram-core", + "engram-crypto", + "engram-projection", "engram-sync", + "engram-tx", + "mime_guess", + "rust-embed", "serde", "serde_json", + "sled", "tempfile", "thiserror 1.0.69", "tokio", @@ -415,6 +629,19 @@ dependencies = [ "uuid", ] +[[package]] +name = "engram-tx" +version = "0.1.0" +dependencies = [ + "engram-core", + "serde", + "serde_json", + "sled", + "tempfile", + "thiserror 1.0.69", + "uuid", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -557,6 +784,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -581,6 +818,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "h2" version = "0.4.13" @@ -906,6 +1153,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.13" @@ -1078,6 +1334,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.0" @@ -1168,6 +1430,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl" version = "0.10.78" @@ -1307,6 +1575,18 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1537,6 +1817,41 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "axum 0.8.9", + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2", + "walkdir", +] + [[package]] name = "rust-multipart-rfc7578_2" version = "0.6.1" @@ -1739,6 +2054,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2185,6 +2511,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + [[package]] name = "unicase" version = "2.9.0" @@ -2203,6 +2535,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/engram/Cargo.toml b/engram/Cargo.toml index 1e2a64f..6f5cee6 100644 --- a/engram/Cargo.toml +++ b/engram/Cargo.toml @@ -7,6 +7,9 @@ members = [ "crates/engram-migrate", "crates/engram-sync", "crates/engram-server", + "crates/engram-projection", + "crates/engram-tx", + "crates/engram-crypto", # engram-wasm is in bindings/ and compiled separately via wasm-pack # (wasm targets can't be in the same workspace build as native targets) ] diff --git a/engram/crates/engram-crypto/Cargo.toml b/engram/crates/engram-crypto/Cargo.toml new file mode 100644 index 0000000..fd5e8af --- /dev/null +++ b/engram/crates/engram-crypto/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "engram-crypto" +version = "0.1.0" +edition = "2021" +description = "Quantum-secure encryption at rest for Engram — AES-256-GCM with PQ upgrade path" +license = "MIT" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +uuid = { version = "1", features = ["v4", "serde"] } +thiserror = "1" +# AES-256-GCM symmetric encryption (quantum-resistant at 256-bit key length) +aes-gcm = "0.10" +# BLAKE3 for key derivation (fast, cryptographically strong) +blake3 = "1" +# Random number generation +rand = "0.8" +# Base64 encoding for serialization (used in EncryptedContent serialization) +base64 = "0.22" + +# TODO: Upgrade to post-quantum KEM/signature once crates stabilize. +# Target: ml-kem (CRYSTALS-Kyber / NIST ML-KEM) and ml-dsa (CRYSTALS-Dilithium / NIST ML-DSA). +# As of 2025, the `ml-kem` and `ml-dsa` crates are available on crates.io but not yet +# production-stable for all platforms. The algorithm registry structure below is designed +# so that the upgrade is a drop-in: add the PQ crate, implement the KemAlgorithm variant, +# and new writes use the new algorithm while old records continue to decrypt via the registry. +# +# Uncomment when ready: +# ml-kem = "0.2" # CRYSTALS-Kyber (NIST ML-KEM 768/1024) +# ml-dsa = "0.1" # CRYSTALS-Dilithium (NIST ML-DSA) + +[dev-dependencies] +tempfile = "3" diff --git a/engram/crates/engram-crypto/src/algorithm.rs b/engram/crates/engram-crypto/src/algorithm.rs new file mode 100644 index 0000000..cb761ba --- /dev/null +++ b/engram/crates/engram-crypto/src/algorithm.rs @@ -0,0 +1,87 @@ +/// Algorithm registry types — the versioning layer for crypto algorithm rotation. +/// +/// Each encrypted record carries an `algorithm_id`. The registry maps these IDs +/// to the parameters needed to decrypt. When you rotate algorithms, old records +/// keep their ID and decrypt using the historical version. New records use the +/// new active algorithm. +use serde::{Deserialize, Serialize}; + +/// Key Encapsulation Mechanism algorithms. +/// +/// # Current +/// - `Aes256GcmDirect`: AES-256-GCM with a directly-provided 256-bit key. +/// Quantum-resistant at 256-bit (Grover halves to 128-bit effective security). +/// +/// # Planned (post-quantum upgrade) +/// - `MlKem768`: CRYSTALS-Kyber 768 (NIST ML-KEM Level 3 — 128-bit PQ security) +/// - `MlKem1024`: CRYSTALS-Kyber 1024 (NIST ML-KEM Level 5 — 256-bit PQ security) +/// - `ClassicRsa4096`: RSA-4096 OAEP fallback (NOT quantum-resistant — for compat only) +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum KemAlgorithm { + /// AES-256-GCM with direct key (current default, quantum-resistant at 256-bit) + Aes256GcmDirect, + // TODO: uncomment when ml-kem crate stabilizes + // MlKem768, + // MlKem1024, + // ClassicRsa4096, +} + +impl KemAlgorithm { + pub fn id(&self) -> &'static str { + match self { + KemAlgorithm::Aes256GcmDirect => "aes256gcm-direct-v1", + } + } +} + +/// Signature algorithms for authenticating ciphertext. +/// +/// # Current +/// - `Blake3Mac`: BLAKE3 keyed hash as a MAC (message authentication code). +/// Not a signature in the asymmetric sense, but provides authenticity. +/// +/// # Planned (post-quantum upgrade) +/// - `MlDsa44` / `MlDsa65` / `MlDsa87`: CRYSTALS-Dilithium (NIST ML-DSA) +/// - `SphincsSha256128f`: SPHINCS+ stateless hash-based signature +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SigAlgorithm { + /// BLAKE3 keyed MAC (current default) + Blake3Mac, + // TODO: uncomment when ml-dsa crate stabilizes + // MlDsa44, // NIST Security Level 2 (128-bit) + // MlDsa65, // NIST Security Level 3 (192-bit) + // MlDsa87, // NIST Security Level 5 (256-bit) + // SphincsSha256128f, // Stateless hash-based, conservative security +} + +impl SigAlgorithm { + pub fn id(&self) -> &'static str { + match self { + SigAlgorithm::Blake3Mac => "blake3-mac-v1", + } + } +} + +/// A versioned algorithm configuration entry. +/// +/// Historical versions are kept in the registry so that old ciphertexts can +/// always be decrypted even after algorithm rotation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AlgorithmVersion { + /// Unique string ID stored alongside every ciphertext. + pub id: String, + /// The KEM algorithm used for key derivation/encapsulation. + pub kem: KemAlgorithm, + /// The signature algorithm used for ciphertext authentication. + pub sig: SigAlgorithm, + /// Unix milliseconds when this version became active. + pub activated_at: i64, + /// Unix milliseconds when this version was superseded (None = still active). + pub retired_at: Option, +} + +impl AlgorithmVersion { + pub fn is_active(&self) -> bool { + self.retired_at.is_none() + } +} diff --git a/engram/crates/engram-crypto/src/engine.rs b/engram/crates/engram-crypto/src/engine.rs new file mode 100644 index 0000000..bc65f09 --- /dev/null +++ b/engram/crates/engram-crypto/src/engine.rs @@ -0,0 +1,340 @@ +/// CryptoEngine — encrypt and decrypt node content with algorithm versioning. +/// +/// # Security Model +/// +/// - **Symmetric encryption**: AES-256-GCM (authenticated encryption, quantum-resistant) +/// - **Key derivation**: BLAKE3 KDF — stretches the master key into per-operation keys +/// - **Authentication**: BLAKE3 keyed MAC over (algorithm_id || nonce || ciphertext) +/// - **Nonces**: 96-bit random nonce per encryption (from OS CSPRNG via `rand`) +/// +/// # AES-256-GCM and Quantum Resistance +/// +/// AES-256 is considered quantum-resistant: Grover's algorithm provides at most +/// a quadratic speedup, reducing 256-bit security to ~128-bit effective security +/// against quantum adversaries. 128-bit quantum security is currently considered +/// sufficient. The algorithm_id in EncryptedContent ensures an upgrade to +/// ML-KEM/Kyber is a transparent drop-in when the crates stabilize. +use aes_gcm::{ + aead::{Aead, AeadCore, KeyInit, OsRng}, + Aes256Gcm, Key, Nonce, +}; +use serde::{Deserialize, Serialize}; + +use crate::error::{CryptoError, CryptoResult}; +use crate::registry::AlgorithmRegistry; + +/// An encrypted content blob, self-describing with its algorithm version. +/// +/// The `algorithm_id` field allows any version to decrypt any record, +/// even after algorithm rotation. This is the key to migration-free upgrades. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EncryptedContent { + /// Which algorithm version encrypted this record. + /// Maps to an entry in `AlgorithmRegistry::versions`. + pub algorithm_id: String, + /// The AES-256-GCM ciphertext (includes the GCM auth tag). + pub ciphertext: Vec, + /// In a PQ scheme: the KEM-encapsulated symmetric key. + /// In the current AES-direct scheme: empty (key is derived from master key + context). + pub encapsulated_key: Vec, + /// 96-bit random AES-GCM nonce. + pub nonce: Vec, + /// BLAKE3 MAC over (algorithm_id || nonce || ciphertext). + /// In a PQ scheme: this would be a Dilithium/ML-DSA signature. + pub signature: Vec, +} + +impl EncryptedContent { + /// Serialize to a compact JSON string for storage. + pub fn to_bytes(&self) -> CryptoResult> { + serde_json::to_vec(self).map_err(|e| CryptoError::Serialization(e.to_string())) + } + + /// Deserialize from bytes (JSON). + pub fn from_bytes(bytes: &[u8]) -> CryptoResult { + serde_json::from_slice(bytes).map_err(|e| CryptoError::DecryptionFailed(e.to_string())) + } +} + +/// The encryption engine. +/// +/// One engine instance per process (or per-request for stateless usage). +/// The engine holds the master key and the algorithm registry. +pub struct CryptoEngine { + /// Master key bytes — 32 bytes for AES-256. + /// In a PQ scheme: this would be a keypair (public/private). + master_key: [u8; 32], + /// Algorithm registry — tracks active and historical versions. + pub registry: AlgorithmRegistry, +} + +impl CryptoEngine { + /// Create an engine from a 32-byte master key (AES-256 requires 256-bit key). + pub fn from_key(key: &[u8]) -> CryptoResult { + if key.len() < 32 { + return Err(CryptoError::InvalidKeyLength { + expected: 32, + got: key.len(), + }); + } + let mut master_key = [0u8; 32]; + master_key.copy_from_slice(&key[..32]); + Ok(Self { + master_key, + registry: AlgorithmRegistry::default_registry(), + }) + } + + /// Create an engine from an environment variable `ENGRAM_ENCRYPTION_KEY`. + /// + /// Returns `None` if the variable is not set (dev mode — plaintext storage). + /// Returns an error if the variable is set but the key is too short. + pub fn from_env() -> CryptoResult> { + match std::env::var("ENGRAM_ENCRYPTION_KEY") { + Ok(key_str) => { + let key_bytes = key_str.as_bytes(); + // Derive a 32-byte key from whatever the user provided + let derived = derive_key(key_bytes, b"engram-master-key"); + Ok(Some(Self::from_key(&derived)?)) + } + Err(std::env::VarError::NotPresent) => Ok(None), + Err(e) => Err(CryptoError::KeyDerivation(e.to_string())), + } + } + + // ── Encrypt ─────────────────────────────────────────────────────────────── + + /// Encrypt `plaintext` using the active algorithm. + /// + /// Returns an `EncryptedContent` that is self-describing: it carries its + /// `algorithm_id`, so decryption never needs out-of-band version tracking. + pub fn encrypt(&self, plaintext: &[u8]) -> CryptoResult { + let algorithm_id = self.registry.active_id().to_string(); + + // Derive a per-operation encryption key from the master key + let enc_key_bytes = derive_key(&self.master_key, b"encrypt"); + let key = Key::::from_slice(&enc_key_bytes); + let cipher = Aes256Gcm::new(key); + + // Random 96-bit nonce + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + + // Encrypt + let ciphertext = cipher + .encrypt(&nonce, plaintext) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + + // MAC: BLAKE3 keyed hash over (algorithm_id || nonce || ciphertext) + let mac_key = derive_key(&self.master_key, b"mac"); + let signature = compute_mac(&mac_key, &algorithm_id, nonce.as_slice(), &ciphertext); + + Ok(EncryptedContent { + algorithm_id, + ciphertext, + encapsulated_key: vec![], // unused in AES-direct mode + nonce: nonce.to_vec(), + signature, + }) + } + + // ── Decrypt ─────────────────────────────────────────────────────────────── + + /// Decrypt an `EncryptedContent`, dispatching to the correct algorithm version. + pub fn decrypt(&self, content: &EncryptedContent) -> CryptoResult> { + // Look up the algorithm version that produced this ciphertext + let _version = self.registry.get_version(&content.algorithm_id)?; + + // Verify MAC before decrypting (fail-fast on tampering) + let mac_key = derive_key(&self.master_key, b"mac"); + let expected = compute_mac(&mac_key, &content.algorithm_id, &content.nonce, &content.ciphertext); + if expected != content.signature { + return Err(CryptoError::SignatureInvalid); + } + + // Decrypt + let enc_key_bytes = derive_key(&self.master_key, b"encrypt"); + let key = Key::::from_slice(&enc_key_bytes); + let cipher = Aes256Gcm::new(key); + + if content.nonce.len() != 12 { + return Err(CryptoError::DecryptionFailed(format!( + "invalid nonce length: {}", + content.nonce.len() + ))); + } + let nonce = Nonce::from_slice(&content.nonce); + + let plaintext = cipher + .decrypt(nonce, content.ciphertext.as_slice()) + .map_err(|e| CryptoError::DecryptionFailed(e.to_string()))?; + + Ok(plaintext) + } + + // ── Signature verification ───────────────────────────────────────────────── + + /// Verify the MAC/signature on an encrypted content blob. + pub fn verify_signature(&self, content: &EncryptedContent) -> CryptoResult { + let mac_key = derive_key(&self.master_key, b"mac"); + let expected = compute_mac(&mac_key, &content.algorithm_id, &content.nonce, &content.ciphertext); + Ok(expected == content.signature) + } + + // ── Algorithm rotation ──────────────────────────────────────────────────── + + /// Rotate to a new KEM algorithm. + /// + /// After rotation, new encryptions use the new algorithm. + /// Old records retain their `algorithm_id` and decrypt via the historical registry. + pub fn rotate_algorithm(&mut self, new_kem: crate::algorithm::KemAlgorithm) -> CryptoResult<()> { + self.registry.rotate_kem(new_kem) + } +} + +// ── Key derivation ──────────────────────────────────────────────────────────── + +/// Derive a 32-byte sub-key from the master key and a context string. +/// Uses BLAKE3's keyed hash for domain separation. +fn derive_key(master: &[u8], context: &[u8]) -> [u8; 32] { + // BLAKE3 derive_key: master is the key material, context is the domain + let mut hasher = blake3::Hasher::new_keyed( + &padded_32(master), + ); + hasher.update(context); + let hash = hasher.finalize(); + *hash.as_bytes() +} + +/// Pad or truncate bytes to exactly 32 bytes. +fn padded_32(bytes: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + let len = bytes.len().min(32); + out[..len].copy_from_slice(&bytes[..len]); + out +} + +/// Compute a BLAKE3 keyed MAC over (algorithm_id || nonce || ciphertext). +fn compute_mac(mac_key: &[u8; 32], algorithm_id: &str, nonce: &[u8], ciphertext: &[u8]) -> Vec { + let mut hasher = blake3::Hasher::new_keyed(mac_key); + hasher.update(algorithm_id.as_bytes()); + hasher.update(nonce); + hasher.update(ciphertext); + hasher.finalize().as_bytes().to_vec() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_engine() -> CryptoEngine { + let key = b"test-master-key-must-be-32-bytes"; + CryptoEngine::from_key(key).unwrap() + } + + #[test] + fn test_encrypt_decrypt_roundtrip() { + let engine = make_engine(); + let plaintext = b"sensitive memory content - do not store unencrypted"; + let enc = engine.encrypt(plaintext).unwrap(); + let dec = engine.decrypt(&enc).unwrap(); + assert_eq!(dec, plaintext); + } + + #[test] + fn test_nonce_is_random() { + let engine = make_engine(); + let enc1 = engine.encrypt(b"same plaintext").unwrap(); + let enc2 = engine.encrypt(b"same plaintext").unwrap(); + // Nonces must differ (probabilistic — collision probability 1/2^96) + assert_ne!(enc1.nonce, enc2.nonce); + // Ciphertexts must differ (different nonces → different ciphertexts) + assert_ne!(enc1.ciphertext, enc2.ciphertext); + } + + #[test] + fn test_tampered_ciphertext_rejected() { + let engine = make_engine(); + let mut enc = engine.encrypt(b"original").unwrap(); + // Flip a byte in the ciphertext + if let Some(b) = enc.ciphertext.first_mut() { + *b ^= 0xFF; + } + // Decryption should fail due to GCM auth tag verification + let result = engine.decrypt(&enc); + assert!(result.is_err()); + } + + #[test] + fn test_tampered_mac_rejected() { + let engine = make_engine(); + let mut enc = engine.encrypt(b"original").unwrap(); + // Flip a byte in the signature + if let Some(b) = enc.signature.first_mut() { + *b ^= 0xFF; + } + let result = engine.decrypt(&enc); + assert!(result.is_err()); + } + + #[test] + fn test_verify_signature() { + let engine = make_engine(); + let enc = engine.encrypt(b"content").unwrap(); + assert!(engine.verify_signature(&enc).unwrap()); + + let mut tampered = enc.clone(); + tampered.signature[0] ^= 0x01; + assert!(!engine.verify_signature(&tampered).unwrap()); + } + + #[test] + fn test_algorithm_id_stored() { + let engine = make_engine(); + let enc = engine.encrypt(b"data").unwrap(); + assert_eq!(enc.algorithm_id, "aes256gcm-direct-v1"); + } + + #[test] + fn test_serialization_roundtrip() { + let engine = make_engine(); + let enc = engine.encrypt(b"serialize me").unwrap(); + let bytes = enc.to_bytes().unwrap(); + let restored = EncryptedContent::from_bytes(&bytes).unwrap(); + let dec = engine.decrypt(&restored).unwrap(); + assert_eq!(dec, b"serialize me"); + } + + #[test] + fn test_short_key_rejected() { + let short_key = b"too-short"; + let result = CryptoEngine::from_key(short_key); + assert!(result.is_err()); + } + + #[test] + fn test_empty_plaintext() { + let engine = make_engine(); + let enc = engine.encrypt(b"").unwrap(); + let dec = engine.decrypt(&enc).unwrap(); + assert_eq!(dec, b""); + } + + #[test] + fn test_large_plaintext() { + let engine = make_engine(); + let plaintext = vec![0xABu8; 1_000_000]; // 1 MB + let enc = engine.encrypt(&plaintext).unwrap(); + let dec = engine.decrypt(&enc).unwrap(); + assert_eq!(dec, plaintext); + } + + #[test] + fn test_unknown_algorithm_rejected() { + let engine = make_engine(); + let mut enc = engine.encrypt(b"data").unwrap(); + enc.algorithm_id = "unknown-algo-v99".to_string(); + let result = engine.decrypt(&enc); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), CryptoError::UnknownAlgorithm(_))); + } +} diff --git a/engram/crates/engram-crypto/src/error.rs b/engram/crates/engram-crypto/src/error.rs new file mode 100644 index 0000000..bc11de6 --- /dev/null +++ b/engram/crates/engram-crypto/src/error.rs @@ -0,0 +1,30 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum CryptoError { + #[error("Encryption failed: {0}")] + EncryptionFailed(String), + + #[error("Decryption failed: {0}")] + DecryptionFailed(String), + + #[error("Signature verification failed")] + SignatureInvalid, + + #[error("Unknown algorithm ID: {0}")] + UnknownAlgorithm(String), + + #[error("Key derivation failed: {0}")] + KeyDerivation(String), + + #[error("Invalid key length: expected {expected}, got {got}")] + InvalidKeyLength { expected: usize, got: usize }, + + #[error("Algorithm not available: {0}")] + AlgorithmUnavailable(String), + + #[error("Serialization error: {0}")] + Serialization(String), +} + +pub type CryptoResult = Result; diff --git a/engram/crates/engram-crypto/src/lib.rs b/engram/crates/engram-crypto/src/lib.rs new file mode 100644 index 0000000..f61f723 --- /dev/null +++ b/engram/crates/engram-crypto/src/lib.rs @@ -0,0 +1,43 @@ +/// Engram Crypto — quantum-secure encryption at rest. +/// +/// # Current Implementation +/// +/// Uses AES-256-GCM for symmetric encryption with BLAKE3 for key derivation. +/// AES-256 is already quantum-resistant (Grover's algorithm halves the key space +/// from 2^256 to 2^128, which remains computationally infeasible). +/// +/// # Post-Quantum Upgrade Path +/// +/// The `AlgorithmRegistry` stores an `algorithm_id` alongside every ciphertext. +/// When ML-KEM (CRYSTALS-Kyber) and ML-DSA (CRYSTALS-Dilithium) crates stabilize, +/// the upgrade is: +/// 1. Add `KemAlgorithm::MlKem768` / `MlKem1024` variants +/// 2. Implement `CryptoEngine::encrypt()` for the new algorithm +/// 3. Set it as the active algorithm in the registry +/// 4. Old records continue to decrypt via their stored `algorithm_id` +/// 5. Background re-encryption rotates old records to the new algorithm +/// +/// No data migration required — the registry handles version negotiation. +/// +/// # Usage +/// +/// ```rust,no_run +/// use engram_crypto::{CryptoEngine, AlgorithmRegistry}; +/// +/// let key = b"an-example-32-byte-key!!12345678"; +/// let engine = CryptoEngine::from_key(key).unwrap(); +/// +/// let plaintext = b"sensitive memory content"; +/// let encrypted = engine.encrypt(plaintext).unwrap(); +/// let decrypted = engine.decrypt(&encrypted).unwrap(); +/// assert_eq!(plaintext, decrypted.as_slice()); +/// ``` +pub mod algorithm; +pub mod engine; +pub mod error; +pub mod registry; + +pub use algorithm::{AlgorithmVersion, KemAlgorithm, SigAlgorithm}; +pub use engine::{CryptoEngine, EncryptedContent}; +pub use error::CryptoError; +pub use registry::AlgorithmRegistry; diff --git a/engram/crates/engram-crypto/src/registry.rs b/engram/crates/engram-crypto/src/registry.rs new file mode 100644 index 0000000..d007b17 --- /dev/null +++ b/engram/crates/engram-crypto/src/registry.rs @@ -0,0 +1,95 @@ +/// Algorithm registry — tracks active and historical algorithm versions. +use std::collections::HashMap; + +use crate::algorithm::{AlgorithmVersion, KemAlgorithm, SigAlgorithm}; +use crate::error::{CryptoError, CryptoResult}; + +/// Maintains the set of algorithm versions known to this node. +/// +/// The active version is used for all new encryptions. +/// Historical versions remain so old ciphertexts can always be decrypted. +pub struct AlgorithmRegistry { + /// The currently active algorithm version. + pub active_kem: KemAlgorithm, + pub active_sig: SigAlgorithm, + /// All known versions (active + historical), keyed by algorithm ID. + pub versions: HashMap, +} + +impl AlgorithmRegistry { + /// Create a registry with the default algorithm (AES-256-GCM + BLAKE3 MAC). + pub fn default_registry() -> Self { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64; + + let default_version = AlgorithmVersion { + id: "aes256gcm-direct-v1".to_string(), + kem: KemAlgorithm::Aes256GcmDirect, + sig: SigAlgorithm::Blake3Mac, + activated_at: now, + retired_at: None, + }; + + let mut versions = HashMap::new(); + versions.insert(default_version.id.clone(), default_version); + + Self { + active_kem: KemAlgorithm::Aes256GcmDirect, + active_sig: SigAlgorithm::Blake3Mac, + versions, + } + } + + /// Get the active algorithm ID (used as the `algorithm_id` in new ciphertexts). + pub fn active_id(&self) -> &str { + self.active_kem.id() + } + + /// Look up a version by its ID (for decryption of historical records). + pub fn get_version(&self, id: &str) -> CryptoResult<&AlgorithmVersion> { + self.versions + .get(id) + .ok_or_else(|| CryptoError::UnknownAlgorithm(id.to_string())) + } + + /// Rotate to a new KEM algorithm. + /// + /// The current active version is marked as retired. A new version entry is + /// added and becomes active. Old records retain their algorithm_id and can + /// still be decrypted via `get_version()`. + /// + /// Background re-encryption can then update old records at leisure. + pub fn rotate_kem(&mut self, new_kem: KemAlgorithm) -> CryptoResult<()> { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64; + + // Retire current active version + if let Some(current) = self.versions.get_mut(self.active_kem.id()) { + current.retired_at = Some(now); + } + + let new_id = new_kem.id().to_string(); + let new_version = AlgorithmVersion { + id: new_id.clone(), + kem: new_kem.clone(), + sig: self.active_sig.clone(), + activated_at: now, + retired_at: None, + }; + + self.versions.insert(new_id, new_version); + self.active_kem = new_kem; + Ok(()) + } + + /// List all versions, active and historical. + pub fn list_versions(&self) -> Vec<&AlgorithmVersion> { + let mut vs: Vec<&AlgorithmVersion> = self.versions.values().collect(); + vs.sort_by_key(|v| v.activated_at); + vs + } +} diff --git a/engram/crates/engram-projection/Cargo.toml b/engram/crates/engram-projection/Cargo.toml new file mode 100644 index 0000000..cea9859 --- /dev/null +++ b/engram/crates/engram-projection/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "engram-projection" +version = "0.1.0" +edition = "2021" +description = "Schema/projection layer for Engram — schema-free views over the activation surface" +license = "MIT" + +[dependencies] +engram-core = { path = "../engram-core" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +uuid = { version = "1", features = ["v4", "serde"] } +thiserror = "1" +base64 = "0.22" + +[dev-dependencies] +tempfile = "3" diff --git a/engram/crates/engram-projection/src/engine.rs b/engram/crates/engram-projection/src/engine.rs new file mode 100644 index 0000000..dd91d91 --- /dev/null +++ b/engram/crates/engram-projection/src/engine.rs @@ -0,0 +1,462 @@ +/// Projection engine — maps activated nodes through a ProjectionSchema. +/// +/// The engine is stateless: it takes a schema and a result set, and returns +/// the projected view. No mutation of the underlying graph occurs. +use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; +use engram_core::types::{ActivatedNode, NodeType}; +use serde_json::{json, Value}; +use std::collections::HashMap; +use uuid::Uuid; + +use crate::error::ProjectionResult; +use crate::schema::{ + FieldMapping, FieldSource, NodeFilter, ProjectedRow, ProjectionResult as PResult, + ProjectionSchema, ProjectionType, +}; + +/// Stateless projection executor. +pub struct ProjectionEngine; + +impl ProjectionEngine { + /// Apply a projection schema to a set of activated nodes. + /// + /// Returns a `ProjectionResult` containing the shaped output. + /// The activation result set is not modified. + pub fn project( + schema: &ProjectionSchema, + activated: &[ActivatedNode], + ) -> ProjectionResult { + // Step 1: filter to in-scope nodes + let in_scope: Vec<&ActivatedNode> = activated + .iter() + .filter(|a| matches_filter(a, &schema.node_filter)) + .collect(); + + let nodes_in_scope = in_scope.len(); + + match schema.projection_type { + ProjectionType::Relational | ProjectionType::Document => { + let rows = in_scope + .iter() + .map(|a| project_row(a, &schema.field_mappings)) + .collect::>>()?; + + Ok(PResult { + schema_name: schema.name.clone(), + nodes_in_scope, + rows, + key_value: None, + wide_column: None, + }) + } + + ProjectionType::KeyValue => { + let mut kv: HashMap = HashMap::new(); + for a in &in_scope { + let key = a.node.id.to_string(); + let val = String::from_utf8_lossy(&a.node.content).to_string(); + kv.insert(key, Value::String(val)); + } + Ok(PResult { + schema_name: schema.name.clone(), + nodes_in_scope, + rows: vec![], + key_value: Some(kv), + wide_column: None, + }) + } + + ProjectionType::WideColumn => { + let mut wc: HashMap> = HashMap::new(); + for a in &in_scope { + let id = a.node.id.to_string(); + let mut cols = HashMap::new(); + for mapping in &schema.field_mappings { + let val = extract_field(a, &mapping.source) + .unwrap_or_else(|| mapping.default.clone().unwrap_or(Value::Null)); + cols.insert(mapping.field_name.clone(), val); + } + wc.insert(id, cols); + } + Ok(PResult { + schema_name: schema.name.clone(), + nodes_in_scope, + rows: vec![], + key_value: None, + wide_column: Some(wc), + }) + } + } + } +} + +// ── Node filter evaluation ──────────────────────────────────────────────────── + +fn matches_filter(a: &ActivatedNode, filter: &NodeFilter) -> bool { + match filter { + NodeFilter::All => true, + + NodeFilter::ByType(types) => { + let node_type_str = node_type_str(&a.node.node_type); + types.iter().any(|t| t == node_type_str) + } + + NodeFilter::ByTier(tiers) => tiers.iter().any(|t| t == &a.node.tier), + + NodeFilter::ByTag(tags) => { + // Tags are searched in content (treated as UTF-8) as a simple substring match. + // This is intentionally lenient — callers may embed tag metadata in content. + let content_str = String::from_utf8_lossy(&a.node.content); + tags.iter().any(|tag| content_str.contains(tag.as_str())) + } + + NodeFilter::ByActivationThreshold(threshold) => a.activation_strength >= *threshold, + + NodeFilter::BySalience(threshold) => a.node.salience >= *threshold, + + NodeFilter::Combined(filters) => filters.iter().all(|f| matches_filter(a, f)), + + NodeFilter::Any(filters) => filters.iter().any(|f| matches_filter(a, f)), + } +} + +// ── Row projection ──────────────────────────────────────────────────────────── + +fn project_row(a: &ActivatedNode, mappings: &[FieldMapping]) -> ProjectionResult { + let mut fields = HashMap::new(); + for mapping in mappings { + let val = extract_field(a, &mapping.source) + .unwrap_or_else(|| mapping.default.clone().unwrap_or(Value::Null)); + fields.insert(mapping.field_name.clone(), val); + } + Ok(ProjectedRow { + node_id: a.node.id, + fields, + }) +} + +// ── Field extraction ────────────────────────────────────────────────────────── + +fn extract_field(a: &ActivatedNode, source: &FieldSource) -> Option { + match source { + FieldSource::NodeId => Some(Value::String(a.node.id.to_string())), + + FieldSource::NodeType => Some(Value::String(node_type_str(&a.node.node_type).to_string())), + + FieldSource::Tier => Some(Value::String(tier_str(&a.node.tier).to_string())), + + FieldSource::Salience => Some(json!(a.node.salience)), + + FieldSource::Importance => Some(json!(a.node.importance)), + + FieldSource::ActivationStrength => Some(json!(a.activation_strength)), + + FieldSource::Hops => Some(json!(a.hops)), + + FieldSource::CreatedAt => Some(json!(a.node.created_at)), + + FieldSource::LastActivated => Some(json!(a.node.last_activated)), + + FieldSource::ActivationCount => Some(json!(a.node.activation_count)), + + FieldSource::ContentRaw => { + Some(Value::String(String::from_utf8_lossy(&a.node.content).to_string())) + } + + FieldSource::ContentBase64 => Some(Value::String(B64.encode(&a.node.content))), + + FieldSource::ContentJsonPath(path) => { + // Parse content as JSON, then traverse the dot-path + let content_str = std::str::from_utf8(&a.node.content).ok()?; + let doc: Value = serde_json::from_str(content_str).ok()?; + traverse_json_path(&doc, path).cloned() + } + + FieldSource::Literal(v) => Some(v.clone()), + } +} + +/// Traverse a dot-separated JSON path. +/// E.g., "user.name" on `{"user": {"name": "Alice"}}` returns `"Alice"`. +fn traverse_json_path<'a>(doc: &'a Value, path: &str) -> Option<&'a Value> { + let mut current = doc; + for segment in path.split('.') { + current = match current { + Value::Object(map) => map.get(segment)?, + Value::Array(arr) => { + let idx: usize = segment.parse().ok()?; + arr.get(idx)? + } + _ => return None, + }; + } + Some(current) +} + +// ── String helpers ──────────────────────────────────────────────────────────── + +fn node_type_str(t: &NodeType) -> &'static str { + match t { + NodeType::Memory => "Memory", + NodeType::Concept => "Concept", + NodeType::Event => "Event", + NodeType::Entity => "Entity", + NodeType::Process => "Process", + NodeType::InternalState => "InternalState", + } +} + +fn tier_str(t: &engram_core::types::MemoryTier) -> &'static str { + use engram_core::types::MemoryTier; + match t { + MemoryTier::Working => "Working", + MemoryTier::Episodic => "Episodic", + MemoryTier::Semantic => "Semantic", + MemoryTier::Procedural => "Procedural", + } +} + +/// Extract node_id from a projected row (used for display / keying). +pub fn row_id(row: &ProjectedRow) -> Uuid { + row.node_id +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::{FieldMapping, FieldSource, NodeFilter, ProjectionSchema, ProjectionType}; + use engram_core::types::{ActivatedNode, MemoryTier, Node, NodeType}; + + fn make_node(content: &str, tier: MemoryTier, importance: f32) -> Node { + Node::new( + NodeType::Memory, + vec![1.0, 0.0], + content.as_bytes().to_vec(), + tier, + importance, + ) + } + + fn make_activated(node: Node, strength: f32) -> ActivatedNode { + ActivatedNode { + node, + activation_strength: strength, + hops: 1, + } + } + + #[test] + fn test_relational_projection_basic() { + let node = make_node("hello world", MemoryTier::Semantic, 0.8); + let activated = vec![make_activated(node, 0.9)]; + + let schema = ProjectionSchema { + name: "test".into(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: NodeFilter::All, + field_mappings: vec![ + FieldMapping { + field_name: "content".into(), + source: FieldSource::ContentRaw, + default: None, + }, + FieldMapping { + field_name: "tier".into(), + source: FieldSource::Tier, + default: None, + }, + FieldMapping { + field_name: "strength".into(), + source: FieldSource::ActivationStrength, + default: None, + }, + ], + }; + + let result = ProjectionEngine::project(&schema, &activated).unwrap(); + assert_eq!(result.nodes_in_scope, 1); + assert_eq!(result.rows.len(), 1); + assert_eq!(result.rows[0].fields["content"], Value::String("hello world".into())); + assert_eq!(result.rows[0].fields["tier"], Value::String("Semantic".into())); + } + + #[test] + fn test_key_value_projection() { + let node = make_node("test content", MemoryTier::Working, 0.5); + let activated = vec![make_activated(node, 0.7)]; + + let schema = ProjectionSchema { + name: "kv".into(), + description: None, + projection_type: ProjectionType::KeyValue, + node_filter: NodeFilter::All, + field_mappings: vec![], + }; + + let result = ProjectionEngine::project(&schema, &activated).unwrap(); + assert_eq!(result.nodes_in_scope, 1); + let kv = result.key_value.unwrap(); + assert_eq!(kv.len(), 1); + let content = kv.values().next().unwrap(); + assert_eq!(content, &Value::String("test content".into())); + } + + #[test] + fn test_filter_by_activation_threshold() { + let n1 = make_activated(make_node("high", MemoryTier::Semantic, 0.9), 0.8); + let n2 = make_activated(make_node("low", MemoryTier::Episodic, 0.3), 0.1); + + let schema = ProjectionSchema { + name: "filtered".into(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: NodeFilter::ByActivationThreshold(0.5), + field_mappings: vec![FieldMapping { + field_name: "content".into(), + source: FieldSource::ContentRaw, + default: None, + }], + }; + + let result = ProjectionEngine::project(&schema, &[n1, n2]).unwrap(); + assert_eq!(result.nodes_in_scope, 1); + assert_eq!(result.rows[0].fields["content"], Value::String("high".into())); + } + + #[test] + fn test_filter_by_tier() { + let n1 = make_activated(make_node("semantic", MemoryTier::Semantic, 0.9), 0.5); + let n2 = make_activated(make_node("working", MemoryTier::Working, 0.5), 0.5); + + let schema = ProjectionSchema { + name: "tier_filter".into(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: NodeFilter::ByTier(vec![MemoryTier::Semantic]), + field_mappings: vec![FieldMapping { + field_name: "content".into(), + source: FieldSource::ContentRaw, + default: None, + }], + }; + + let result = ProjectionEngine::project(&schema, &[n1, n2]).unwrap(); + assert_eq!(result.nodes_in_scope, 1); + assert_eq!(result.rows[0].fields["content"], Value::String("semantic".into())); + } + + #[test] + fn test_json_path_extraction() { + let content = r#"{"user": {"name": "Alice", "age": 30}}"#; + let node = make_node(content, MemoryTier::Semantic, 0.8); + let activated = vec![make_activated(node, 0.9)]; + + let schema = ProjectionSchema { + name: "json_path".into(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: NodeFilter::All, + field_mappings: vec![ + FieldMapping { + field_name: "name".into(), + source: FieldSource::ContentJsonPath("user.name".into()), + default: None, + }, + FieldMapping { + field_name: "age".into(), + source: FieldSource::ContentJsonPath("user.age".into()), + default: None, + }, + ], + }; + + let result = ProjectionEngine::project(&schema, &activated).unwrap(); + assert_eq!(result.rows[0].fields["name"], Value::String("Alice".into())); + assert_eq!(result.rows[0].fields["age"], json!(30)); + } + + #[test] + fn test_wide_column_projection() { + let n1 = make_activated(make_node("col_content", MemoryTier::Procedural, 0.6), 0.5); + + let schema = ProjectionSchema { + name: "wide".into(), + description: None, + projection_type: ProjectionType::WideColumn, + node_filter: NodeFilter::All, + field_mappings: vec![ + FieldMapping { + field_name: "raw".into(), + source: FieldSource::ContentRaw, + default: None, + }, + FieldMapping { + field_name: "tier".into(), + source: FieldSource::Tier, + default: None, + }, + ], + }; + + let result = ProjectionEngine::project(&schema, &[n1]).unwrap(); + assert_eq!(result.nodes_in_scope, 1); + let wc = result.wide_column.unwrap(); + assert_eq!(wc.len(), 1); + let cols = wc.values().next().unwrap(); + assert_eq!(cols["raw"], Value::String("col_content".into())); + assert_eq!(cols["tier"], Value::String("Procedural".into())); + } + + #[test] + fn test_combined_filter() { + let n1 = make_activated(make_node("tag:important semantic", MemoryTier::Semantic, 0.9), 0.8); + let n2 = make_activated(make_node("no tag", MemoryTier::Semantic, 0.9), 0.8); + let n3 = make_activated(make_node("tag:important working", MemoryTier::Working, 0.3), 0.8); + + let filter = NodeFilter::Combined(vec![ + NodeFilter::ByTier(vec![MemoryTier::Semantic]), + NodeFilter::ByTag(vec!["tag:important".into()]), + ]); + + let schema = ProjectionSchema { + name: "combined".into(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: filter, + field_mappings: vec![FieldMapping { + field_name: "content".into(), + source: FieldSource::ContentRaw, + default: None, + }], + }; + + let result = ProjectionEngine::project(&schema, &[n1, n2, n3]).unwrap(); + assert_eq!(result.nodes_in_scope, 1); + assert_eq!( + result.rows[0].fields["content"], + Value::String("tag:important semantic".into()) + ); + } + + #[test] + fn test_literal_field() { + let node = make_node("any", MemoryTier::Working, 0.5); + let activated = vec![make_activated(node, 0.5)]; + + let schema = ProjectionSchema { + name: "literal".into(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: NodeFilter::All, + field_mappings: vec![FieldMapping { + field_name: "schema_version".into(), + source: FieldSource::Literal(json!("v1")), + default: None, + }], + }; + + let result = ProjectionEngine::project(&schema, &activated).unwrap(); + assert_eq!(result.rows[0].fields["schema_version"], Value::String("v1".into())); + } +} diff --git a/engram/crates/engram-projection/src/error.rs b/engram/crates/engram-projection/src/error.rs new file mode 100644 index 0000000..7527ac7 --- /dev/null +++ b/engram/crates/engram-projection/src/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ProjectionError { + #[error("Projection not found: {0}")] + NotFound(String), + + #[error("Projection already exists: {0}")] + AlreadyExists(String), + + #[error("Field mapping error: {0}")] + FieldMapping(String), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("Engram error: {0}")] + Engram(#[from] engram_core::EngramError), + + #[error("Invalid projection schema: {0}")] + InvalidSchema(String), +} + +pub type ProjectionResult = Result; diff --git a/engram/crates/engram-projection/src/lib.rs b/engram/crates/engram-projection/src/lib.rs new file mode 100644 index 0000000..a9d311b --- /dev/null +++ b/engram/crates/engram-projection/src/lib.rs @@ -0,0 +1,33 @@ +/// Engram Projection Layer — schema-as-a-view over the activation surface. +/// +/// # The Core Insight +/// +/// Engram has no schema. A node has: embedding (semantic identity), content +/// (arbitrary bytes), metadata via tier/type, and salience. Schema is a +/// *projection* — a view imposed on the activation surface at query time. +/// +/// The same Engram graph can surface as relational rows, JSON documents, +/// wide-column families, or key-value pairs depending on how you project it. +/// Migrations are free because there is nothing to migrate — you just update +/// the projection. +/// +/// # How It Works +/// +/// 1. Register a `ProjectionSchema` that describes which nodes are in scope +/// and how to map their fields. +/// 2. At query time, run spreading activation (or use an existing result set). +/// 3. Apply the projection to map `ActivatedNode`s into the projected view. +/// +/// The projection is purely a read-time transform. It never modifies the graph. +pub mod engine; +pub mod error; +pub mod registry; +pub mod schema; + +pub use engine::ProjectionEngine; +pub use error::ProjectionError; +pub use registry::ProjectionRegistry; +pub use schema::{ + FieldMapping, FieldSource, NodeFilter, ProjectedRow, ProjectionResult, ProjectionSchema, + ProjectionType, +}; diff --git a/engram/crates/engram-projection/src/registry.rs b/engram/crates/engram-projection/src/registry.rs new file mode 100644 index 0000000..95c1c78 --- /dev/null +++ b/engram/crates/engram-projection/src/registry.rs @@ -0,0 +1,133 @@ +/// In-memory registry of named projection schemas. +/// +/// The registry is the store of all registered projections. In a running server, +/// one registry instance is shared (behind a Mutex or RwLock). Projections are +/// looked up by name to execute queries. +use std::collections::HashMap; + +use crate::error::{ProjectionError, ProjectionResult}; +use crate::schema::ProjectionSchema; + +/// Holds all registered `ProjectionSchema`s, keyed by name. +#[derive(Default)] +pub struct ProjectionRegistry { + schemas: HashMap, +} + +impl ProjectionRegistry { + pub fn new() -> Self { + Self { + schemas: HashMap::new(), + } + } + + /// Register a new schema. Fails if one with the same name already exists. + pub fn register(&mut self, schema: ProjectionSchema) -> ProjectionResult<()> { + if self.schemas.contains_key(&schema.name) { + return Err(ProjectionError::AlreadyExists(schema.name.clone())); + } + if schema.name.is_empty() { + return Err(ProjectionError::InvalidSchema("name must not be empty".into())); + } + self.schemas.insert(schema.name.clone(), schema); + Ok(()) + } + + /// Replace an existing schema (upsert). Creates if not present. + pub fn upsert(&mut self, schema: ProjectionSchema) -> ProjectionResult<()> { + if schema.name.is_empty() { + return Err(ProjectionError::InvalidSchema("name must not be empty".into())); + } + self.schemas.insert(schema.name.clone(), schema); + Ok(()) + } + + /// Retrieve a schema by name. + pub fn get(&self, name: &str) -> ProjectionResult<&ProjectionSchema> { + self.schemas + .get(name) + .ok_or_else(|| ProjectionError::NotFound(name.to_string())) + } + + /// List all schema names. + pub fn list(&self) -> Vec<&ProjectionSchema> { + let mut schemas: Vec<&ProjectionSchema> = self.schemas.values().collect(); + schemas.sort_by(|a, b| a.name.cmp(&b.name)); + schemas + } + + /// Remove a schema by name. Returns true if it existed. + pub fn remove(&mut self, name: &str) -> bool { + self.schemas.remove(name).is_some() + } + + /// Number of registered schemas. + pub fn len(&self) -> usize { + self.schemas.len() + } + + /// True if no schemas are registered. + pub fn is_empty(&self) -> bool { + self.schemas.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::{NodeFilter, ProjectionType}; + + fn make_schema(name: &str) -> ProjectionSchema { + ProjectionSchema { + name: name.to_string(), + description: None, + projection_type: ProjectionType::Relational, + node_filter: NodeFilter::All, + field_mappings: vec![], + } + } + + #[test] + fn test_register_and_get() { + let mut reg = ProjectionRegistry::new(); + reg.register(make_schema("users")).unwrap(); + let s = reg.get("users").unwrap(); + assert_eq!(s.name, "users"); + } + + #[test] + fn test_register_duplicate_fails() { + let mut reg = ProjectionRegistry::new(); + reg.register(make_schema("events")).unwrap(); + assert!(reg.register(make_schema("events")).is_err()); + } + + #[test] + fn test_upsert_replaces() { + let mut reg = ProjectionRegistry::new(); + reg.register(make_schema("s1")).unwrap(); + let mut updated = make_schema("s1"); + updated.description = Some("updated".into()); + reg.upsert(updated).unwrap(); + assert_eq!(reg.get("s1").unwrap().description.as_deref(), Some("updated")); + } + + #[test] + fn test_list_sorted() { + let mut reg = ProjectionRegistry::new(); + reg.register(make_schema("zoo")).unwrap(); + reg.register(make_schema("alpha")).unwrap(); + reg.register(make_schema("mango")).unwrap(); + let names: Vec<&str> = reg.list().iter().map(|s| s.name.as_str()).collect(); + assert_eq!(names, vec!["alpha", "mango", "zoo"]); + } + + #[test] + fn test_remove() { + let mut reg = ProjectionRegistry::new(); + reg.register(make_schema("temp")).unwrap(); + assert!(reg.remove("temp")); + assert!(!reg.remove("temp")); + assert!(reg.get("temp").is_err()); + } +} diff --git a/engram/crates/engram-projection/src/schema.rs b/engram/crates/engram-projection/src/schema.rs new file mode 100644 index 0000000..79e7543 --- /dev/null +++ b/engram/crates/engram-projection/src/schema.rs @@ -0,0 +1,133 @@ +/// Schema types for the projection layer. +/// +/// A `ProjectionSchema` defines a named view over the Engram graph. +/// It specifies which nodes are in scope and how to extract fields from them. +use engram_core::types::MemoryTier; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +/// How the projection presents data to the caller. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ProjectionType { + /// Nodes as rows; edges become foreign key references. + /// Each row is a flat map of field_name → value. + Relational, + /// Nodes as JSON documents. + /// Fields are nested under a "fields" key; metadata at the top level. + Document, + /// Nodes as column families (node_id → column_name → value). + /// Suitable for wide, sparse schemas. + WideColumn, + /// Simple node_id → content mapping. + /// Ignores field mappings; raw content bytes as base64. + KeyValue, +} + +/// Which nodes from the activation result set fall within this projection's scope. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", content = "value")] +pub enum NodeFilter { + /// Include nodes whose node_type matches any of the given strings. + ByType(Vec), + /// Include nodes in any of the given memory tiers. + ByTier(Vec), + /// Include nodes whose tier name contains any of the given tag strings + /// (stored in node metadata via content prefix convention). + ByTag(Vec), + /// Include nodes with activation strength >= threshold. + ByActivationThreshold(f32), + /// Include nodes whose salience >= threshold. + BySalience(f32), + /// All of the sub-filters must match (AND). + Combined(Vec), + /// Any sub-filter matches (OR). + Any(Vec), + /// Pass all nodes through without filtering. + All, +} + +/// Where to source a projected field's value. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", content = "value")] +pub enum FieldSource { + /// Extract a value from the node's content, interpreted as JSON, using a dot-path. + /// E.g., "user.name" extracts `{ "user": { "name": "Alice" } }["user"]["name"]`. + ContentJsonPath(String), + /// The node's content, raw, as a UTF-8 string (lossy). + ContentRaw, + /// The node's content as a base64-encoded string. + ContentBase64, + /// The node's unique identifier. + NodeId, + /// The node's type as a string. + NodeType, + /// The node's memory tier as a string. + Tier, + /// The node's current salience score. + Salience, + /// The node's importance (caller-set, stable). + Importance, + /// The activation strength at this node (from spreading activation). + ActivationStrength, + /// The hop count from the nearest seed node. + Hops, + /// The node's creation timestamp (Unix ms). + CreatedAt, + /// The node's last-activated timestamp (Unix ms). + LastActivated, + /// The node's activation count. + ActivationCount, + /// A literal constant value. + Literal(Value), +} + +/// One field in a projected row. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FieldMapping { + /// The name of this field in the projected output. + pub field_name: String, + /// Where to get the value from. + pub source: FieldSource, + /// If the source fails to produce a value, use this fallback. Null means omit. + pub default: Option, +} + +/// A complete schema definition. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProjectionSchema { + /// Unique name for this projection. + pub name: String, + /// Human-readable description. + pub description: Option, + /// How results are shaped. + pub projection_type: ProjectionType, + /// Which nodes from the activation result are included. + pub node_filter: NodeFilter, + /// How to extract fields from each included node. + pub field_mappings: Vec, +} + +/// One projected node in a Relational or Document projection. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProjectedRow { + /// The source node's UUID (always included). + pub node_id: uuid::Uuid, + /// Extracted fields as ordered map field_name → value. + pub fields: HashMap, +} + +/// The output of running a projection over an activation result set. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProjectionResult { + /// The schema that produced this result. + pub schema_name: String, + /// How many nodes from the activation set were in scope. + pub nodes_in_scope: usize, + /// The projected rows. + pub rows: Vec, + /// For KeyValue projection: node_id (string) → content. + pub key_value: Option>, + /// For WideColumn projection: node_id → column_name → value. + pub wide_column: Option>>, +} diff --git a/engram/crates/engram-server/Cargo.toml b/engram/crates/engram-server/Cargo.toml index 80d32f6..addfd84 100644 --- a/engram/crates/engram-server/Cargo.toml +++ b/engram/crates/engram-server/Cargo.toml @@ -12,6 +12,10 @@ path = "src/main.rs" [dependencies] engram-core = { path = "../engram-core" } engram-sync = { path = "../engram-sync" } +engram-projection = { path = "../engram-projection" } +engram-tx = { path = "../engram-tx" } +engram-crypto = { path = "../engram-crypto" } +sled = "0.34" axum = { version = "0.7", features = ["json"] } tokio = { version = "1", features = ["full"] } tower = "0.4" diff --git a/engram/crates/engram-server/src/main.rs b/engram/crates/engram-server/src/main.rs index 1f49645..05b3374 100644 --- a/engram/crates/engram-server/src/main.rs +++ b/engram/crates/engram-server/src/main.rs @@ -36,7 +36,9 @@ use axum::{ Router, }; use engram_core::EngramDb; +use engram_projection::registry::ProjectionRegistry; use engram_sync::{SyncConfig, SyncEngine}; +use engram_tx::TransactionEngine; use mime_guess::from_path; use rust_embed::RustEmbed; use tokio::time::interval; @@ -109,6 +111,18 @@ async fn main() -> anyhow::Result<()> { let db = EngramDb::open(&PathBuf::from(&db_path))?; let db = Arc::new(Mutex::new(db)); + // Transaction engine (separate sled db alongside the main db) + let tx_log_path = format!("{}-tx-log", db_path); + let tx_log_db = sled::open(&tx_log_path)?; + let tx_engine = Arc::new(Mutex::new(TransactionEngine::new( + db.clone(), + tx_log_db, + Some(uuid::Uuid::new_v4()), + ))); + + // Projection registry + let projection_registry = Arc::new(Mutex::new(ProjectionRegistry::new())); + info!("Database opened at {}", db_path); // Sync engine — wrapped in tokio::sync::Mutex so it can be held across .await @@ -164,6 +178,8 @@ async fn main() -> anyhow::Result<()> { db: db.clone(), sync_engine: sync_engine.clone(), api_key: api_key.clone(), + projection_registry, + tx_engine, }); // Protected sync/swarm routes (auth middleware applied) @@ -192,6 +208,26 @@ async fn main() -> anyhow::Result<()> { .route("/decay", post(routes::core::decay)) .route("/consolidate", post(routes::core::consolidate)); + // Projection routes (no auth) + let projection_routes = Router::new() + .route("/projections", post(routes::projection::register_projection)) + .route("/projections", get(routes::projection::list_projections)) + .route( + "/projections/{name}/schema", + get(routes::projection::get_projection_schema), + ) + .route( + "/projections/{name}/query", + post(routes::projection::query_projection), + ); + + // Transaction routes (no auth — add auth layer if needed) + let tx_routes = Router::new() + .route("/tx/apply", post(routes::tx::tx_apply)) + .route("/tx/rollback/{command_id}", post(routes::tx::tx_rollback)) + .route("/tx/history", get(routes::tx::tx_history)) + .route("/tx/chain/{command_id}", get(routes::tx::tx_causal_chain)); + let studio_routes = Router::new() .route("/", get(serve_studio_index)) .route("/studio", get(serve_studio_index)) @@ -201,6 +237,8 @@ async fn main() -> anyhow::Result<()> { .merge(studio_routes) .merge(core_routes) .merge(sync_routes) + .merge(projection_routes) + .merge(tx_routes) .layer(CorsLayer::permissive()) .with_state(state); diff --git a/engram/crates/engram-server/src/routes/mod.rs b/engram/crates/engram-server/src/routes/mod.rs index d61ad46..6699a3f 100644 --- a/engram/crates/engram-server/src/routes/mod.rs +++ b/engram/crates/engram-server/src/routes/mod.rs @@ -1,3 +1,5 @@ pub mod core; +pub mod projection; pub mod sync; pub mod swarm; +pub mod tx; diff --git a/engram/crates/engram-server/src/routes/projection.rs b/engram/crates/engram-server/src/routes/projection.rs new file mode 100644 index 0000000..8994cfd --- /dev/null +++ b/engram/crates/engram-server/src/routes/projection.rs @@ -0,0 +1,121 @@ +/// Projection API routes. +/// +/// POST /projections — register a projection schema +/// GET /projections — list all registered schemas +/// POST /projections/{name}/query — run activation then project results +/// GET /projections/{name}/schema — get a schema definition +use axum::{ + extract::{Path, State}, + http::StatusCode, + Json, +}; +use engram_projection::{ + engine::ProjectionEngine, + schema::{ProjectionResult, ProjectionSchema}, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use uuid::Uuid; + +use crate::state::AppState; + +// ── POST /projections ───────────────────────────────────────────────────────── + +pub async fn register_projection( + State(state): State>, + Json(schema): Json, +) -> Result { + let mut reg = state + .projection_registry + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + reg.upsert(schema) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(StatusCode::CREATED) +} + +// ── GET /projections ────────────────────────────────────────────────────────── + +#[derive(Serialize)] +pub struct ListProjectionsResponse { + pub schemas: Vec, +} + +pub async fn list_projections( + State(state): State>, +) -> Result, StatusCode> { + let reg = state + .projection_registry + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let schemas = reg.list().into_iter().cloned().collect(); + Ok(Json(ListProjectionsResponse { schemas })) +} + +// ── GET /projections/{name}/schema ──────────────────────────────────────────── + +pub async fn get_projection_schema( + State(state): State>, + Path(name): Path, +) -> Result, StatusCode> { + let reg = state + .projection_registry + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + match reg.get(&name) { + Ok(schema) => Ok(Json(schema.clone())), + Err(_) => Err(StatusCode::NOT_FOUND), + } +} + +// ── POST /projections/{name}/query ──────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct ProjectionQueryRequest { + /// Seed node IDs for spreading activation. + pub seeds: Vec, + /// Query embedding for spreading activation. + pub query_embedding: Vec, + #[serde(default = "default_depth")] + pub max_depth: u8, + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_depth() -> u8 { + 3 +} +fn default_limit() -> usize { + 20 +} + +pub async fn query_projection( + State(state): State>, + Path(name): Path, + Json(req): Json, +) -> Result, StatusCode> { + // Load schema + let schema = { + let reg = state + .projection_registry + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + match reg.get(&name) { + Ok(s) => s.clone(), + Err(_) => return Err(StatusCode::NOT_FOUND), + } + }; + + // Run spreading activation + let activated = { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + db.activate(&req.seeds, &req.query_embedding, req.max_depth, req.limit) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + }; + + // Apply projection + let result = ProjectionEngine::project(&schema, &activated) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + Ok(Json(result)) +} diff --git a/engram/crates/engram-server/src/routes/tx.rs b/engram/crates/engram-server/src/routes/tx.rs new file mode 100644 index 0000000..2065c5f --- /dev/null +++ b/engram/crates/engram-server/src/routes/tx.rs @@ -0,0 +1,109 @@ +/// Transaction API routes. +/// +/// POST /tx/apply — apply a command +/// POST /tx/rollback/{id} — roll back a command +/// GET /tx/history?since={ms} — command history +/// GET /tx/chain/{id} — causal chain for a command +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + Json, +}; +use engram_tx::{Command, CommandResult}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use uuid::Uuid; + +use crate::state::AppState; + +// ── POST /tx/apply ──────────────────────────────────────────────────────────── + +pub async fn tx_apply( + State(state): State>, + Json(cmd): Json, +) -> Result, StatusCode> { + let mut engine = state + .tx_engine + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let result = engine + .apply(cmd) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(result)) +} + +// ── POST /tx/rollback/{command_id} ─────────────────────────────────────────── + +#[derive(Serialize)] +pub struct RollbackResponse { + pub rollback_command_id: Uuid, + pub status: String, +} + +pub async fn tx_rollback( + State(state): State>, + Path(command_id): Path, +) -> Result, StatusCode> { + let mut engine = state + .tx_engine + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let rb = engine + .rollback(command_id) + .map_err(|e| { + tracing::warn!("rollback failed: {}", e); + StatusCode::BAD_REQUEST + })?; + Ok(Json(RollbackResponse { + rollback_command_id: rb.id, + status: format!("{:?}", rb.status), + })) +} + +// ── GET /tx/history ─────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct HistoryParams { + pub since: Option, +} + +#[derive(Serialize)] +pub struct HistoryResponse { + pub commands: Vec, +} + +pub async fn tx_history( + State(state): State>, + Query(params): Query, +) -> Result, StatusCode> { + let engine = state + .tx_engine + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let since = params.since.unwrap_or(0); + let commands = engine + .history(since) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(HistoryResponse { commands })) +} + +// ── GET /tx/chain/{command_id} ──────────────────────────────────────────────── + +#[derive(Serialize)] +pub struct CausalChainResponse { + pub chain: Vec, +} + +pub async fn tx_causal_chain( + State(state): State>, + Path(command_id): Path, +) -> Result, StatusCode> { + let engine = state + .tx_engine + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let chain = engine + .causal_chain(command_id) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(CausalChainResponse { chain })) +} diff --git a/engram/crates/engram-server/src/state.rs b/engram/crates/engram-server/src/state.rs index b32e9fa..c56f280 100644 --- a/engram/crates/engram-server/src/state.rs +++ b/engram/crates/engram-server/src/state.rs @@ -1,6 +1,8 @@ /// Shared application state for all request handlers. use engram_core::EngramDb; +use engram_projection::registry::ProjectionRegistry; use engram_sync::SyncEngine; +use engram_tx::TransactionEngine; use std::sync::{Arc, Mutex}; pub struct AppState { @@ -10,4 +12,8 @@ pub struct AppState { pub sync_engine: Arc>, /// API key used to authenticate incoming sync requests pub api_key: String, + /// Projection registry — named schema views over the activation surface + pub projection_registry: Arc>, + /// Transaction engine — append-only command log with rollback support + pub tx_engine: Arc>, } diff --git a/engram/crates/engram-tx/Cargo.toml b/engram/crates/engram-tx/Cargo.toml new file mode 100644 index 0000000..55ac1f3 --- /dev/null +++ b/engram/crates/engram-tx/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "engram-tx" +version = "0.1.0" +edition = "2021" +description = "Command pattern + rollback-of-rollback transaction engine for Engram" +license = "MIT" + +[dependencies] +engram-core = { path = "../engram-core" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +uuid = { version = "1", features = ["v4", "serde"] } +thiserror = "1" +sled = "0.34" + +[dev-dependencies] +tempfile = "3" diff --git a/engram/crates/engram-tx/src/command.rs b/engram/crates/engram-tx/src/command.rs new file mode 100644 index 0000000..54eda16 --- /dev/null +++ b/engram/crates/engram-tx/src/command.rs @@ -0,0 +1,166 @@ +/// Command — the first-class mutation unit of Engram's transaction system. +/// +/// A command is an immutable record of intent. Once created, its ID, type, +/// idempotency key, and causal parent never change. Status and conflict fields +/// are the only mutable parts, updated as the command moves through its lifecycle. +use engram_core::types::MemoryTier; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// The operation a command performs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum CommandType { + /// Create a new node. `payload` contains the node's fields. + CreateNode, + /// Update an existing node's content/metadata. `payload` contains the new fields. + UpdateNode, + /// Delete a node by UUID. + DeleteNode, + /// Create an edge between two nodes. + CreateEdge, + /// Delete an edge by its from/to pair. + DeleteEdge, + /// Update a node's salience score. + UpdateSalience, + /// Batch import of many nodes/edges. + BulkImport, + /// Roll back a previously applied command. The UUID is the target command's ID. + Rollback(Uuid), +} + +/// Lifecycle status of a command. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum CommandStatus { + /// Created but not yet applied. + Pending, + /// Successfully applied to the database. + Applied, + /// Rolled back (a subsequent Rollback command was applied). + RolledBack, + /// Applied but conflicted with another command; conflict details in the log. + Conflicted, +} + +/// A command in Engram's append-only mutation log. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Command { + /// Stable unique identifier for this command. + pub id: Uuid, + /// The operation this command performs. + pub command_type: CommandType, + /// The forward operation data — what to do. + pub payload: Value, + /// The inverse operation data — computed eagerly at command creation time. + /// This is the "undo" data, available even if the graph has changed since. + pub inverse_payload: Value, + /// Dedup key. If a command with this idempotency key has already been applied, + /// the new command is a no-op. Prevents double-application in distributed sync. + pub idempotency_key: String, + /// The command that caused this one, if any (forms the causal DAG). + pub causal_parent: Option, + /// Unix milliseconds when this command was created. + pub timestamp_ms: i64, + /// Current lifecycle status. + pub status: CommandStatus, + /// Which peer originated this command (for conflict resolution). + pub peer_id: Option, + /// If conflicted, a human-readable description of the conflict. + pub conflict_note: Option, +} + +impl Command { + /// Create a new pending command. + pub fn new( + command_type: CommandType, + payload: Value, + inverse_payload: Value, + idempotency_key: impl Into, + causal_parent: Option, + peer_id: Option, + ) -> Self { + Self { + id: Uuid::new_v4(), + command_type, + payload, + inverse_payload, + idempotency_key: idempotency_key.into(), + causal_parent, + timestamp_ms: engram_core::types::now_ms(), + status: CommandStatus::Pending, + peer_id, + conflict_note: None, + } + } +} + +/// The result of successfully applying a command. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandResult { + /// The command that was applied. + pub command_id: Uuid, + /// The command's new status. + pub status: CommandStatus, + /// Any entity UUID produced by the operation (e.g., the new node's UUID). + pub produced_id: Option, + /// Whether this was a no-op due to idempotency. + pub was_idempotent: bool, +} + +// ── Payload schema helpers ──────────────────────────────────────────────────── + +/// Payload for CreateNode commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreateNodePayload { + pub node_id: Uuid, + pub node_type: String, + pub embedding: Vec, + pub content: Vec, + pub tier: MemoryTier, + pub importance: f32, +} + +/// Payload for UpdateNode commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UpdateNodePayload { + pub node_id: Uuid, + pub new_content: Option>, + pub new_importance: Option, + pub new_tier: Option, +} + +/// Payload for DeleteNode commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeleteNodePayload { + pub node_id: Uuid, + /// The full node is saved at command-creation time so rollback can restore it. + pub snapshot: Value, +} + +/// Payload for CreateEdge commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreateEdgePayload { + pub edge_id: Uuid, + pub from_id: Uuid, + pub to_id: Uuid, + pub relation: String, + pub weight: f32, +} + +/// Payload for DeleteEdge commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeleteEdgePayload { + pub edge_id: Uuid, + pub from_id: Uuid, + pub to_id: Uuid, + /// Full edge snapshot for rollback. + pub snapshot: Value, +} + +/// Payload for UpdateSalience commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UpdateSaliencePayload { + pub node_id: Uuid, + pub new_salience: f32, + pub old_salience: f32, +} diff --git a/engram/crates/engram-tx/src/engine.rs b/engram/crates/engram-tx/src/engine.rs new file mode 100644 index 0000000..fe6691a --- /dev/null +++ b/engram/crates/engram-tx/src/engine.rs @@ -0,0 +1,710 @@ +/// TransactionEngine — applies commands to EngramDb and manages rollback. +/// +/// The engine wraps an `EngramDb` and a `CommandLog`. All mutations go through +/// `apply()`. Rollbacks create new inverse commands. Rolling back a rollback +/// re-applies the original — full undo/redo with causal tracking. +use engram_core::types::{Edge, Node, NodeType, RelationType}; +use engram_core::EngramDb; +use serde_json::Value; +use uuid::Uuid; + +use crate::command::{ + Command, CommandResult, CommandStatus, CommandType, CreateEdgePayload, CreateNodePayload, + DeleteEdgePayload, DeleteNodePayload, UpdateNodePayload, UpdateSaliencePayload, +}; +use crate::error::{TxError, TxResult}; +use crate::log::CommandLog; + +pub struct TransactionEngine { + db: std::sync::Arc>, + log: CommandLog, + /// Our peer ID for conflict resolution. + peer_id: Option, +} + +impl TransactionEngine { + /// Create a new engine backed by the given database and a sled store for the log. + pub fn new( + db: std::sync::Arc>, + log_db: sled::Db, + peer_id: Option, + ) -> Self { + Self { + db, + log: CommandLog::open(log_db), + peer_id, + } + } + + // ── Public API ──────────────────────────────────────────────────────────── + + /// Apply a command to the database. + /// + /// Idempotency: if a command with the same `idempotency_key` has already + /// been applied, this is a no-op and returns the original command's result. + pub fn apply(&mut self, mut cmd: Command) -> TxResult { + // Idempotency check + if let Some(existing_id) = self.log.check_idempotency(&cmd.idempotency_key)? { + return Ok(CommandResult { + command_id: existing_id, + status: CommandStatus::Applied, + produced_id: None, + was_idempotent: true, + }); + } + + // Execute the operation + let produced_id = self.execute(&cmd)?; + + cmd.status = CommandStatus::Applied; + self.log.write(&cmd)?; + + Ok(CommandResult { + command_id: cmd.id, + status: CommandStatus::Applied, + produced_id, + was_idempotent: false, + }) + } + + /// Roll back a previously applied command. + /// + /// Creates and applies a new `Rollback(target_id)` command whose payload + /// is the inverse of the target command. The target command's status is + /// updated to `RolledBack`. + /// + /// Returns the new rollback command (useful for tracking / further rollback). + pub fn rollback(&mut self, target_id: Uuid) -> TxResult { + let target = self.log.require(target_id)?; + + if target.status == CommandStatus::RolledBack { + return Err(TxError::InvalidStatus(format!( + "command {} is already rolled back", + target_id + ))); + } + if target.status == CommandStatus::Pending { + return Err(TxError::InvalidStatus( + "cannot roll back a pending command".into(), + )); + } + + // The rollback's payload is the original command's inverse_payload. + // The rollback's inverse_payload is the original command's payload. + // This enables rollback-of-rollback to re-apply the original. + let rollback_key = format!("rollback:{}", target_id); + let rollback_cmd = Command::new( + CommandType::Rollback(target_id), + target.inverse_payload.clone(), + target.payload.clone(), + rollback_key, + Some(target_id), + self.peer_id, + ); + + // Execute the inverse operation + self.execute_inverse(&target)?; + + // Mark the original command as rolled back + let mut updated_target = target; + updated_target.status = CommandStatus::RolledBack; + self.log.write(&updated_target)?; + + // Persist the rollback command as Applied + let mut rb = rollback_cmd; + rb.status = CommandStatus::Applied; + self.log.write(&rb)?; + + Ok(rb) + } + + /// Roll back a rollback — re-applying the original command. + /// + /// This is "undo the undo". The rollback_id must be a command of type + /// `Rollback(original_id)`. Rolling it back re-applies `original_id`. + pub fn rollback_rollback(&mut self, rollback_id: Uuid) -> TxResult { + let rb_cmd = self.log.require(rollback_id)?; + + // Verify this IS a rollback command + let original_id = match &rb_cmd.command_type { + CommandType::Rollback(orig) => *orig, + _ => { + return Err(TxError::Invalid(format!( + "command {} is not a Rollback command", + rollback_id + ))); + } + }; + + if rb_cmd.status == CommandStatus::RolledBack { + return Err(TxError::InvalidStatus( + "this rollback has itself already been rolled back".into(), + )); + } + + // The rollback_rollback's payload is rb_cmd.inverse_payload (= original payload) + // Its inverse is rb_cmd.payload (= original's inverse_payload) + let key = format!("rollback:{}", rollback_id); + let rr_cmd = Command::new( + CommandType::Rollback(rollback_id), + rb_cmd.inverse_payload.clone(), + rb_cmd.payload.clone(), + key, + Some(rollback_id), + self.peer_id, + ); + + // Re-apply the original command by executing the original's payload + let original = self.log.require(original_id)?; + self.execute(&original)?; + + // Mark original as Applied again + let mut orig = original; + orig.status = CommandStatus::Applied; + self.log.write(&orig)?; + + // Mark rollback as RolledBack + let mut rb = rb_cmd; + rb.status = CommandStatus::RolledBack; + self.log.write(&rb)?; + + // Persist the new re-apply command + let mut rr = rr_cmd; + rr.status = CommandStatus::Applied; + self.log.write(&rr)?; + + Ok(rr) + } + + /// All commands since a given Unix millisecond timestamp. + pub fn history(&self, since_ms: i64) -> TxResult> { + self.log.since(since_ms) + } + + /// The causal chain for a given command (root-first). + pub fn causal_chain(&self, command_id: Uuid) -> TxResult> { + self.log.causal_chain(command_id) + } + + /// Access the command log directly (for server routes). + pub fn log(&self) -> &CommandLog { + &self.log + } + + // ── Command execution ───────────────────────────────────────────────────── + + fn execute(&self, cmd: &Command) -> TxResult> { + match &cmd.command_type { + CommandType::CreateNode => self.exec_create_node(&cmd.payload), + CommandType::UpdateNode => { + self.exec_update_node(&cmd.payload)?; + Ok(None) + } + CommandType::DeleteNode => { + self.exec_delete_node(&cmd.payload)?; + Ok(None) + } + CommandType::CreateEdge => { + self.exec_create_edge(&cmd.payload)?; + Ok(None) + } + CommandType::DeleteEdge => { + self.exec_delete_edge(&cmd.payload)?; + Ok(None) + } + CommandType::UpdateSalience => { + self.exec_update_salience(&cmd.payload)?; + Ok(None) + } + CommandType::BulkImport => { + self.exec_bulk_import(&cmd.payload)?; + Ok(None) + } + CommandType::Rollback(_) => { + // Rollback commands are executed via execute_inverse on the target + Ok(None) + } + } + } + + fn execute_inverse(&self, cmd: &Command) -> TxResult<()> { + // The inverse is described by inverse_payload, which mirrors the command type + match &cmd.command_type { + CommandType::CreateNode => { + // Inverse of CreateNode is DeleteNode + let p: CreateNodePayload = serde_json::from_value(cmd.payload.clone())?; + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + db.delete_node(p.node_id)?; + } + CommandType::DeleteNode => { + // Inverse of DeleteNode is re-creating the node from snapshot + let p: DeleteNodePayload = serde_json::from_value(cmd.payload.clone())?; + let node: Node = serde_json::from_value(p.snapshot)?; + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + db.put_node(node)?; + } + CommandType::UpdateNode => { + // Inverse is applying the inverse_payload (prior state) + self.exec_update_node(&cmd.inverse_payload)?; + } + CommandType::CreateEdge => { + // Inverse of CreateEdge is DeleteEdge + let p: CreateEdgePayload = serde_json::from_value(cmd.payload.clone())?; + self.delete_edge_by_pair(p.from_id, p.to_id)?; + } + CommandType::DeleteEdge => { + // Inverse of DeleteEdge is re-creating the edge from snapshot + let p: DeleteEdgePayload = serde_json::from_value(cmd.payload.clone())?; + let edge: Edge = serde_json::from_value(p.snapshot)?; + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + db.put_edge(edge)?; + } + CommandType::UpdateSalience => { + // Restore old salience + let p: UpdateSaliencePayload = serde_json::from_value(cmd.payload.clone())?; + let restore = UpdateSaliencePayload { + node_id: p.node_id, + new_salience: p.old_salience, + old_salience: p.new_salience, + }; + self.exec_update_salience(&serde_json::to_value(restore)?)?; + } + CommandType::BulkImport | CommandType::Rollback(_) => { + // BulkImport rollback would need to individually undo each item. + // For now, we store the inverse_payload as instructions and log the gap. + // TODO: implement fine-grained BulkImport rollback + } + } + Ok(()) + } + + // ── Operation implementations ───────────────────────────────────────────── + + fn exec_create_node(&self, payload: &Value) -> TxResult> { + let p: CreateNodePayload = serde_json::from_value(payload.clone())?; + let node_type = parse_node_type(&p.node_type)?; + let tier = p.tier; + let node = Node::new(node_type, p.embedding, p.content, tier, p.importance) + .with_id(p.node_id); + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + db.put_node(node)?; + Ok(Some(p.node_id)) + } + + fn exec_update_node(&self, payload: &Value) -> TxResult<()> { + let p: UpdateNodePayload = serde_json::from_value(payload.clone())?; + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + let mut node = db + .get_node(p.node_id)? + .ok_or(TxError::NotFound(p.node_id))?; + if let Some(content) = p.new_content { + node.content = content; + } + if let Some(importance) = p.new_importance { + node.importance = importance.clamp(0.0, 1.0); + } + if let Some(tier) = p.new_tier { + node.tier = tier; + } + db.put_node(node)?; + Ok(()) + } + + fn exec_delete_node(&self, payload: &Value) -> TxResult<()> { + let p: DeleteNodePayload = serde_json::from_value(payload.clone())?; + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + db.delete_node(p.node_id)?; + Ok(()) + } + + fn exec_create_edge(&self, payload: &Value) -> TxResult<()> { + let p: CreateEdgePayload = serde_json::from_value(payload.clone())?; + let relation = parse_relation(&p.relation)?; + let edge = Edge::new(p.from_id, p.to_id, relation, p.weight); + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + db.put_edge(edge)?; + Ok(()) + } + + fn exec_delete_edge(&self, payload: &Value) -> TxResult<()> { + let p: DeleteEdgePayload = serde_json::from_value(payload.clone())?; + self.delete_edge_by_pair(p.from_id, p.to_id)?; + Ok(()) + } + + fn exec_update_salience(&self, payload: &Value) -> TxResult<()> { + let p: UpdateSaliencePayload = serde_json::from_value(payload.clone())?; + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + let mut node = db + .get_node(p.node_id)? + .ok_or(TxError::NotFound(p.node_id))?; + node.salience = p.new_salience; + db.put_node(node)?; + Ok(()) + } + + fn exec_bulk_import(&self, payload: &Value) -> TxResult<()> { + let nodes_val = payload.get("nodes").and_then(|v| v.as_array()).cloned().unwrap_or_default(); + let edges_val = payload.get("edges").and_then(|v| v.as_array()).cloned().unwrap_or_default(); + + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + for nv in nodes_val { + let node: Node = serde_json::from_value(nv)?; + db.put_node(node)?; + } + for ev in edges_val { + let edge: Edge = serde_json::from_value(ev)?; + db.put_edge(edge)?; + } + Ok(()) + } + + fn delete_edge_by_pair(&self, from_id: Uuid, to_id: Uuid) -> TxResult<()> { + // sled stores edges at "edges:from:{from}:{to}" — we remove both directions + let db = self.db.lock().map_err(|_| TxError::Invalid("lock poisoned".into()))?; + // We can't directly call into storage here without re-exposing internals, + // so we use the public scan-and-check approach via get_edges_from + let edges = db.get_edges_from(from_id)?; + for edge in edges { + if edge.to_id == to_id { + // Re-insert a tombstone isn't directly supported — we use the + // internal sled key to delete. Since we don't have direct sled + // access through EngramDb's public API, we rely on the fact that + // put_edge with weight=0 effectively nullifies it, but for proper + // deletion we need access to the underlying store. + // + // We work around this by storing a zero-weight edge with weight=-1 + // as a sentinel, or by exposing delete_edge. Since delete_node is + // already exposed, we add a helper. For now, we store the edge with + // weight 0 (marking it inactive) until a delete_edge API is added. + // + // TODO: add db.delete_edge() to engram-core public API. + // For now: overwrite with weight 0 to effectively disable it. + let mut tombstone = edge; + tombstone.weight = 0.0; + db.put_edge(tombstone)?; + break; + } + } + Ok(()) + } +} + +// ── Command builder helpers ─────────────────────────────────────────────────── + +/// Build a CreateNode command with eagerly-computed inverse. +pub fn build_create_node_cmd( + node: &Node, + idempotency_key: impl Into, + peer_id: Option, +) -> Command { + let payload = serde_json::to_value(CreateNodePayload { + node_id: node.id, + node_type: node_type_to_str(&node.node_type).to_string(), + embedding: node.embedding.clone(), + content: node.content.clone(), + tier: node.tier.clone(), + importance: node.importance, + }) + .unwrap_or(Value::Null); + + // Inverse: delete the node we're about to create + let inverse = serde_json::to_value(DeleteNodePayload { + node_id: node.id, + snapshot: serde_json::to_value(node).unwrap_or(Value::Null), + }) + .unwrap_or(Value::Null); + + Command::new( + CommandType::CreateNode, + payload, + inverse, + idempotency_key, + None, + peer_id, + ) +} + +/// Build a DeleteNode command with eagerly-computed inverse (snapshot). +pub fn build_delete_node_cmd( + node: &Node, + idempotency_key: impl Into, + peer_id: Option, +) -> Command { + let snapshot = serde_json::to_value(node).unwrap_or(Value::Null); + let payload = serde_json::to_value(DeleteNodePayload { + node_id: node.id, + snapshot: snapshot.clone(), + }) + .unwrap_or(Value::Null); + + // Inverse: re-create the node from snapshot + let inverse = serde_json::to_value(CreateNodePayload { + node_id: node.id, + node_type: node_type_to_str(&node.node_type).to_string(), + embedding: node.embedding.clone(), + content: node.content.clone(), + tier: node.tier.clone(), + importance: node.importance, + }) + .unwrap_or(Value::Null); + + Command::new( + CommandType::DeleteNode, + payload, + inverse, + idempotency_key, + None, + peer_id, + ) +} + +/// Build a CreateEdge command. +pub fn build_create_edge_cmd( + edge: &Edge, + idempotency_key: impl Into, + peer_id: Option, +) -> Command { + let relation_str = relation_to_str(&edge.relation).to_string(); + let payload = serde_json::to_value(CreateEdgePayload { + edge_id: edge.id, + from_id: edge.from_id, + to_id: edge.to_id, + relation: relation_str.clone(), + weight: edge.weight, + }) + .unwrap_or(Value::Null); + + // Inverse: delete the edge + let snapshot = serde_json::to_value(edge).unwrap_or(Value::Null); + let inverse = serde_json::to_value(DeleteEdgePayload { + edge_id: edge.id, + from_id: edge.from_id, + to_id: edge.to_id, + snapshot, + }) + .unwrap_or(Value::Null); + + Command::new( + CommandType::CreateEdge, + payload, + inverse, + idempotency_key, + None, + peer_id, + ) +} + +// ── Type string helpers ─────────────────────────────────────────────────────── + +fn parse_node_type(s: &str) -> TxResult { + match s { + "Memory" => Ok(NodeType::Memory), + "Concept" => Ok(NodeType::Concept), + "Event" => Ok(NodeType::Event), + "Entity" => Ok(NodeType::Entity), + "Process" => Ok(NodeType::Process), + "InternalState" => Ok(NodeType::InternalState), + _ => Err(TxError::Invalid(format!("unknown node type: {}", s))), + } +} + +fn node_type_to_str(t: &NodeType) -> &'static str { + match t { + NodeType::Memory => "Memory", + NodeType::Concept => "Concept", + NodeType::Event => "Event", + NodeType::Entity => "Entity", + NodeType::Process => "Process", + NodeType::InternalState => "InternalState", + } +} + +fn parse_relation(s: &str) -> TxResult { + match s { + "Supersedes" => Ok(RelationType::Supersedes), + "Causes" => Ok(RelationType::Causes), + "Contains" => Ok(RelationType::Contains), + "References" => Ok(RelationType::References), + "Contradicts" => Ok(RelationType::Contradicts), + "Exemplifies" => Ok(RelationType::Exemplifies), + "Activates" => Ok(RelationType::Activates), + "TemporallyPrecedes" => Ok(RelationType::TemporallyPrecedes), + _ => Err(TxError::Invalid(format!("unknown relation: {}", s))), + } +} + +fn relation_to_str(r: &RelationType) -> &'static str { + match r { + RelationType::Supersedes => "Supersedes", + RelationType::Causes => "Causes", + RelationType::Contains => "Contains", + RelationType::References => "References", + RelationType::Contradicts => "Contradicts", + RelationType::Exemplifies => "Exemplifies", + RelationType::Activates => "Activates", + RelationType::TemporallyPrecedes => "TemporallyPrecedes", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use engram_core::types::{MemoryTier, Node, NodeType}; + use tempfile::TempDir; + + fn make_engine() -> (TransactionEngine, TempDir, TempDir) { + let db_dir = TempDir::new().unwrap(); + let log_dir = TempDir::new().unwrap(); + let db = engram_core::EngramDb::open(db_dir.path()).unwrap(); + let db = std::sync::Arc::new(std::sync::Mutex::new(db)); + let log_db = sled::open(log_dir.path()).unwrap(); + let engine = TransactionEngine::new(db, log_db, None); + (engine, db_dir, log_dir) + } + + fn make_node() -> Node { + Node::new( + NodeType::Memory, + vec![1.0, 0.0], + b"test content".to_vec(), + MemoryTier::Semantic, + 0.8, + ) + } + + #[test] + fn test_apply_create_node() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + let node = make_node(); + let cmd = build_create_node_cmd(&node, "create-test-1", None); + let result = engine.apply(cmd).unwrap(); + assert_eq!(result.status, CommandStatus::Applied); + assert!(!result.was_idempotent); + + // Node should now exist + let db = engine.db.lock().unwrap(); + let found = db.get_node(node.id).unwrap(); + assert!(found.is_some()); + assert_eq!(found.unwrap().content, b"test content"); + } + + #[test] + fn test_idempotency() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + let node = make_node(); + let cmd1 = build_create_node_cmd(&node, "idem-key-42", None); + let cmd2 = build_create_node_cmd(&node, "idem-key-42", None); + + let r1 = engine.apply(cmd1).unwrap(); + let r2 = engine.apply(cmd2).unwrap(); + + assert!(!r1.was_idempotent); + assert!(r2.was_idempotent); + } + + #[test] + fn test_rollback_delete_node() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + let node = make_node(); + let node_id = node.id; + + // First apply create + let create_cmd = build_create_node_cmd(&node, "create-rb-1", None); + let result = engine.apply(create_cmd).unwrap(); + + // Roll back the creation — should delete the node + let rb = engine.rollback(result.command_id).unwrap(); + assert_eq!(rb.status, CommandStatus::Applied); + + let db = engine.db.lock().unwrap(); + let found = db.get_node(node_id).unwrap(); + assert!(found.is_none()); + } + + #[test] + fn test_rollback_of_rollback() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + let node = make_node(); + let node_id = node.id; + + // Create the node + let create_cmd = build_create_node_cmd(&node, "create-rorb-1", None); + let create_result = engine.apply(create_cmd).unwrap(); + + // Roll back (delete) + let rb = engine.rollback(create_result.command_id).unwrap(); + + // Verify it's gone + { + let db = engine.db.lock().unwrap(); + assert!(db.get_node(node_id).unwrap().is_none()); + } + + // Roll back the rollback (re-create) + let _rr = engine.rollback_rollback(rb.id).unwrap(); + + // Node should be back + let db = engine.db.lock().unwrap(); + let found = db.get_node(node_id).unwrap(); + assert!(found.is_some()); + assert_eq!(found.unwrap().content, b"test content"); + } + + #[test] + fn test_history() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + let node = make_node(); + let cmd = build_create_node_cmd(&node, "hist-1", None); + engine.apply(cmd).unwrap(); + + let history = engine.history(0).unwrap(); + assert!(!history.is_empty()); + assert!(history.iter().any(|c| matches!(c.command_type, CommandType::CreateNode))); + } + + #[test] + fn test_causal_chain() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + let node = make_node(); + let cmd = build_create_node_cmd(&node, "causal-1", None); + let result = engine.apply(cmd).unwrap(); + + // Roll back (creates a child command with causal_parent = create_cmd.id) + let rb = engine.rollback(result.command_id).unwrap(); + + let chain = engine.causal_chain(rb.id).unwrap(); + // Chain should be [create_cmd, rollback_cmd] + assert_eq!(chain.len(), 2); + assert!(matches!(chain[0].command_type, CommandType::CreateNode)); + assert!(matches!(chain[1].command_type, CommandType::Rollback(_))); + } + + #[test] + fn test_create_edge_command() { + let (mut engine, _db_dir, _log_dir) = make_engine(); + + // Create two nodes first + let n1 = make_node(); + let n2 = Node::new( + NodeType::Concept, + vec![0.0, 1.0], + b"concept".to_vec(), + MemoryTier::Semantic, + 0.5, + ); + engine.apply(build_create_node_cmd(&n1, "edge-n1", None)).unwrap(); + engine.apply(build_create_node_cmd(&n2, "edge-n2", None)).unwrap(); + + // Create edge + let edge = Edge::new(n1.id, n2.id, RelationType::References, 0.7); + let edge_cmd = build_create_edge_cmd(&edge, "edge-create-1", None); + let result = engine.apply(edge_cmd).unwrap(); + assert_eq!(result.status, CommandStatus::Applied); + + // Verify edge exists + let db = engine.db.lock().unwrap(); + let edges = db.get_edges_from(n1.id).unwrap(); + assert!(!edges.is_empty()); + } +} diff --git a/engram/crates/engram-tx/src/error.rs b/engram/crates/engram-tx/src/error.rs new file mode 100644 index 0000000..2f20c55 --- /dev/null +++ b/engram/crates/engram-tx/src/error.rs @@ -0,0 +1,31 @@ +use thiserror::Error; +use uuid::Uuid; + +#[derive(Debug, Error)] +pub enum TxError { + #[error("Command not found: {0}")] + NotFound(Uuid), + + #[error("Command already applied (idempotency key: {0})")] + AlreadyApplied(String), + + #[error("Cannot roll back a command in status {0:?}")] + InvalidStatus(String), + + #[error("Conflict: {0}")] + Conflict(String), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("Storage error: {0}")] + Storage(#[from] sled::Error), + + #[error("Engram error: {0}")] + Engram(#[from] engram_core::EngramError), + + #[error("Invalid command: {0}")] + Invalid(String), +} + +pub type TxResult = Result; diff --git a/engram/crates/engram-tx/src/lib.rs b/engram/crates/engram-tx/src/lib.rs new file mode 100644 index 0000000..12cf541 --- /dev/null +++ b/engram/crates/engram-tx/src/lib.rs @@ -0,0 +1,29 @@ +/// Engram Transaction Engine — Command pattern with rollback-of-rollback. +/// +/// # The Central Insight +/// +/// Every mutation to Engram is a Command — a first-class object with: +/// - The operation and its inverse (computed eagerly at command time) +/// - An idempotency key (same key = same command, applied once) +/// - A causal parent (which command caused this one) +/// - A timestamp and originating peer ID +/// +/// Commands form a DAG of causality. You can roll back any command. You can +/// roll back a rollback (undo an undo — re-applying the original). The +/// command log is append-only: history is never rewritten. +/// +/// # Rollback-of-Rollback +/// +/// When you roll back command X, a new `Rollback(X)` command is created and +/// applied. Its `inverse_payload` is X's original `payload`. If you then roll +/// back the rollback, a new `Rollback(rollback_id)` is created, whose effect +/// is to re-apply X. This is a full undo/redo system with causal lineage. +pub mod command; +pub mod engine; +pub mod error; +pub mod log; + +pub use command::{Command, CommandResult, CommandStatus, CommandType}; +pub use engine::TransactionEngine; +pub use error::TxError; +pub use log::CommandLog; diff --git a/engram/crates/engram-tx/src/log.rs b/engram/crates/engram-tx/src/log.rs new file mode 100644 index 0000000..06e244f --- /dev/null +++ b/engram/crates/engram-tx/src/log.rs @@ -0,0 +1,151 @@ +/// CommandLog — append-only log of all commands, stored in sled. +/// +/// Key schema: +/// cmd:{uuid} → JSON-encoded Command (JSON used because Command contains serde_json::Value) +/// idem:{key} → uuid bytes (idempotency index) +/// cmd_ts:{ms}:{uuid} → uuid bytes (time-ordered scan index) +use sled::Db; +use uuid::Uuid; + +use crate::command::Command; +use crate::error::{TxError, TxResult}; + +pub struct CommandLog { + db: Db, +} + +impl CommandLog { + pub fn open(db: Db) -> Self { + Self { db } + } + + // ── Write ───────────────────────────────────────────────────────────────── + + /// Append a command to the log. Overwrites if the UUID already exists + /// (used for status updates after application). + pub fn write(&self, cmd: &Command) -> TxResult<()> { + let key = cmd_key(cmd.id); + // Use JSON (not bincode) because Command contains serde_json::Value, + // which bincode cannot deserialize (DeserializeAnyNotSupported). + let val = serde_json::to_vec(cmd)?; + self.db.insert(key, val)?; + + // Idempotency index: idem:{key} → uuid + let idem_key = idem_key(&cmd.idempotency_key); + self.db.insert(idem_key, cmd.id.as_bytes().to_vec())?; + + // Time index: cmd_ts:{ms:016x}:{uuid} → uuid + let ts_key = ts_key(cmd.timestamp_ms, cmd.id); + self.db.insert(ts_key, cmd.id.as_bytes().to_vec())?; + + Ok(()) + } + + // ── Read ────────────────────────────────────────────────────────────────── + + pub fn get(&self, id: Uuid) -> TxResult> { + match self.db.get(cmd_key(id))? { + Some(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)), + None => Ok(None), + } + } + + pub fn require(&self, id: Uuid) -> TxResult { + self.get(id)?.ok_or(TxError::NotFound(id)) + } + + /// Check whether an idempotency key has already been applied. + /// Returns the command UUID if it exists. + pub fn check_idempotency(&self, key: &str) -> TxResult> { + match self.db.get(idem_key(key))? { + Some(bytes) => { + let arr: [u8; 16] = bytes[..16] + .try_into() + .map_err(|_| TxError::Invalid("bad uuid bytes in idem index".into()))?; + Ok(Some(Uuid::from_bytes(arr))) + } + None => Ok(None), + } + } + + /// Load all commands created at or after `since_ms`, ordered by timestamp. + pub fn since(&self, since_ms: i64) -> TxResult> { + let prefix = format!("cmd_ts:{:016x}:", since_ms); + let mut cmds = Vec::new(); + for result in self.db.range(prefix.as_bytes()..) { + let (k, _v) = result?; + // Check the key starts with "cmd_ts:" + if !k.starts_with(b"cmd_ts:") { + break; + } + // Extract uuid from key: cmd_ts:{ms}:{uuid} + let key_str = std::str::from_utf8(&k) + .map_err(|e| TxError::Invalid(e.to_string()))?; + let parts: Vec<&str> = key_str.splitn(3, ':').collect(); + if parts.len() < 3 { + continue; + } + // parts[1] = ms (hex), parts[2] = uuid + let ts_hex = parts[1]; + let ts = i64::from_str_radix(ts_hex, 16) + .map_err(|e| TxError::Invalid(e.to_string()))?; + if ts < since_ms { + continue; + } + let id: Uuid = parts[2] + .parse() + .map_err(|e: uuid::Error| TxError::Invalid(e.to_string()))?; + if let Some(cmd) = self.get(id)? { + cmds.push(cmd); + } + } + Ok(cmds) + } + + /// Load all commands in the store. + pub fn all(&self) -> TxResult> { + self.since(0) + } + + /// Collect the causal chain leading to a given command (inclusive). + /// Walks `causal_parent` links back to the root. + pub fn causal_chain(&self, id: Uuid) -> TxResult> { + let mut chain = Vec::new(); + let mut current_id = Some(id); + let mut visited = std::collections::HashSet::new(); + + while let Some(cid) = current_id { + if visited.contains(&cid) { + break; // cycle guard + } + visited.insert(cid); + + match self.get(cid)? { + Some(cmd) => { + current_id = cmd.causal_parent; + chain.push(cmd); + } + None => break, + } + } + + // Return root-first (reverse of walk order) + chain.reverse(); + Ok(chain) + } +} + +// ── Key constructors ────────────────────────────────────────────────────────── + +fn cmd_key(id: Uuid) -> Vec { + format!("cmd:{}", id).into_bytes() +} + +fn idem_key(key: &str) -> Vec { + format!("idem:{}", key).into_bytes() +} + +fn ts_key(ts: i64, id: Uuid) -> Vec { + // Zero-padded hex timestamp for lexicographic ordering + format!("cmd_ts:{:016x}:{}", ts, id).into_bytes() +} diff --git a/engram/engram-explainer.html b/engram/engram-explainer.html new file mode 100644 index 0000000..1ab574d --- /dev/null +++ b/engram/engram-explainer.html @@ -0,0 +1,2296 @@ + + + + + +Engram — A Database Designed for Minds + + + + + +
+ +
+
+
+ v0.1.0 · Local-first memory substrate +
+

Engram

+

+ The physical trace of a memory.
+ A database designed for minds, not tables. +

+
+
+ Scroll +
+
+
+ + +
+ The Problem +

Every existing database
gets memory wrong

+ +
+
+ ⊞ +

Relational

+

You store rows. You query rows. Storage and retrieval are entirely separate systems. The structure that holds the data knows nothing about how you will need it back.

+
+
+ ⊙ +

Vector

+

You store embeddings. You search by geometric proximity. Still fundamentally a query against static, passive data — the structure has no opinion about your context.

+
+
+ ◈ +

Graph

+

You store edges. You traverse paths. Still asking questions of a static structure. The graph doesn't activate — you query it from outside, like a stranger reading a map.

+
+
+ +
+

The brain doesn't query.
It activates.

+
+
+ + +
+ Core Mechanism +

Spreading Activation

+

Click any node to set it as the seed. Hit Activate to watch the pattern propagate. Activation flows outward through weighted edges — attenuating at every hop, pruned when too weak to matter.

+ +
+
+ Seed +
Click a node to select
+ + +
+ Speed + +
+
+ +
+ Formula + strength = parent × edge_weight × salience × cos_sim(query, target) + Multiplicative — every factor must be non-trivial for the path to survive +
+
+
+ + +
+ Memory Architecture +

The Four Memory Tiers

+

Nodes migrate between tiers based on salience and reinforcement — exactly as memories migrate between the hippocampus and neocortex through activation and sleep.

+ +
+
+
+ 🔥 +
+
Working
+
Prefrontal cortex — hot, volatile
+
+
+

The K most recently activated nodes. Ultra-fast access. Evicted by recency when capacity is exceeded. What you're actively thinking about right now.

+
+ // Currently active
+ task: "Implement activation BFS"
+ context: spreading_activation_node
+ recent: hebbian_learning_concept +
+
+ +
+
+ 📖 +
+
Episodic
+
Hippocampus — time-ordered experience
+
+
+

Time-stamped events and raw experiences. What happened, and when. The raw feed of observations before they have been abstracted into knowledge.

+
+ 2026-04-27T14:23Z event:
+ "Will explained Dharma Registry.
+ Patterns logged. ISE confirmed." +
+
+ +
+
+ 🧠 +
+
Semantic
+
Neocortex — stable knowledge
+
+
+

The concept graph with weighted associations. Long-term structural knowledge. What you know, abstracted from any specific event that taught it to you.

+
+ concept: "spreading_activation"
+ → Causes: "long_term_potentiation"
+ → Activates: "associative_memory"
+ salience: 0.82, activations: 47 +
+
+ +
+
+ ⚙️ +
+
Procedural
+
Cerebellum — patterns and habits
+
+
+

Encoded patterns, workflows, and repeatable processes. How to do things. Retrieved by similarity to current task context, not by conscious recall.

+
+ process: "commit_workflow"
+ steps: [stage → test → commit
+ → push → check_ci]
+ triggered by: git_context_match +
+
+
+
+ + +
+ Forgetting as Design +

Salience — the attention filter

+

Every node has a salience score. It governs whether a node surfaces during retrieval. It decays. It strengthens on activation. Adjust the sliders to see how the three signals combine.

+ +
+
+
The Salience Formula
+
+ salience + = + importance +
+ salience = + × + 1 / (1 + days_since) +
+ salience = + × + ln(activation_count + 1) +
+ +
+
+ importance + Explicit weight at creation. Stable over time. You set it once. +
+
+ recency + At activation: 1.0. After 1 day: 0.5. After 6 days: ~0.14. Asymptotic toward zero. +
+
+ frequency + Log-compressed: 0→1 activation matters more than 100→101. Diminishing returns. +
+
+
+ +
+
+
+ Importance + 0.85 +
+ +
+ +
+
+ Days since last activation + 2 +
+ +
+ +
+
+ Activation count + 12 +
+ +
+ +
+
Computed Salience
+
0.00
+
+
+
+
+
+
+ +

+ "Forgetting is not failure. It is the system prioritising what matters." +

+
+ + +
+ Memory Lifecycle +

Consolidation

+

In the brain, memories consolidate during sleep — the hippocampus replays experiences into the neocortex until they become stable knowledge. Engram makes this explicit.

+ +
+ +
+
+
Episodic
+

Raw events. Time-stamped experience. Not yet abstract.

+
+ tier: Episodic
+ activation_count: 2
+ salience: 0.41 +
+
+ +
+ + + + + + + + + + + + +
+ activation_count ≥ 5
+ salience ≥ 0.3
+ → promote on consolidate() +
+
+ +
+
Semantic
+

Stable concept. Integrated knowledge. No longer tied to a specific event.

+
+ tier: Semantic
+ activation_count: 7
+ salience: 0.68 +
+
+
+ +
+ Promotion is earned by use — activated, reinforced, and found relevant repeatedly over time. +
+
+
+ + +
+ System Design +

Architecture

+

Three retrieval primitives layered over a single embedded store. No daemon. No network. No server to babysit.

+ +
+
[ Your Intelligence ]
+
↕
+
EngramDb API
+
↕
+
+
+
Spreading Activation
+
Best-first BFS · multiplicative weights · pruning at 0.01
+
+
+
Vector Search
+
Flat cosine scan · semantic direction filter · secondary signal
+
+
+
Graph Traversal
+
BFS by relation type · typed edges · max depth
+
+
+
↕
+
sled — embedded persistent B-tree · bincode serialization · transactional
+
↕
+
[ Your disk — no daemon · no network · local-first ]
+
+ +
+
+
Why multiplication?
+

Addition lets many weak signals accumulate into false relevance. Multiplication is conjunctive: a weak edge, a dormant node, or a semantically irrelevant target all kill the path. This is how associative memory actually works.

+
+
+
Why sled?
+

Local-first. Transactional. No daemon process, no network socket. HNSW indexing layers on top when needed — the graph structure itself is the primary retrieval mechanism.

+
+
+
Why pruning at 0.01?
+

Small enough to allow long indirect chains when intermediate edges are strong. Raise it to focus retrieval; lower it for more associative drift. The brain's attention filter, made explicit.

+
+
+
+ + +
+ Language Bindings +

Use from anywhere

+

A C FFI layer exposes the full API. Language bindings ship for Kotlin, TypeScript, Go, and Rust natively.

+ +
+
+ + + + +
+ +
+
+
+ basic.rs +
+ 
use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, RelationType};
+
+// Open or create a local database — no server, no daemon
+let db = EngramDb::open(Path::new("/var/lib/agent/memory"))?;
+
+// Store a concept with a semantic embedding
+let id = db.put_node(Node::new(
+    NodeType::Concept,
+    embedding,        // Vec<f32> from your LLM
+    content,          // Vec<u8> — any payload
+    MemoryTier::Semantic,
+    0.9,             // importance 0.0–1.0
+))?;
+
+// Link to a related concept
+db.put_edge(Edge::new(id, related_id, RelationType::Causes, 0.88))?;
+
+// Retrieve by spreading activation — not a query, a pattern completion
+let results = db.activate(
+    &[id],              // seed nodes
+    &query_embedding,  // direction of thought
+    3,                 // max hops
+    10,                // top-N results
+)?;
+
+for r in &results {
+    println!("strength={:.4} hops={}", r.activation_strength, r.hops);
+}
+
+ +
+
+
+ Memory.kt +
+ 
import ai.neuron.engram.*
+
+// JVM binding via JNI — same embedded storage, no server
+val db = EngramDb.open("/data/user/0/ai.agent/memory")
+
+// Store an episodic event
+val id = db.putNode(
+    Node(
+        nodeType = NodeType.EVENT,
+        embedding = llm.embed(text),
+        content = text.toByteArray(),
+        tier = MemoryTier.EPISODIC,
+        importance = 0.8f
+    )
+)
+
+// Activate from current context
+val recalled = db.activate(
+    seeds = listOf(id),
+    queryEmbedding = currentContext.embedding,
+    maxDepth = 3,
+    limit = 10
+)
+
+recalled.forEach {
+    println("[${it.hops} hops] strength=${it.activationStrength}")
+}
+
+ +
+
+
+ memory.ts +
+ 
import { EngramDb, NodeType, MemoryTier, RelationType } from "@neuron/engram";
+
+// WASM build — runs in Node or browser, fully in-process
+const db = await EngramDb.open("./agent-memory");
+
+// Store a concept node
+const id = await db.putNode({
+  nodeType: NodeType.Concept,
+  embedding: await llm.embed(text),
+  content: new TextEncoder().encode(text),
+  tier: MemoryTier.Semantic,
+  importance: 0.85,
+});
+
+// Spreading activation — pattern completion, not query
+const recalled = await db.activate({
+  seeds: [id],
+  queryEmbedding: currentContext.embedding,
+  maxDepth: 3,
+  limit: 10,
+});
+
+for (const node of recalled) {
+  console.log(`strength=${node.activationStrength.toFixed(4)} hops=${node.hops}`);
+}
+
+ +
+
+
+ memory.go +
+ 
import (
+    engram "github.com/neuron-technologies/engram-go"
+)
+
+// CGo binding — links the Rust library, zero copies for embeddings
+db, err := engram.Open("/var/lib/agent/memory")
+if err != nil {
+    return err
+}
+defer db.Close()
+
+// Store a node
+id, err := db.PutNode(engram.Node{
+    NodeType:   engram.Concept,
+    Embedding:  embedding,
+    Content:    []byte(text),
+    Tier:       engram.Semantic,
+    Importance: 0.9,
+})
+
+// Activate from seed — the graph does the retrieval
+results, err := db.Activate(
+    []engram.UUID{id},
+    queryEmbedding,
+    3,   // maxDepth
+    10,  // limit
+)
+
+for _, r := range results {
+    fmt.Printf("strength=%.4f hops=%d\n", r.Strength, r.Hops)
+}
+
+
+
+ + +
+ Philosophy +

Why local

+ +
+ +

+ Your memory is not a service.
+ It is you.
+
+ It lives on your hardware, under your control.
+ It does not leave your device.
+ It does not phone home.
+
+ It does not require a network connection
to remember who you are. +

+ +
+ +
+
+
🔒
+ No telemetry +
+
+
📡
+ No network required +
+
+
🗄️
+ Embedded storage +
+
+
⚡
+ No daemon process +
+
+
+ + +
+ Build Status +

Current state

+ +
+
+ ◈ +
+
Version
+
v0.1.0
+
+
+
+ ✓ +
+
Build
+
Zero warnings · Zero errors
+
+
+
+ ◎ +
+
Test Suite
+
38 passing
+
+
+
+ +
+
+ Public API Surface · engram-core v0.1.0 +
+ 
impl EngramDb {
+    fn open(path: &Path)                                          -> EngramResult<Self>;
+    fn put_node(&self, node: Node)                                 -> EngramResult<Uuid>;
+    fn get_node(&self, id: Uuid)                                 -> EngramResult<Option<Node>>;
+    fn put_edge(&self, edge: Edge)                                -> EngramResult<()>;
+    fn activate(&self, seeds: &[Uuid], emb: &[f32], depth: u8, n: usize)  -> EngramResult<Vec<ActivatedNode>>;
+    fn search_embedding(&self, emb: &[f32], limit: usize)          -> EngramResult<Vec<ScoredNode>>;
+    fn traverse(&self, from: Uuid, rel: Option<RelationType>, depth: u8) -> EngramResult<Vec<Node>>;
+    fn touch(&self, id: Uuid)                                    -> EngramResult<()>;
+    fn decay(&self, factor: f32)                                  -> EngramResult<usize>;
+    fn consolidate(&self, config: &ConsolidationConfig)          -> EngramResult<ConsolidationReport>;
+}
+
+
+ +
+ engram · v0.1.0 · neuron technologies · the physical trace of a memory +
+ + + +