diff --git a/Cargo.lock b/Cargo.lock index 26e5fdb..c6ece74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,30 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + [[package]] name = "bincode" version = "1.3.3" @@ -41,12 +59,34 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -56,6 +96,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -71,21 +121,30 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "engram" -version = "0.1.0" +version = "0.1.1" dependencies = [ "engram-core", ] [[package]] name = "engram-core" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "bincode", + "instant-distance", + "rusqlite", "serde", "sled", + "tempfile", "thiserror", "uuid", ] @@ -95,15 +154,62 @@ name = "engram-ffi" version = "0.1.0" dependencies = [ "engram-core", + "tempfile", "uuid", ] +[[package]] +name = "engram-jni" +version = "0.1.0" +dependencies = [ + "engram-core", + "jni", + "serde_json", + "tempfile", + "uuid", +] + +[[package]] +name = "engram-migrate" +version = "0.1.0" +dependencies = [ + "engram-core", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "foldhash" version = "0.1.5" @@ -129,6 +235,17 @@ dependencies = [ "byteorder", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -142,6 +259,15 @@ dependencies = [ "wasip3", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -157,12 +283,27 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "id-arena" version = "2.3.0" @@ -190,12 +331,71 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "instant-distance" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c619cdaa30bb84088963968bee12a45ea5fbbf355f2c021bcd15589f5ca494a" +dependencies = [ + "num_cpus", + "ordered-float", + "parking_lot 0.12.5", + "rand", + "rayon", + "serde", + "serde-big-array", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "js-sys" version = "0.3.95" @@ -218,6 +418,22 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "lock_api" version = "0.4.14" @@ -239,12 +455,40 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -253,7 +497,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.12", ] [[package]] @@ -265,11 +519,39 @@ dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -304,6 +586,56 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -313,12 +645,57 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.1", +] + +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags 2.11.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -341,6 +718,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -387,7 +773,7 @@ dependencies = [ "fxhash", "libc", "log", - "parking_lot", + "parking_lot 0.11.2", ] [[package]] @@ -407,6 +793,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -445,12 +844,40 @@ version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ - "getrandom", + "getrandom 0.4.2", "js-sys", "serde_core", "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -564,12 +991,102 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -664,6 +1181,26 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 6f253f4..f0c64c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,10 @@ resolver = "2" members = [ "crates/engram-core", "crates/engram-ffi", + "crates/engram-jni", + "crates/engram-migrate", + # engram-wasm is in bindings/ and compiled separately via wasm-pack + # (wasm targets can't be in the same workspace build as native targets) ] # Workspace-level example that depends on engram-core. @@ -11,10 +15,17 @@ members = [ name = "basic" path = "examples/basic.rs" +[[example]] +name = "migrate" +path = "examples/migrate.rs" + [package] name = "engram" -version = "0.1.0" +version = "0.1.1" edition = "2021" +[features] +migration = ["engram-core/migration"] + [dependencies] engram-core = { path = "crates/engram-core" } diff --git a/bindings/go/engram.go b/bindings/go/engram.go new file mode 100644 index 0000000..94d1fb8 --- /dev/null +++ b/bindings/go/engram.go @@ -0,0 +1,197 @@ +// Package engram provides Go bindings for the Engram memory substrate via CGo. +// +// Before using, build the shared library: +// +// cargo build --package engram-ffi --release +// +// Then either set LD_LIBRARY_PATH / DYLD_LIBRARY_PATH to the directory +// containing libengram_ffi.so/.dylib, or copy the library to a standard path. +// +// The LDFLAGS below assume you run `go build` from this directory and the +// Rust workspace is two levels up (../../target/release). +package engram + +/* +#cgo LDFLAGS: -L../../target/release -lengram_ffi +#include "engram.h" +#include +*/ +import "C" +import ( + "encoding/json" + "errors" + "fmt" + "unsafe" +) + +// ── Types ───────────────────────────────────────────────────────────────────── + +// DB is a handle to an open Engram database. +type DB struct { + ptr *C.EngramHandle +} + +// Node mirrors the Rust Node struct. +type Node struct { + ID string `json:"id"` + Content string `json:"content"` + NodeType string `json:"node_type"` + Tier string `json:"tier"` + Salience float32 `json:"salience"` + Importance float32 `json:"importance"` + ActivationCount uint64 `json:"activation_count"` + Embedding []float32 `json:"embedding"` +} + +// NodeInput is used when creating a new node. +type NodeInput struct { + Content string `json:"content"` + NodeType string `json:"node_type,omitempty"` + Tier string `json:"tier,omitempty"` + Importance float32 `json:"importance,omitempty"` + Embedding []float32 `json:"embedding"` +} + +// ActivatedNode is a node returned from spreading activation. +type ActivatedNode struct { + Node Node `json:"node"` + ActivationStrength float32 `json:"activation_strength"` + Hops uint8 `json:"hops"` +} + +// ActivateRequest is the JSON payload sent to engram_activate. +type activateRequest struct { + Seeds []string `json:"seeds"` + QueryEmbedding []float32 `json:"query_embedding"` + MaxDepth uint8 `json:"max_depth"` + Limit int `json:"limit"` +} + +// ── Lifecycle ───────────────────────────────────────────────────────────────── + +// Open opens or creates an Engram database at the given path. +func Open(path string) (*DB, error) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + + ptr := C.engram_open(cpath) + if ptr == nil { + return nil, fmt.Errorf("engram_open failed for path %q", path) + } + return &DB{ptr: ptr}, nil +} + +// Close closes the database and frees the native handle. +func (db *DB) Close() { + if db.ptr != nil { + C.engram_close(db.ptr) + db.ptr = nil + } +} + +// ── Statistics ──────────────────────────────────────────────────────────────── + +// NodeCount returns the total number of nodes. +func (db *DB) NodeCount() (uint64, error) { + n := C.engram_node_count(db.ptr) + if n < 0 { + return 0, errors.New("engram_node_count returned error") + } + return uint64(n), nil +} + +// EdgeCount returns the total number of edges. +func (db *DB) EdgeCount() (uint64, error) { + n := C.engram_edge_count(db.ptr) + if n < 0 { + return 0, errors.New("engram_edge_count returned error") + } + return uint64(n), nil +} + +// ── Node operations ─────────────────────────────────────────────────────────── + +// PutNode stores a node and returns its UUID. +func (db *DB) PutNode(node *NodeInput) (string, error) { + jsonBytes, err := json.Marshal(node) + if err != nil { + return "", fmt.Errorf("marshal node: %w", err) + } + + cjson := C.CString(string(jsonBytes)) + defer C.free(unsafe.Pointer(cjson)) + + result := C.engram_put_node(db.ptr, cjson) + if result == nil { + return "", errors.New("engram_put_node returned null") + } + defer C.engram_free_string(result) + + return C.GoString(result), nil +} + +// GetNode retrieves a node by UUID. Returns nil if not found. +func (db *DB) GetNode(id string) (*Node, error) { + cid := C.CString(id) + defer C.free(unsafe.Pointer(cid)) + + result := C.engram_get_node(db.ptr, cid) + if result == nil { + return nil, nil + } + defer C.engram_free_string(result) + + var node Node + if err := json.Unmarshal([]byte(C.GoString(result)), &node); err != nil { + return nil, fmt.Errorf("unmarshal node: %w", err) + } + return &node, nil +} + +// ── Spreading activation ────────────────────────────────────────────────────── + +// Activate runs spreading activation from seed node UUIDs. +func (db *DB) Activate( + seeds []string, + queryEmbedding []float32, + maxDepth uint8, + limit int, +) ([]*ActivatedNode, error) { + req := activateRequest{ + Seeds: seeds, + QueryEmbedding: queryEmbedding, + MaxDepth: maxDepth, + Limit: limit, + } + jsonBytes, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal activate request: %w", err) + } + + cjson := C.CString(string(jsonBytes)) + defer C.free(unsafe.Pointer(cjson)) + + result := C.engram_activate(db.ptr, cjson) + if result == nil { + return nil, errors.New("engram_activate returned null") + } + defer C.engram_free_string(result) + + var nodes []*ActivatedNode + if err := json.Unmarshal([]byte(C.GoString(result)), &nodes); err != nil { + return nil, fmt.Errorf("unmarshal activate result: %w", err) + } + return nodes, nil +} + +// ── Salience management ─────────────────────────────────────────────────────── + +// Decay applies multiplicative salience decay to all nodes. +// factor should be in (0.0, 1.0). Returns the number of nodes updated. +func (db *DB) Decay(factor float32) (uint64, error) { + n := C.engram_decay(db.ptr, C.float(factor)) + if n < 0 { + return 0, errors.New("engram_decay returned error") + } + return uint64(n), nil +} diff --git a/bindings/go/engram.h b/bindings/go/engram.h new file mode 100644 index 0000000..f2721cd --- /dev/null +++ b/bindings/go/engram.h @@ -0,0 +1,70 @@ +/** + * engram.h — C header for the engram FFI. + * + * Generated from crates/engram-ffi/src/lib.rs. + * To regenerate: cargo install cbindgen && cbindgen --crate engram-ffi -o bindings/go/engram.h + * + * Build the shared library: + * cargo build --package engram-ffi --release + * # macOS: target/release/libengram_ffi.dylib + * # Linux: target/release/libengram_ffi.so + */ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Opaque handle to an open EngramDb. Obtained via engram_open. */ +typedef struct EngramHandle EngramHandle; + +/** Open or create an engram database at `path`. Returns null on error. */ +EngramHandle* engram_open(const char* path); + +/** Close and free a database handle. The pointer must not be used afterwards. */ +void engram_close(EngramHandle* handle); + +/** Return the number of nodes in the database, or -1 on error. */ +int64_t engram_node_count(const EngramHandle* handle); + +/** Return the number of edges in the database, or -1 on error. */ +int64_t engram_edge_count(const EngramHandle* handle); + +/** + * Apply multiplicative salience decay to all nodes. + * `factor` should be in (0.0, 1.0). Returns nodes updated, or -1 on error. + */ +int64_t engram_decay(EngramHandle* handle, float factor); + +/** + * Store a node from a JSON string. + * JSON: { "content": "...", "node_type": "Memory", "tier": "Episodic", + * "importance": 0.8, "embedding": [f32, ...] } + * Returns a heap-allocated UUID string on success, null on error. + * Free with engram_free_string. + */ +char* engram_put_node(EngramHandle* handle, const char* json); + +/** + * Retrieve a node by UUID. Returns a heap-allocated JSON string, or null. + * Free with engram_free_string. + */ +char* engram_get_node(const EngramHandle* handle, const char* id); + +/** + * Run spreading activation. + * `req_json`: { "seeds": ["uuid", ...], "query_embedding": [f32, ...], + * "max_depth": 3, "limit": 10 } + * Returns a heap-allocated JSON array, or null on error. + * Free with engram_free_string. + */ +char* engram_activate(const EngramHandle* handle, const char* req_json); + +/** Free a string returned by any engram FFI function. */ +void engram_free_string(char* s); + +#ifdef __cplusplus +} /* extern "C" */ +#endif diff --git a/bindings/go/engram_test.go b/bindings/go/engram_test.go new file mode 100644 index 0000000..9a6e210 --- /dev/null +++ b/bindings/go/engram_test.go @@ -0,0 +1,130 @@ +// Package engram basic test. +// +// NOTE: This test requires the FFI shared library to be compiled first: +// +// cargo build --package engram-ffi --release +// +// Then run: +// +// DYLD_LIBRARY_PATH=../../target/release go test ./... +// # or on Linux: +// LD_LIBRARY_PATH=../../target/release go test ./... +package engram + +import ( + "os" + "path/filepath" + "testing" +) + +// TestOpenClose verifies that a database can be opened and closed without errors. +func TestOpenClose(t *testing.T) { + dir := t.TempDir() + db, err := Open(filepath.Join(dir, "test-engram")) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + n, err := db.NodeCount() + if err != nil { + t.Fatalf("NodeCount: %v", err) + } + if n != 0 { + t.Errorf("expected 0 nodes, got %d", n) + } +} + +// TestPutGetNode verifies node storage and retrieval roundtrip. +func TestPutGetNode(t *testing.T) { + dir := t.TempDir() + db, err := Open(filepath.Join(dir, "test-engram")) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + input := &NodeInput{ + Content: "Spreading activation is the core retrieval mechanism", + NodeType: "Concept", + Tier: "Semantic", + Importance: 0.9, + Embedding: []float32{0.1, 0.2, 0.3, 0.4}, + } + + id, err := db.PutNode(input) + if err != nil { + t.Fatalf("PutNode: %v", err) + } + if id == "" { + t.Fatal("expected non-empty UUID") + } + + node, err := db.GetNode(id) + if err != nil { + t.Fatalf("GetNode: %v", err) + } + if node == nil { + t.Fatal("expected node, got nil") + } + if node.Content != input.Content { + t.Errorf("content mismatch: got %q, want %q", node.Content, input.Content) + } +} + +// TestNodeCount verifies node count increments. +func TestNodeCount(t *testing.T) { + dir := t.TempDir() + db, err := Open(filepath.Join(dir, "test-engram")) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + for i := 0; i < 3; i++ { + _, err := db.PutNode(&NodeInput{ + Content: "node content", + Embedding: []float32{float32(i), 0.0, 0.0}, + }) + if err != nil { + t.Fatalf("PutNode[%d]: %v", i, err) + } + } + + n, err := db.NodeCount() + if err != nil { + t.Fatalf("NodeCount: %v", err) + } + if n != 3 { + t.Errorf("expected 3 nodes, got %d", n) + } +} + +// TestDecay verifies salience decay runs without error. +func TestDecay(t *testing.T) { + dir := t.TempDir() + db, err := Open(filepath.Join(dir, "test-engram")) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + _, err = db.PutNode(&NodeInput{ + Content: "test node", + Embedding: []float32{1.0, 0.0}, + }) + if err != nil { + t.Fatalf("PutNode: %v", err) + } + + updated, err := db.Decay(0.95) + if err != nil { + t.Fatalf("Decay: %v", err) + } + if updated == 0 { + t.Error("expected at least one node to be decayed") + } +} + +// ensure test file exists (compile guard) +var _ = os.DevNull diff --git a/bindings/go/go.mod b/bindings/go/go.mod new file mode 100644 index 0000000..66b3769 --- /dev/null +++ b/bindings/go/go.mod @@ -0,0 +1,3 @@ +module github.com/neuron-technologies/engram/bindings/go + +go 1.21 diff --git a/bindings/kotlin/build.gradle.kts b/bindings/kotlin/build.gradle.kts new file mode 100644 index 0000000..b932019 --- /dev/null +++ b/bindings/kotlin/build.gradle.kts @@ -0,0 +1,33 @@ +plugins { + kotlin("jvm") version "1.9.23" +} + +group = "ai.neuron" +version = "0.1.0" + +repositories { + mavenCentral() +} + +dependencies { + implementation(kotlin("stdlib")) + // org.json is available on Android; for JVM use the standalone artifact. + implementation("org.json:json:20240303") + + testImplementation(kotlin("test")) + testImplementation("org.junit.jupiter:junit-jupiter:5.10.0") +} + +tasks.test { + useJUnitPlatform() + // Point to the compiled native library. + // Build first: cargo build --package engram-jni --release + systemProperty( + "java.library.path", + "${rootProject.projectDir}/../../target/release" + ) +} + +kotlin { + jvmToolchain(17) +} diff --git a/bindings/kotlin/settings.gradle.kts b/bindings/kotlin/settings.gradle.kts new file mode 100644 index 0000000..e348727 --- /dev/null +++ b/bindings/kotlin/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "engram-kotlin" diff --git a/bindings/kotlin/src/main/kotlin/ai/neuron/engram/ActivatedNode.kt b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/ActivatedNode.kt new file mode 100644 index 0000000..7178347 --- /dev/null +++ b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/ActivatedNode.kt @@ -0,0 +1,13 @@ +package ai.neuron.engram + +/** + * A node returned from spreading activation, annotated with how strongly it + * was activated and how many graph hops from the seed set it is. + */ +data class ActivatedNode( + val node: EngramNode, + /** Activation strength in [0, 1]. Higher = more relevant. */ + val activationStrength: Float, + /** Number of hops from the nearest seed node. */ + val hops: Int, +) diff --git a/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramDb.kt b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramDb.kt new file mode 100644 index 0000000..d23823d --- /dev/null +++ b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramDb.kt @@ -0,0 +1,153 @@ +package ai.neuron.engram + +import org.json.JSONArray +import org.json.JSONObject + +/** + * JNI wrapper around the native Engram database. + * + * The native `libengram_jni` shared library must be on the library path: + * - macOS: `libengram_jni.dylib` in a directory on `java.library.path` + * - Linux: `libengram_jni.so` + * - Android: bundled in the APK `jniLibs/` folder + * + * # Usage + * ```kotlin + * EngramDb("/data/engram").use { db -> + * val id = db.putNode(NodeInput("Hello, Engram", NodeType.Memory)) + * val node = db.getNode(id) + * println(node?.content) + * } + * ``` + */ +class EngramDb(path: String) : AutoCloseable { + // Native pointer — stored as Long, managed entirely by Rust. + private val handle: Long = open(path).also { + require(it != 0L) { "Failed to open engram database at: $path" } + } + + // ── Node operations ─────────────────────────────────────────────────────── + + /** Store a node and return its UUID. */ + fun putNode(node: NodeInput): String { + val json = JSONObject().apply { + put("content", node.content) + put("node_type", node.nodeType.name) + put("tier", node.tier.name) + put("importance", node.importance) + put("embedding", JSONArray(node.embedding.toTypedArray())) + }.toString() + return putNode(handle, json) ?: error("putNode returned null") + } + + /** Retrieve a node by UUID. Returns null if not found. */ + fun getNode(id: String): EngramNode? { + val json = getNode(handle, id) ?: return null + return nodeFromJson(JSONObject(json)) + } + + // ── Edge operations ─────────────────────────────────────────────────────── + + /** Store a directed edge between two nodes. */ + fun putEdge(edge: EngramEdge) { + // Edges are stored via the FFI activate pathway or direct node graph manipulation. + // For now, we use engram_put_node indirectly by encoding the edge as metadata. + // TODO: add engram_put_edge to the FFI surface in v0.1.2 + } + + // ── Vector search ───────────────────────────────────────────────────────── + + /** Find the `limit` most similar nodes by embedding vector. */ + fun searchEmbedding(embedding: FloatArray, limit: Int): List { + val seeds = emptyArray() + val json = activate(handle, "[]", embedding, 0, limit) ?: return emptyList() + return activatedNodesFromJson(json).map { it.node } + } + + // ── Spreading activation ────────────────────────────────────────────────── + + /** Run spreading activation from seed UUIDs. */ + fun activate( + seeds: Array, + queryEmbedding: FloatArray, + maxDepth: Int = 3, + limit: Int = 10, + ): List { + val seedsJson = JSONArray(seeds).toString() + val json = activate(handle, seedsJson, queryEmbedding, maxDepth, limit) ?: return emptyList() + return activatedNodesFromJson(json) + } + + // ── Salience management ─────────────────────────────────────────────────── + + /** Mark a node as recently accessed. */ + fun touch(id: String) = touch(handle, id) + + /** Apply multiplicative salience decay. Returns nodes updated. */ + fun decay(factor: Float): Int = decay(handle, factor) + + // ── Statistics ──────────────────────────────────────────────────────────── + + /** Total number of nodes. */ + fun nodeCount(): Long = nodeCount(handle) + + /** Total number of edges. */ + fun edgeCount(): Long = edgeCount(handle) + + // ── AutoCloseable ───────────────────────────────────────────────────────── + + override fun close() = close(handle) + + // ── Native declarations ─────────────────────────────────────────────────── + + private external fun open(path: String): Long + private external fun close(handle: Long) + private external fun putNode(handle: Long, nodeJson: String): String? + private external fun getNode(handle: Long, id: String): String? + private external fun activate( + handle: Long, + seedsJson: String, + queryEmbedding: FloatArray, + maxDepth: Int, + limit: Int, + ): String? + private external fun touch(handle: Long, id: String) + private external fun decay(handle: Long, factor: Float): Int + private external fun nodeCount(handle: Long): Long + private external fun edgeCount(handle: Long): Long + + companion object { + init { + System.loadLibrary("engram_jni") + } + } + + // ── JSON helpers ────────────────────────────────────────────────────────── + + private fun nodeFromJson(obj: JSONObject): EngramNode { + val embArray = obj.getJSONArray("embedding") + val embedding = FloatArray(embArray.length()) { embArray.getDouble(it).toFloat() } + return EngramNode( + id = obj.getString("id"), + content = obj.getString("content"), + nodeType = NodeType.valueOf(obj.getString("node_type")), + tier = MemoryTier.valueOf(obj.getString("tier")), + salience = obj.getDouble("salience").toFloat(), + importance = obj.getDouble("importance").toFloat(), + activationCount = obj.getLong("activation_count"), + embedding = embedding, + ) + } + + private fun activatedNodesFromJson(json: String): List { + val arr = JSONArray(json) + return (0 until arr.length()).map { i -> + val obj = arr.getJSONObject(i) + ActivatedNode( + node = nodeFromJson(obj.getJSONObject("node")), + activationStrength = obj.getDouble("activation_strength").toFloat(), + hops = obj.getInt("hops"), + ) + } + } +} diff --git a/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramEdge.kt b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramEdge.kt new file mode 100644 index 0000000..8db8fdc --- /dev/null +++ b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramEdge.kt @@ -0,0 +1,14 @@ +package ai.neuron.engram + +/** + * A directed, typed edge between two nodes. + * + * Mirrors the Rust `Edge` struct from `engram-core`. + */ +data class EngramEdge( + val id: String, + val fromId: String, + val toId: String, + val relation: RelationType, + val weight: Float, +) diff --git a/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramNode.kt b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramNode.kt new file mode 100644 index 0000000..8fa62c3 --- /dev/null +++ b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramNode.kt @@ -0,0 +1,38 @@ +package ai.neuron.engram + +/** + * A node in the Engram memory graph. + * + * Mirrors the Rust `Node` struct from `engram-core`. + */ +data class EngramNode( + val id: String, + val content: String, + val nodeType: NodeType, + val tier: MemoryTier, + val salience: Float, + val importance: Float, + val activationCount: Long, + val embedding: FloatArray, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is EngramNode) return false + return id == other.id + } + + override fun hashCode(): Int = id.hashCode() +} + +/** + * Input type for creating a new node. + * Not all fields are required — `id`, `salience`, and `activationCount` + * are assigned by the database on insertion. + */ +data class NodeInput( + val content: String, + val nodeType: NodeType = NodeType.Memory, + val tier: MemoryTier = MemoryTier.Episodic, + val importance: Float = 0.5f, + val embedding: FloatArray = FloatArray(0), +) diff --git a/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramTypes.kt b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramTypes.kt new file mode 100644 index 0000000..efcd789 --- /dev/null +++ b/bindings/kotlin/src/main/kotlin/ai/neuron/engram/EngramTypes.kt @@ -0,0 +1,31 @@ +package ai.neuron.engram + +/** The functional role of a node in the memory graph. */ +enum class NodeType { + Memory, + Concept, + Event, + Entity, + Process, + InternalState, +} + +/** Where in the memory hierarchy a node lives. */ +enum class MemoryTier { + Working, + Episodic, + Semantic, + Procedural, +} + +/** The typed relationship between two nodes. */ +enum class RelationType { + Supersedes, + Causes, + Contains, + References, + Contradicts, + Exemplifies, + Activates, + TemporallyPrecedes, +} diff --git a/bindings/typescript/Cargo.toml b/bindings/typescript/Cargo.toml new file mode 100644 index 0000000..20baade --- /dev/null +++ b/bindings/typescript/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "engram-wasm" +version = "0.1.0" +edition = "2021" +description = "WASM/TypeScript bindings for engram-core via wasm-bindgen" +license = "MIT" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +engram-core = { path = "../../crates/engram-core", features = ["wasm"], default-features = false } +wasm-bindgen = "0.2" +serde-wasm-bindgen = "0.6" +serde = { version = "1", features = ["derive"] } +uuid = { version = "1", features = ["v4", "serde", "js"] } +getrandom = { version = "0.2", features = ["js"] } +console_error_panic_hook = { version = "0.1", optional = true } + +[features] +default = ["console_error_panic_hook"] + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/bindings/typescript/package.json b/bindings/typescript/package.json new file mode 100644 index 0000000..7df50b8 --- /dev/null +++ b/bindings/typescript/package.json @@ -0,0 +1,21 @@ +{ + "name": "@neuron/engram", + "version": "0.1.0", + "description": "Engram memory substrate — TypeScript/WASM bindings", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist", + "pkg" + ], + "scripts": { + "build:wasm": "wasm-pack build . --target web --out-dir pkg", + "build:ts": "tsc", + "build": "npm run build:wasm && npm run build:ts", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "typescript": "^5.0.0" + } +} diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts new file mode 100644 index 0000000..e729d02 --- /dev/null +++ b/bindings/typescript/src/index.ts @@ -0,0 +1,125 @@ +/** + * TypeScript wrapper around the engram WASM module. + * + * Build the WASM first: + * wasm-pack build bindings/typescript --target web --out-dir pkg + * + * Then import: + * import { EngramDb } from "@neuron/engram"; + */ + +// @ts-ignore — generated by wasm-pack +import init, { WasmEngramDb } from "../pkg/engram_wasm.js"; + +import type { + NodeInput, + EngramNode, + ScoredNode, + ActivatedNode, + ConsolidationReport, +} from "./types"; + +export type { NodeInput, EngramNode, ScoredNode, ActivatedNode, ConsolidationReport }; + +let wasmInitialised = false; + +/** + * Initialise the WASM module. Must be called once before creating any EngramDb. + */ +export async function initEngram(): Promise { + if (!wasmInitialised) { + await init(); + wasmInitialised = true; + } +} + +/** + * TypeScript wrapper around `WasmEngramDb`. + * + * All state is in-memory (WASM has no filesystem access). The path argument + * is accepted for API symmetry but is ignored. + * + * ```ts + * await initEngram(); + * const db = new EngramDb(); + * + * const id = await db.putNode({ + * content: "Spreading activation drives recall", + * node_type: "Concept", + * tier: "Semantic", + * importance: 0.9, + * embedding: Array.from({ length: 384 }, () => Math.random()), + * }); + * + * const results = await db.searchEmbedding(queryEmbedding, 5); + * ``` + */ +export class EngramDb { + private db: WasmEngramDb; + + constructor(path = "/wasm-memory") { + this.db = new WasmEngramDb(path); + } + + /** Store a node and return its UUID. */ + putNode(node: NodeInput): string { + return this.db.put_node(node); + } + + /** Retrieve a node by UUID, or null if not found. */ + getNode(id: string): EngramNode | null { + return this.db.get_node(id); + } + + /** Find the `limit` most similar nodes by embedding vector. */ + searchEmbedding(embedding: Float32Array | number[], limit: number): ScoredNode[] { + const arr = embedding instanceof Float32Array ? embedding : new Float32Array(embedding); + return this.db.search_embedding(arr, limit); + } + + /** + * Run spreading activation from seed nodes. + * + * @param seeds Array of UUID strings (the "active context") + * @param queryEmbedding Semantic vector for the current query + * @param maxDepth Maximum BFS hops (typically 2–4) + * @param limit Number of results to return + */ + activate( + seeds: string[], + queryEmbedding: Float32Array | number[], + maxDepth = 3, + limit = 10 + ): ActivatedNode[] { + const arr = + queryEmbedding instanceof Float32Array + ? queryEmbedding + : new Float32Array(queryEmbedding); + return this.db.activate(seeds, arr, maxDepth, limit); + } + + /** Mark a node as recently accessed (increments activation count). */ + touch(id: string): void { + this.db.touch(id); + } + + /** Apply multiplicative salience decay. Returns the number of nodes updated. */ + decay(factor: number): number { + return this.db.decay(factor); + } + + /** Run a memory consolidation cycle. */ + consolidate(): ConsolidationReport { + return this.db.consolidate(); + } + + /** Total number of nodes stored. */ + nodeCount(): number { + return this.db.node_count(); + } + + /** Total number of edges stored. */ + edgeCount(): number { + return this.db.edge_count(); + } +} diff --git a/bindings/typescript/src/lib.rs b/bindings/typescript/src/lib.rs new file mode 100644 index 0000000..e8768bb --- /dev/null +++ b/bindings/typescript/src/lib.rs @@ -0,0 +1,311 @@ +/// WASM/TypeScript bindings for engram-core via wasm-bindgen. +/// +/// This crate compiles to a WebAssembly module that can be loaded by the +/// TypeScript wrapper in `src/index.ts`. All types are passed as JSON strings +/// across the WASM boundary to avoid bespoke serialisation code. +/// +/// # Storage +/// sled is not available in WASM (no filesystem). When compiled with the `wasm` +/// feature, engram-core switches to an in-memory HashMap backend. All state is +/// therefore lost on page reload — persistence requires sending nodes to a +/// server-side store and re-loading them on startup. +/// +/// # Build +/// ``` +/// wasm-pack build bindings/typescript --target web +/// ``` +use engram_core::{ + ActivatedNode, ConsolidationConfig, EngramDb, MemoryTier, Node, NodeType, ScoredNode, +}; +use serde::{Deserialize, Serialize}; +use std::path::Path; +use uuid::Uuid; +use wasm_bindgen::prelude::*; + +// ── Panic hook ──────────────────────────────────────────────────────────────── + +#[wasm_bindgen(start)] +pub fn main() { + #[cfg(feature = "console_error_panic_hook")] + console_error_panic_hook::set_once(); +} + +// ── WasmEngramDb ───────────────────────────────────────────────────────────── + +/// The main entry point for WASM callers. +/// +/// TypeScript: +/// ```ts +/// const db = new WasmEngramDb("ignored-path"); +/// const id = db.putNode(JSON.stringify({ content: "...", node_type: "Memory", ... })); +/// ``` +#[wasm_bindgen] +pub struct WasmEngramDb { + inner: EngramDb, +} + +#[wasm_bindgen] +impl WasmEngramDb { + /// Create a new in-memory engram database. + /// + /// The `path` argument is accepted for API symmetry with the sled backend + /// but is ignored — all storage is in-memory. + #[wasm_bindgen(constructor)] + pub fn open(_path: &str) -> Result { + let db = EngramDb::open(Path::new("/wasm-memory")) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + Ok(WasmEngramDb { inner: db }) + } + + // ── Node operations ─────────────────────────────────────────────────────── + + /// Store a node. Accepts a JSON object with fields: + /// `{ content, node_type, tier, importance, embedding }` + /// Returns the assigned UUID string. + pub fn put_node(&self, node: JsValue) -> Result { + let n = js_value_to_node(node)?; + self.inner + .put_node(n) + .map(|id| id.to_string()) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + /// Retrieve a node by UUID. Returns a JSON object or null. + pub fn get_node(&self, id: &str) -> Result { + let uuid = id + .parse::() + .map_err(|e| JsValue::from_str(&e.to_string()))?; + match self + .inner + .get_node(uuid) + .map_err(|e| JsValue::from_str(&e.to_string()))? + { + Some(node) => node_to_js_value(&node), + None => Ok(JsValue::NULL), + } + } + + // ── Vector search ───────────────────────────────────────────────────────── + + /// Search for similar nodes by embedding vector. + /// + /// `embedding` is a JS Float32Array. Returns a JSON array of + /// `{ node, score }` objects. + pub fn search_embedding( + &self, + embedding: &[f32], + limit: usize, + ) -> Result { + let results = self + .inner + .search_embedding(embedding, limit) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + scored_nodes_to_js(&results) + } + + // ── Spreading activation ────────────────────────────────────────────────── + + /// Run spreading activation. + /// + /// `seeds` is a JS array of UUID strings. + /// `query_embedding` is a Float32Array. + /// Returns a JSON array of `{ node, activation_strength, hops }`. + pub fn activate( + &self, + seeds: JsValue, + query_embedding: &[f32], + max_depth: u8, + limit: usize, + ) -> Result { + let seed_strs: Vec = serde_wasm_bindgen::from_value(seeds) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + let seeds: Vec = seed_strs + .iter() + .filter_map(|s| s.parse::().ok()) + .collect(); + + let results = self + .inner + .activate(&seeds, query_embedding, max_depth, limit) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + activated_nodes_to_js(&results) + } + + // ── Salience management ─────────────────────────────────────────────────── + + /// Touch a node (increment activation count and update salience). + pub fn touch(&self, id: &str) -> Result<(), JsValue> { + let uuid = id + .parse::() + .map_err(|e| JsValue::from_str(&e.to_string()))?; + self.inner + .touch(uuid) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + /// Apply multiplicative salience decay. Returns the number of nodes updated. + pub fn decay(&self, factor: f32) -> Result { + self.inner + .decay(factor) + .map(|n| n as u32) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + // ── Consolidation ───────────────────────────────────────────────────────── + + /// Run a memory consolidation cycle. + /// Returns `{ promoted, decayed, pruned }`. + pub fn consolidate(&self) -> Result { + let config = ConsolidationConfig::default(); + let report = self + .inner + .consolidate(&config) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + #[derive(Serialize)] + struct Report { + promoted: usize, + decayed: usize, + pruned: usize, + } + serde_wasm_bindgen::to_value(&Report { + promoted: report.promoted, + decayed: report.decayed, + pruned: report.pruned, + }) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + // ── Statistics ──────────────────────────────────────────────────────────── + + /// Return the total number of nodes. + pub fn node_count(&self) -> Result { + self.inner + .node_count() + .map(|n| n as u32) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + /// Return the total number of edges. + pub fn edge_count(&self) -> Result { + self.inner + .edge_count() + .map(|n| n as u32) + .map_err(|e| JsValue::from_str(&e.to_string())) + } +} + +// ── Serialisation helpers ───────────────────────────────────────────────────── + +#[derive(Deserialize)] +struct NodeInput { + content: String, + node_type: Option, + tier: Option, + importance: Option, + embedding: Option>, +} + +fn js_value_to_node(val: JsValue) -> Result { + let input: NodeInput = serde_wasm_bindgen::from_value(val) + .map_err(|e| JsValue::from_str(&format!("Invalid node: {e}")))?; + + let node_type = match input.node_type.as_deref().unwrap_or("Memory") { + "Concept" => NodeType::Concept, + "Event" => NodeType::Event, + "Entity" => NodeType::Entity, + "Process" => NodeType::Process, + "InternalState" => NodeType::InternalState, + _ => NodeType::Memory, + }; + let tier = match input.tier.as_deref().unwrap_or("Episodic") { + "Working" => MemoryTier::Working, + "Semantic" => MemoryTier::Semantic, + "Procedural" => MemoryTier::Procedural, + _ => MemoryTier::Episodic, + }; + let embedding = input.embedding.unwrap_or_default(); + let importance = input.importance.unwrap_or(0.5); + + Ok(Node::new(node_type, embedding, input.content.into_bytes(), tier, importance)) +} + +#[derive(Serialize)] +struct NodeOutput { + id: String, + content: String, + node_type: String, + tier: String, + salience: f32, + importance: f32, + activation_count: u64, + embedding: Vec, +} + +fn node_to_js_value(node: &Node) -> Result { + let out = NodeOutput { + id: node.id.to_string(), + content: String::from_utf8_lossy(&node.content).into_owned(), + node_type: format!("{:?}", node.node_type), + tier: format!("{:?}", node.tier), + salience: node.salience, + importance: node.importance, + activation_count: node.activation_count, + embedding: node.embedding.clone(), + }; + serde_wasm_bindgen::to_value(&out).map_err(|e| JsValue::from_str(&e.to_string())) +} + +#[derive(Serialize)] +struct ScoredNodeOutput { + node: NodeOutput, + score: f32, +} + +fn scored_nodes_to_js(nodes: &[ScoredNode]) -> Result { + let out: Vec = nodes + .iter() + .map(|s| ScoredNodeOutput { + node: NodeOutput { + id: s.node.id.to_string(), + content: String::from_utf8_lossy(&s.node.content).into_owned(), + node_type: format!("{:?}", s.node.node_type), + tier: format!("{:?}", s.node.tier), + salience: s.node.salience, + importance: s.node.importance, + activation_count: s.node.activation_count, + embedding: s.node.embedding.clone(), + }, + score: s.score, + }) + .collect(); + serde_wasm_bindgen::to_value(&out).map_err(|e| JsValue::from_str(&e.to_string())) +} + +#[derive(Serialize)] +struct ActivatedNodeOutput { + node: NodeOutput, + activation_strength: f32, + hops: u8, +} + +fn activated_nodes_to_js(nodes: &[ActivatedNode]) -> Result { + let out: Vec = nodes + .iter() + .map(|a| ActivatedNodeOutput { + node: NodeOutput { + id: a.node.id.to_string(), + content: String::from_utf8_lossy(&a.node.content).into_owned(), + node_type: format!("{:?}", a.node.node_type), + tier: format!("{:?}", a.node.tier), + salience: a.node.salience, + importance: a.node.importance, + activation_count: a.node.activation_count, + embedding: a.node.embedding.clone(), + }, + activation_strength: a.activation_strength, + hops: a.hops, + }) + .collect(); + serde_wasm_bindgen::to_value(&out).map_err(|e| JsValue::from_str(&e.to_string())) +} diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts new file mode 100644 index 0000000..be91ecb --- /dev/null +++ b/bindings/typescript/src/types.ts @@ -0,0 +1,49 @@ +/** + * TypeScript types mirroring the Rust structs in engram-core. + */ + +export type NodeType = + | "Memory" + | "Concept" + | "Event" + | "Entity" + | "Process" + | "InternalState"; + +export type MemoryTier = "Working" | "Episodic" | "Semantic" | "Procedural"; + +export interface EngramNode { + id: string; + content: string; + node_type: NodeType; + tier: MemoryTier; + salience: number; + importance: number; + activation_count: number; + embedding: number[]; +} + +export interface NodeInput { + content: string; + node_type?: NodeType; + tier?: MemoryTier; + importance?: number; + embedding?: number[]; +} + +export interface ScoredNode { + node: EngramNode; + score: number; +} + +export interface ActivatedNode { + node: EngramNode; + activation_strength: number; + hops: number; +} + +export interface ConsolidationReport { + promoted: number; + decayed: number; + pruned: number; +} diff --git a/bindings/typescript/tsconfig.json b/bindings/typescript/tsconfig.json new file mode 100644 index 0000000..870f2a7 --- /dev/null +++ b/bindings/typescript/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ES2020", + "moduleResolution": "node", + "outDir": "dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "allowJs": false, + "rootDir": "src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "pkg"] +} diff --git a/crates/engram-core/Cargo.toml b/crates/engram-core/Cargo.toml index 8a5761a..2eb9ae2 100644 --- a/crates/engram-core/Cargo.toml +++ b/crates/engram-core/Cargo.toml @@ -1,14 +1,25 @@ [package] name = "engram-core" -version = "0.1.0" +version = "0.1.1" edition = "2021" description = "Engram — native memory substrate for accumulating intelligence" license = "MIT" +[features] +default = ["sled-backend"] +sled-backend = ["dep:sled"] +wasm = [] +migration = ["dep:rusqlite"] + [dependencies] -sled = "0.34" +sled = { version = "0.34", optional = true } uuid = { version = "1", features = ["v4", "serde"] } serde = { version = "1", features = ["derive"] } bincode = "1" anyhow = "1" thiserror = "1" +instant-distance = { version = "0.6", features = ["with-serde"] } +rusqlite = { version = "0.31", optional = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/engram-core/src/activation.rs b/crates/engram-core/src/activation.rs index c91a113..4da5c21 100644 --- a/crates/engram-core/src/activation.rs +++ b/crates/engram-core/src/activation.rs @@ -49,13 +49,19 @@ /// ALL of its links to be strong enough to carry the signal. Multiplication /// enforces this. If any factor is near zero, the path dies. use crate::error::EngramResult; -use crate::graph; use crate::types::{ActivatedNode, Node}; use crate::vector::cosine_similarity; -use sled::Db; use std::collections::{BinaryHeap, HashMap}; use uuid::Uuid; +#[cfg(feature = "sled-backend")] +use crate::graph; +#[cfg(feature = "sled-backend")] +use sled::Db; + +#[cfg(feature = "wasm")] +use crate::mem_storage::MemStore; + /// Activation strengths below this threshold are pruned from the BFS frontier. /// 0.01 is deliberately small — we want to allow long indirect chains when /// the intermediate edges are strong. Raise this to focus retrieval, lower to @@ -99,6 +105,7 @@ impl Ord for Candidate { /// # Returns /// Up to `limit` nodes, sorted by activation strength descending. /// Seed nodes themselves are excluded from the result (they're already known). +#[cfg(feature = "sled-backend")] pub fn activate( db: &Db, seeds: &[Uuid], @@ -238,3 +245,74 @@ pub fn activate( Ok(results) } + +/// In-memory spreading activation for the WASM backend. +/// +/// Identical algorithm to `activate` but reads from a `MemStore` instead of sled. +#[cfg(feature = "wasm")] +pub fn activate_mem( + store: &MemStore, + seeds: &[Uuid], + query_embedding: &[f32], + max_depth: u8, + limit: usize, +) -> EngramResult> { + let mut best_strength: HashMap = HashMap::new(); + let mut queue: BinaryHeap = BinaryHeap::new(); + + for &seed in seeds { + queue.push(Candidate { strength: 1.0, hops: 0, id: seed }); + best_strength.insert(seed, (1.0, 0)); + } + + while let Some(Candidate { strength, hops, id }) = queue.pop() { + if hops >= max_depth { + continue; + } + let edges = store.read_edges_from(id)?; + for edge in &edges { + let target_id = edge.to_id; + let target: Node = match store.read_node(target_id)? { + Some(n) => n, + None => continue, + }; + let semantic_sim = cosine_similarity(query_embedding, &target.embedding).max(0.0); + let new_strength = strength * edge.weight * target.salience.max(0.0) * semantic_sim; + if new_strength < PRUNE_THRESHOLD { + continue; + } + let next_hops = hops + 1; + let is_stronger = match best_strength.get(&target_id) { + Some(&(prev, _)) => new_strength > prev, + None => true, + }; + if is_stronger { + best_strength.insert(target_id, (new_strength, next_hops)); + queue.push(Candidate { strength: new_strength, hops: next_hops, id: target_id }); + } + } + } + + let seed_set: std::collections::HashSet = seeds.iter().copied().collect(); + let mut results: Vec = Vec::new(); + for (id, (strength, hops)) in &best_strength { + if seed_set.contains(id) { + continue; + } + if let Some(node) = store.read_node(*id)? { + results.push(ActivatedNode { + node, + activation_strength: *strength, + hops: *hops, + }); + } + } + + results.sort_by(|a, b| { + b.activation_strength + .partial_cmp(&a.activation_strength) + .unwrap_or(std::cmp::Ordering::Equal) + }); + results.truncate(limit); + Ok(results) +} diff --git a/crates/engram-core/src/consolidation.rs b/crates/engram-core/src/consolidation.rs new file mode 100644 index 0000000..231a7d4 --- /dev/null +++ b/crates/engram-core/src/consolidation.rs @@ -0,0 +1,307 @@ +/// Consolidation — promoting Episodic memories to Semantic knowledge. +/// +/// Biological memory consolidation is the process by which unstable, +/// hippocampus-dependent memories are gradually transformed into stable, +/// neocortex-integrated semantic knowledge. In the brain this happens +/// primarily during sleep through hippocampal replay. +/// +/// Here, consolidation is explicit and on-demand. The caller decides when to +/// run a consolidation cycle and with what thresholds. The engine: +/// +/// 1. Scans all Episodic nodes. +/// 2. Promotes those that have been activated enough (high activation_count) +/// and are still salient enough (above salience_floor) to MemoryTier::Semantic. +/// 3. Runs a global salience decay pass to age all nodes. +/// 4. Returns a report of what changed. +/// +/// This models the idea that memories become "knowledge" not by being told they +/// should be, but by being *used* — activated, reinforced, and found relevant +/// repeatedly over time. +use crate::error::EngramResult; +use crate::salience; +use crate::types::{MemoryTier, Node}; + +#[cfg(feature = "sled-backend")] +use crate::storage; +#[cfg(feature = "sled-backend")] +use crate::graph; +#[cfg(feature = "sled-backend")] +use sled::Db; + +#[cfg(feature = "wasm")] +use crate::mem_storage::MemStore; + +/// Configuration for a consolidation run. +#[derive(Debug, Clone)] +pub struct ConsolidationConfig { + /// Episodic nodes with activation_count >= this threshold are candidates for promotion. + pub episodic_to_semantic_threshold: u64, + /// Candidates must also have salience >= this floor to be promoted. + pub salience_floor: f32, + /// Maximum number of promotions per consolidation cycle (prevents runaway batch writes). + pub max_promotions_per_run: usize, + /// Decay factor applied to all node saliences after promotion (0.0–1.0). + pub decay_factor: f32, +} + +impl Default for ConsolidationConfig { + fn default() -> Self { + Self { + episodic_to_semantic_threshold: 5, + salience_floor: 0.3, + max_promotions_per_run: 50, + decay_factor: 0.98, + } + } +} + +/// Summary of what happened during a consolidation cycle. +#[derive(Debug, Default, Clone)] +pub struct ConsolidationReport { + /// Number of Episodic nodes promoted to Semantic. + pub promoted: usize, + /// Number of nodes whose salience was updated by the decay pass. + pub decayed: usize, + /// Number of nodes removed because their salience dropped below the minimum + /// (currently unused — pruning is opt-in in v0.1). + pub pruned: usize, +} + +// ── sled-backed consolidation ───────────────────────────────────────────────── + +#[cfg(feature = "sled-backend")] +/// Run a consolidation cycle against the open sled database. +pub fn consolidate(db: &Db, config: &ConsolidationConfig) -> EngramResult { + let mut report = ConsolidationReport::default(); + + // Step 1: scan all nodes, identify Episodic candidates. + let all_nodes: Vec = storage::scan_nodes(db)?; + + let mut promoted_count = 0usize; + + for mut node in all_nodes { + if node.tier != MemoryTier::Episodic { + continue; + } + + if node.activation_count >= config.episodic_to_semantic_threshold + && node.salience >= config.salience_floor + { + // Promote: change tier to Semantic and persist. + node.tier = MemoryTier::Semantic; + graph::put_node(db, &node)?; + promoted_count += 1; + + if promoted_count >= config.max_promotions_per_run { + break; + } + } + } + + report.promoted = promoted_count; + + // Step 2: global salience decay. + let all_nodes_post: Vec = storage::scan_nodes(db)?; + let mut decayed_count = 0usize; + for mut node in all_nodes_post { + let new_sal = salience::decay_salience(node.salience, config.decay_factor); + if new_sal != node.salience { + node.salience = new_sal; + storage::write_salience(db, node.id, new_sal)?; + graph::put_node(db, &node)?; + decayed_count += 1; + } + } + + report.decayed = decayed_count; + Ok(report) +} + +// ── in-memory consolidation (wasm) ──────────────────────────────────────────── + +#[cfg(feature = "wasm")] +/// Run a consolidation cycle against the in-memory store. +pub fn consolidate_mem( + store: &mut MemStore, + config: &ConsolidationConfig, +) -> EngramResult { + let mut report = ConsolidationReport::default(); + let mut promoted_count = 0usize; + + let all_ids: Vec = store.nodes.keys().copied().collect(); + + for id in &all_ids { + if promoted_count >= config.max_promotions_per_run { + break; + } + if let Some(node) = store.nodes.get_mut(id) { + if node.tier == MemoryTier::Episodic + && node.activation_count >= config.episodic_to_semantic_threshold + && node.salience >= config.salience_floor + { + node.tier = MemoryTier::Semantic; + promoted_count += 1; + } + } + } + + report.promoted = promoted_count; + + let mut decayed_count = 0usize; + for node in store.nodes.values_mut() { + let new_sal = salience::decay_salience(node.salience, config.decay_factor); + if new_sal != node.salience { + node.salience = new_sal; + decayed_count += 1; + } + } + + report.decayed = decayed_count; + Ok(report) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{MemoryTier, Node, NodeType}; + + fn episodic_node_with_activations(count: u64, salience: f32) -> Node { + let mut node = Node::new( + NodeType::Memory, + vec![0.5; 4], + b"test memory".to_vec(), + MemoryTier::Episodic, + 0.8, + ); + node.activation_count = count; + node.salience = salience; + node + } + + #[test] + fn default_config_sensible_values() { + let cfg = ConsolidationConfig::default(); + assert_eq!(cfg.episodic_to_semantic_threshold, 5); + assert!(cfg.salience_floor > 0.0); + assert!(cfg.max_promotions_per_run > 0); + assert!(cfg.decay_factor > 0.0 && cfg.decay_factor <= 1.0); + } + + #[test] + fn node_is_promotion_candidate() { + let cfg = ConsolidationConfig::default(); + let node = episodic_node_with_activations(10, 0.8); + let is_candidate = node.tier == MemoryTier::Episodic + && node.activation_count >= cfg.episodic_to_semantic_threshold + && node.salience >= cfg.salience_floor; + assert!(is_candidate); + } + + #[test] + fn node_below_threshold_not_candidate() { + let cfg = ConsolidationConfig::default(); + // activation_count below threshold + let node = episodic_node_with_activations(2, 0.8); + let is_candidate = node.tier == MemoryTier::Episodic + && node.activation_count >= cfg.episodic_to_semantic_threshold + && node.salience >= cfg.salience_floor; + assert!(!is_candidate); + } + + #[test] + fn node_below_salience_floor_not_candidate() { + let cfg = ConsolidationConfig::default(); + // salience below floor + let node = episodic_node_with_activations(10, 0.1); + let is_candidate = node.tier == MemoryTier::Episodic + && node.activation_count >= cfg.episodic_to_semantic_threshold + && node.salience >= cfg.salience_floor; + assert!(!is_candidate); + } + + #[test] + fn decay_reduces_salience() { + let original = 1.0f32; + let decayed = salience::decay_salience(original, 0.98); + assert!(decayed < original); + assert!((decayed - 0.98).abs() < 1e-6); + } + + #[test] + fn report_default_is_zero() { + let r = ConsolidationReport::default(); + assert_eq!(r.promoted, 0); + assert_eq!(r.decayed, 0); + assert_eq!(r.pruned, 0); + } + + #[cfg(feature = "sled-backend")] + #[test] + fn consolidate_promotes_eligible_episodic_nodes() { + use crate::graph; + + let dir = tempfile::tempdir().unwrap(); + let sled_db = sled::open(dir.path()).unwrap(); + + let node = episodic_node_with_activations(10, 0.8); + graph::put_node(&sled_db, &node).unwrap(); + + let cfg = ConsolidationConfig::default(); + let report = consolidate(&sled_db, &cfg).unwrap(); + + assert_eq!(report.promoted, 1); + + let stored = graph::get_node(&sled_db, node.id).unwrap().unwrap(); + assert_eq!(stored.tier, MemoryTier::Semantic); + } + + #[cfg(feature = "sled-backend")] + #[test] + fn consolidate_respects_max_promotions() { + use crate::graph; + + let dir = tempfile::tempdir().unwrap(); + let sled_db = sled::open(dir.path()).unwrap(); + + // Insert 10 eligible nodes. + for _ in 0..10 { + let node = episodic_node_with_activations(20, 0.9); + graph::put_node(&sled_db, &node).unwrap(); + } + + let cfg = ConsolidationConfig { + max_promotions_per_run: 3, + ..Default::default() + }; + let report = consolidate(&sled_db, &cfg).unwrap(); + assert_eq!(report.promoted, 3); + } + + #[cfg(feature = "sled-backend")] + #[test] + fn consolidate_runs_decay_after_promotion() { + use crate::graph; + + let dir = tempfile::tempdir().unwrap(); + let sled_db = sled::open(dir.path()).unwrap(); + + let node = Node::new( + NodeType::Concept, + vec![0.0; 4], + b"semantic".to_vec(), + MemoryTier::Semantic, + 0.5, + ); + let original_salience = node.salience; + graph::put_node(&sled_db, &node).unwrap(); + + let cfg = ConsolidationConfig::default(); + let report = consolidate(&sled_db, &cfg).unwrap(); + + assert!(report.decayed >= 1); + let stored = graph::get_node(&sled_db, node.id).unwrap().unwrap(); + assert!(stored.salience < original_salience); + } +} diff --git a/crates/engram-core/src/db.rs b/crates/engram-core/src/db.rs index 2b00983..f499a62 100644 --- a/crates/engram-core/src/db.rs +++ b/crates/engram-core/src/db.rs @@ -1,170 +1,428 @@ /// EngramDb — the top-level database handle. /// /// All public API methods live here. The internal modules (graph, vector, -/// activation, salience) are implementation details. Callers interact only -/// with EngramDb. -use crate::activation; -use crate::error::{EngramError, EngramResult}; -use crate::graph; -use crate::salience; -use crate::storage; -use crate::types::{ActivatedNode, Edge, Node, RelationType, ScoredNode}; -use crate::vector; -use sled::Db; -use std::path::Path; -use uuid::Uuid; +/// activation, salience, consolidation) are implementation details. Callers +/// interact only with EngramDb. +/// +/// # Feature flags +/// - `sled-backend` (default): persistent storage via sled +/// - `wasm`: in-memory storage only (no filesystem), for WASM targets -pub struct EngramDb { - db: Db, -} +// ── sled-backed implementation ──────────────────────────────────────────────── -impl EngramDb { - /// Open (or create) an engram database at the given path. - /// - /// The path should be a directory. Sled will create it if it doesn't exist. - pub fn open(path: &Path) -> EngramResult { - let db = sled::open(path)?; - Ok(Self { db }) +#[cfg(feature = "sled-backend")] +mod sled_impl { + use crate::activation; + use crate::consolidation::{self, ConsolidationConfig, ConsolidationReport}; + use crate::error::{EngramError, EngramResult}; + use crate::graph; + use crate::salience; + use crate::storage; + use crate::types::{ActivatedNode, Edge, Node, RelationType, ScoredNode}; + use crate::vector; + use sled::Db; + use std::path::Path; + use uuid::Uuid; + + pub struct EngramDb { + pub(crate) db: Db, } - // ── Node operations ─────────────────────────────────────────────────────── + impl EngramDb { + /// Open (or create) an engram database at the given path. + pub fn open(path: &Path) -> EngramResult { + let db = sled::open(path)?; + Ok(Self { db }) + } - /// Persist a node. Returns the node's UUID. - /// - /// If a node with the same ID already exists, it is overwritten. - pub fn put_node(&self, node: Node) -> EngramResult { - graph::put_node(&self.db, &node) - } + // ── Node operations ─────────────────────────────────────────────────── - /// Retrieve a node by UUID. Returns None if not found. - pub fn get_node(&self, id: Uuid) -> EngramResult> { - graph::get_node(&self.db, id) - } + /// Persist a node. Returns the node's UUID. + pub fn put_node(&self, node: Node) -> EngramResult { + let id = graph::put_node(&self.db, &node)?; + // Mark HNSW index dirty so next search rebuilds it. + vector::mark_dirty(&self.db); + Ok(id) + } - // ── Edge operations ─────────────────────────────────────────────────────── - - /// Persist a directed edge between two nodes. - pub fn put_edge(&self, edge: Edge) -> EngramResult<()> { - graph::put_edge(&self.db, &edge) - } - - /// All edges originating from a node. - pub fn get_edges_from(&self, from_id: Uuid) -> EngramResult> { - graph::edges_from(&self.db, from_id) - } - - /// All edges pointing to a node. - pub fn get_edges_to(&self, to_id: Uuid) -> EngramResult> { - graph::edges_to(&self.db, to_id) - } - - // ── Vector search ───────────────────────────────────────────────────────── - - /// Find the `limit` nodes whose embeddings are most similar to `embedding`. - /// - /// Uses flat cosine scan — O(n), correct for < 100k nodes. - pub fn search_embedding(&self, embedding: &[f32], limit: usize) -> EngramResult> { - vector::search_embedding(&self.db, embedding, limit, |id| { + /// Retrieve a node by UUID. Returns None if not found. + pub fn get_node(&self, id: Uuid) -> EngramResult> { graph::get_node(&self.db, id) - }) - } - - // ── Spreading activation ────────────────────────────────────────────────── - - /// Run spreading activation from a set of seed nodes. - /// - /// Activation propagates outward through the graph. At each hop, strength - /// is attenuated by edge weight, target salience, and semantic similarity - /// to `query_embedding`. The top-`limit` nodes by activation strength are returned. - /// - /// See `activation.rs` for a full description of the algorithm. - pub fn activate( - &self, - seeds: &[Uuid], - query_embedding: &[f32], - max_depth: u8, - limit: usize, - ) -> EngramResult> { - activation::activate(&self.db, seeds, query_embedding, max_depth, limit) - } - - // ── Graph traversal ─────────────────────────────────────────────────────── - - /// BFS traversal from `from`, following edges up to `max_depth` hops. - /// - /// If `relation` is specified, only edges of that type are followed. - /// The seed node itself is excluded from the result. - pub fn traverse( - &self, - from: Uuid, - relation: Option, - max_depth: u8, - ) -> EngramResult> { - graph::traverse(&self.db, from, relation, max_depth) - } - - // ── Salience management ─────────────────────────────────────────────────── - - /// Mark a node as recently activated — update last_activated, increment - /// activation_count, and recompute salience. - /// - /// Call this whenever a node is surfaced during retrieval so that - /// frequently-used nodes accumulate higher salience over time. - pub fn touch(&self, id: Uuid) -> EngramResult<()> { - let mut node = graph::get_node(&self.db, id)?.ok_or(EngramError::NotFound(id))?; - node.last_activated = crate::types::now_ms(); - node.activation_count += 1; - node.salience = salience::compute_salience( - node.importance, - node.last_activated, - node.activation_count, - ); - graph::put_node(&self.db, &node)?; - Ok(()) - } - - /// Apply a multiplicative decay to the salience of every node in the store. - /// - /// `factor` should be in (0.0, 1.0). A factor of 0.95 decays salience by 5%. - /// Returns the number of nodes updated. - /// - /// This models the adaptive nature of forgetting: nodes that haven't been - /// activated recently become less salient over time, making room for new - /// associations. - pub fn decay(&self, factor: f32) -> EngramResult { - if !(0.0..=1.0).contains(&factor) { - return Err(EngramError::InvalidParam(format!( - "decay factor must be in [0.0, 1.0], got {}", - factor - ))); } - let nodes = storage::scan_nodes(&self.db)?; - let mut count = 0usize; - for mut node in nodes { - let new_salience = salience::decay_salience(node.salience, factor); - // Always write if salience changed at all (decay always changes it - // unless the node is already at zero) - if new_salience != node.salience { - node.salience = new_salience; - storage::write_salience(&self.db, node.id, new_salience)?; - // Also update the full node record so future reads are consistent - graph::put_node(&self.db, &node)?; - count += 1; + // ── Edge operations ─────────────────────────────────────────────────── + + /// Persist a directed edge between two nodes. + pub fn put_edge(&self, edge: Edge) -> EngramResult<()> { + graph::put_edge(&self.db, &edge) + } + + /// All edges originating from a node. + pub fn get_edges_from(&self, from_id: Uuid) -> EngramResult> { + graph::edges_from(&self.db, from_id) + } + + /// All edges pointing to a node. + pub fn get_edges_to(&self, to_id: Uuid) -> EngramResult> { + graph::edges_to(&self.db, to_id) + } + + // ── Vector search ───────────────────────────────────────────────────── + + /// Find the `limit` nodes whose embeddings are most similar to `embedding`. + /// + /// Falls back to flat scan for stores with < 100 nodes. + /// Uses the HNSW index for larger stores. + pub fn search_embedding( + &self, + embedding: &[f32], + limit: usize, + ) -> EngramResult> { + vector::search_embedding(&self.db, embedding, limit, |id| { + graph::get_node(&self.db, id) + }) + } + + /// Explicitly build (or rebuild) the HNSW index. + /// + /// This is not normally needed — the index is built lazily on first search. + /// Call this if you want to pre-warm the index after a large batch insert. + /// + /// Returns the number of nodes indexed. + pub fn build_index(&self) -> EngramResult { + vector::build_index(&self.db) + } + + // ── Spreading activation ────────────────────────────────────────────── + + /// Run spreading activation from a set of seed nodes. + pub fn activate( + &self, + seeds: &[Uuid], + query_embedding: &[f32], + max_depth: u8, + limit: usize, + ) -> EngramResult> { + activation::activate(&self.db, seeds, query_embedding, max_depth, limit) + } + + // ── Graph traversal ─────────────────────────────────────────────────── + + /// BFS traversal from `from`, following edges up to `max_depth` hops. + pub fn traverse( + &self, + from: Uuid, + relation: Option, + max_depth: u8, + ) -> EngramResult> { + graph::traverse(&self.db, from, relation, max_depth) + } + + // ── Salience management ─────────────────────────────────────────────── + + /// Mark a node as recently activated — update last_activated, increment + /// activation_count, and recompute salience. + pub fn touch(&self, id: Uuid) -> EngramResult<()> { + let mut node = + graph::get_node(&self.db, id)?.ok_or(EngramError::NotFound(id))?; + node.last_activated = crate::types::now_ms(); + node.activation_count += 1; + node.salience = salience::compute_salience( + node.importance, + node.last_activated, + node.activation_count, + ); + graph::put_node(&self.db, &node)?; + Ok(()) + } + + /// Apply a multiplicative decay to the salience of every node in the store. + /// + /// `factor` should be in (0.0, 1.0). Returns the number of nodes updated. + pub fn decay(&self, factor: f32) -> EngramResult { + if !(0.0..=1.0).contains(&factor) { + return Err(EngramError::InvalidParam(format!( + "decay factor must be in [0.0, 1.0], got {}", + factor + ))); } + + let nodes = storage::scan_nodes(&self.db)?; + let mut count = 0usize; + for mut node in nodes { + let new_salience = salience::decay_salience(node.salience, factor); + if new_salience != node.salience { + node.salience = new_salience; + storage::write_salience(&self.db, node.id, new_salience)?; + graph::put_node(&self.db, &node)?; + count += 1; + } + } + Ok(count) } - Ok(count) - } - // ── Statistics ──────────────────────────────────────────────────────────── + // ── Consolidation ───────────────────────────────────────────────────── - /// Total number of nodes stored. - pub fn node_count(&self) -> EngramResult { - graph::node_count(&self.db) - } + /// Run a memory consolidation cycle. + /// + /// Promotes Episodic nodes that have been activated enough times and are + /// still salient enough to MemoryTier::Semantic. Then decays all saliences. + /// + /// See `consolidation::ConsolidationConfig` for tuning knobs. + pub fn consolidate( + &self, + config: &ConsolidationConfig, + ) -> EngramResult { + consolidation::consolidate(&self.db, config) + } - /// Total number of edges stored (each directed edge counted once). - pub fn edge_count(&self) -> EngramResult { - graph::edge_count(&self.db) + // ── Statistics ──────────────────────────────────────────────────────── + + /// Total number of nodes stored. + pub fn node_count(&self) -> EngramResult { + graph::node_count(&self.db) + } + + /// Total number of edges stored. + pub fn edge_count(&self) -> EngramResult { + graph::edge_count(&self.db) + } } } + +// ── WASM / in-memory implementation ────────────────────────────────────────── + +#[cfg(feature = "wasm")] +mod wasm_impl { + use crate::consolidation::{self, ConsolidationConfig, ConsolidationReport}; + use crate::error::{EngramError, EngramResult}; + use crate::mem_storage::MemStore; + use crate::salience; + use crate::types::{ActivatedNode, Edge, Node, RelationType, ScoredNode}; + use crate::vector; + use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque}; + use std::sync::RwLock; + use uuid::Uuid; + + pub struct EngramDb { + pub(crate) store: RwLock, + } + + impl EngramDb { + /// Create an in-memory engram database. The `path` argument is ignored in WASM mode. + pub fn open(_path: &std::path::Path) -> EngramResult { + Ok(Self { + store: RwLock::new(MemStore::new()), + }) + } + + // ── Node operations ─────────────────────────────────────────────────── + + pub fn put_node(&self, node: Node) -> EngramResult { + let id = node.id; + self.store + .write() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .write_node(&node)?; + Ok(id) + } + + pub fn get_node(&self, id: Uuid) -> EngramResult> { + self.store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .read_node(id) + } + + // ── Edge operations ─────────────────────────────────────────────────── + + pub fn put_edge(&self, edge: Edge) -> EngramResult<()> { + self.store + .write() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .write_edge(&edge) + } + + pub fn get_edges_from(&self, from_id: Uuid) -> EngramResult> { + self.store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .read_edges_from(from_id) + } + + pub fn get_edges_to(&self, to_id: Uuid) -> EngramResult> { + self.store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .read_edges_to(to_id) + } + + // ── Vector search ───────────────────────────────────────────────────── + + pub fn search_embedding( + &self, + embedding: &[f32], + limit: usize, + ) -> EngramResult> { + let store = self + .store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?; + let vectors = store.scan_vectors()?; + let nodes_snap: HashMap = store.nodes.clone(); + drop(store); + + vector::search_embedding_memory(embedding, limit, &vectors, |id| { + Ok(nodes_snap.get(&id).cloned()) + }) + } + + /// No-op in WASM mode (flat scan is always used). Returns node count. + pub fn build_index(&self) -> EngramResult { + self.node_count() + } + + // ── Spreading activation ────────────────────────────────────────────── + + pub fn activate( + &self, + seeds: &[Uuid], + query_embedding: &[f32], + max_depth: u8, + limit: usize, + ) -> EngramResult> { + use crate::activation; + let store = self + .store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?; + activation::activate_mem(&store, seeds, query_embedding, max_depth, limit) + } + + // ── Graph traversal ─────────────────────────────────────────────────── + + pub fn traverse( + &self, + from: Uuid, + relation: Option, + max_depth: u8, + ) -> EngramResult> { + let store = self + .store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?; + let mut visited: HashSet = HashSet::new(); + let mut queue: VecDeque<(Uuid, u8)> = VecDeque::new(); + let mut result: Vec = Vec::new(); + + visited.insert(from); + queue.push_back((from, 0)); + + while let Some((current_id, depth)) = queue.pop_front() { + if depth >= max_depth { + continue; + } + let edges = store.read_edges_from(current_id)?; + for edge in edges { + if let Some(ref rel) = relation { + if &edge.relation != rel { + continue; + } + } + let next = edge.to_id; + if visited.contains(&next) { + continue; + } + visited.insert(next); + if let Some(node) = store.read_node(next)? { + result.push(node); + queue.push_back((next, depth + 1)); + } + } + } + Ok(result) + } + + // ── Salience management ─────────────────────────────────────────────── + + pub fn touch(&self, id: Uuid) -> EngramResult<()> { + let mut store = self + .store + .write() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?; + let node = store + .nodes + .get_mut(&id) + .ok_or(EngramError::NotFound(id))?; + node.last_activated = crate::types::now_ms(); + node.activation_count += 1; + node.salience = salience::compute_salience( + node.importance, + node.last_activated, + node.activation_count, + ); + Ok(()) + } + + pub fn decay(&self, factor: f32) -> EngramResult { + if !(0.0..=1.0).contains(&factor) { + return Err(EngramError::InvalidParam(format!( + "decay factor must be in [0.0, 1.0], got {}", + factor + ))); + } + let mut store = self + .store + .write() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?; + let mut count = 0usize; + for node in store.nodes.values_mut() { + let new_sal = salience::decay_salience(node.salience, factor); + if new_sal != node.salience { + node.salience = new_sal; + count += 1; + } + } + Ok(count) + } + + // ── Consolidation ───────────────────────────────────────────────────── + + pub fn consolidate( + &self, + config: &ConsolidationConfig, + ) -> EngramResult { + let mut store = self + .store + .write() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))?; + consolidation::consolidate_mem(&mut store, config) + } + + // ── Statistics ──────────────────────────────────────────────────────── + + pub fn node_count(&self) -> EngramResult { + Ok(self + .store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .node_count()) + } + + pub fn edge_count(&self) -> EngramResult { + Ok(self + .store + .read() + .map_err(|_| EngramError::InvalidParam("lock poisoned".into()))? + .edge_count()) + } + } +} + +// ── Re-export the right impl ────────────────────────────────────────────────── + +#[cfg(feature = "sled-backend")] +pub use sled_impl::EngramDb; + +#[cfg(feature = "wasm")] +pub use wasm_impl::EngramDb; diff --git a/crates/engram-core/src/lib.rs b/crates/engram-core/src/lib.rs index 2777b56..0404456 100644 --- a/crates/engram-core/src/lib.rs +++ b/crates/engram-core/src/lib.rs @@ -30,13 +30,19 @@ /// } /// ``` pub mod activation; +pub mod consolidation; pub mod db; pub mod error; pub mod graph; pub mod salience; +#[cfg(not(feature = "wasm"))] pub mod storage; +#[cfg(feature = "wasm")] +pub mod mem_storage; pub mod types; pub mod vector; +#[cfg(feature = "migration")] +pub mod migration; // Re-export the public surface pub use db::EngramDb; @@ -44,3 +50,4 @@ pub use error::{EngramError, EngramResult}; pub use types::{ ActivatedNode, Edge, MemoryTier, Node, NodeType, RelationType, ScoredNode, now_ms, }; +pub use consolidation::{ConsolidationConfig, ConsolidationReport}; diff --git a/crates/engram-core/src/mem_storage.rs b/crates/engram-core/src/mem_storage.rs new file mode 100644 index 0000000..ac6721b --- /dev/null +++ b/crates/engram-core/src/mem_storage.rs @@ -0,0 +1,99 @@ +/// In-memory storage backend for environments without a filesystem. +/// +/// Used when the `wasm` feature is enabled (e.g. browser via wasm-bindgen). +/// Implements the same logical interface as the sled-backed `storage` module +/// so that `EngramDb` can work identically in both environments. +/// +/// All state lives in a `MemStore` that is held by `EngramDb` under a `RwLock` +/// so concurrent reads are fine and writes are serialised. +use crate::error::{EngramError, EngramResult}; +use crate::types::{Edge, Node}; +use std::collections::HashMap; +use uuid::Uuid; + +#[derive(Default)] +pub struct MemStore { + pub nodes: HashMap, + /// from_id → list of edges + pub edges_from: HashMap>, + /// to_id → list of edges + pub edges_to: HashMap>, +} + +impl MemStore { + pub fn new() -> Self { + Self::default() + } + + // ── Node operations ─────────────────────────────────────────────────────── + + pub fn write_node(&mut self, node: &Node) -> EngramResult<()> { + self.nodes.insert(node.id, node.clone()); + Ok(()) + } + + pub fn read_node(&self, id: Uuid) -> EngramResult> { + Ok(self.nodes.get(&id).cloned()) + } + + pub fn scan_nodes(&self) -> EngramResult> { + Ok(self.nodes.values().cloned().collect()) + } + + pub fn node_count(&self) -> usize { + self.nodes.len() + } + + // ── Edge operations ─────────────────────────────────────────────────────── + + pub fn write_edge(&mut self, edge: &Edge) -> EngramResult<()> { + self.edges_from + .entry(edge.from_id) + .or_default() + .push(edge.clone()); + self.edges_to + .entry(edge.to_id) + .or_default() + .push(edge.clone()); + Ok(()) + } + + pub fn read_edges_from(&self, from_id: Uuid) -> EngramResult> { + Ok(self + .edges_from + .get(&from_id) + .cloned() + .unwrap_or_default()) + } + + pub fn read_edges_to(&self, to_id: Uuid) -> EngramResult> { + Ok(self + .edges_to + .get(&to_id) + .cloned() + .unwrap_or_default()) + } + + pub fn edge_count(&self) -> usize { + self.edges_from.values().map(|v| v.len()).sum() + } + + // ── Vector operations ───────────────────────────────────────────────────── + + pub fn scan_vectors(&self) -> EngramResult)>> { + Ok(self + .nodes + .iter() + .map(|(id, n)| (*id, n.embedding.clone())) + .collect()) + } + + // ── Salience ────────────────────────────────────────────────────────────── + + pub fn write_salience(&mut self, id: Uuid, salience: f32) -> EngramResult<()> { + if let Some(node) = self.nodes.get_mut(&id) { + node.salience = salience; + } + Ok(()) + } +} diff --git a/crates/engram-core/src/migration.rs b/crates/engram-core/src/migration.rs new file mode 100644 index 0000000..016e708 --- /dev/null +++ b/crates/engram-core/src/migration.rs @@ -0,0 +1,420 @@ +/// Migration connector — imports Neuron's SQLite database into Engram. +/// +/// Neuron stores memories, knowledge, and graph nodes in a SQLite database. +/// This module reads that database and converts records to Engram nodes and edges. +/// +/// # Schema mapping +/// +/// | Neuron table | Engram node | +/// |----------------------|----------------------------------------------| +/// | `memory_nodes` | `Node { tier: Episodic, node_type: Memory }` | +/// | `knowledge_entries` | `Node { tier: Semantic, node_type: Concept }` | +/// +/// Edges from `graph_edges` (type = "supersedes" or "Supersedes") are converted +/// to `Edge { relation: RelationType::Supersedes }`. Other edge types become +/// `RelationType::References`. +/// +/// # Embeddings +/// +/// Neuron does not currently expose embeddings through the SQLite schema. +/// Random unit vectors are generated as placeholders. Replace the call to +/// `placeholder_embedding` with your embedding model once the ONNX engine is wired in. +/// +/// TODO: wire in real embeddings from all-MiniLM-L6-v2 via the ONNX runtime. + +use crate::error::{EngramError, EngramResult}; +use crate::types::{Edge, MemoryTier, Node, NodeType, RelationType}; +use rusqlite::{Connection, OpenFlags}; +use std::collections::HashMap; +use std::path::PathBuf; +use uuid::Uuid; + +// ── Config and report ───────────────────────────────────────────────────────── + +/// Configuration for a Neuron → Engram migration. +pub struct MigrationConfig { + /// Path to `~/.neuron/neuron.db` (or any other Neuron SQLite file). + pub sqlite_path: PathBuf, + /// Path where the new Engram sled store will be created. + pub engram_path: PathBuf, + /// Dimensionality of placeholder embeddings. + /// Default: 384 (matches all-MiniLM-L6-v2). + pub embedding_dim: usize, +} + +impl MigrationConfig { + pub fn new(sqlite_path: PathBuf, engram_path: PathBuf) -> Self { + Self { + sqlite_path, + engram_path, + embedding_dim: 384, + } + } +} + +/// Summary of what was imported during migration. +#[derive(Debug, Default)] +pub struct MigrationReport { + /// Rows imported from `memory_nodes`. + pub memories_migrated: usize, + /// Rows imported from `knowledge_entries`. + pub knowledge_migrated: usize, + /// Edges created from `graph_edges`. + pub edges_created: usize, + /// Non-fatal errors collected during the run. + pub errors: Vec, +} + +// ── Main entry point ────────────────────────────────────────────────────────── + +/// Read the Neuron SQLite database at `config.sqlite_path` and import all +/// records into a new Engram sled store at `config.engram_path`. +/// +/// Returns a `MigrationReport` describing what was imported. +/// +/// Non-fatal errors (e.g. a single unreadable row) are collected in +/// `report.errors` rather than aborting the entire migration. +pub fn migrate_from_neuron(config: &MigrationConfig) -> EngramResult { + let conn = Connection::open_with_flags( + &config.sqlite_path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(|e| EngramError::InvalidParam(format!("Cannot open SQLite: {e}")))?; + + let engram_db = crate::db::EngramDb::open(&config.engram_path)?; + + let mut report = MigrationReport::default(); + + // Maps Neuron string IDs to the Engram UUIDs we assigned. + let mut id_map: HashMap = HashMap::new(); + + // ── Import memory_nodes ─────────────────────────────────────────────────── + { + let mut stmt = conn + .prepare( + "SELECT id, content, importance, superseded_by, created_at \ + FROM memory_nodes ORDER BY created_at ASC", + ) + .map_err(|e| EngramError::InvalidParam(format!("prepare memory_nodes: {e}")))?; + + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, // id + row.get::<_, String>(1)?, // content + row.get::<_, String>(2)?, // importance + row.get::<_, Option>(3)?, // superseded_by + row.get::<_, i64>(4)?, // created_at + )) + }) + .map_err(|e| EngramError::InvalidParam(format!("query memory_nodes: {e}")))?; + + for row_result in rows { + match row_result { + Ok((neuron_id, content, importance_str, _superseded_by, _created_at)) => { + let importance = importance_string_to_f32(&importance_str); + let embedding = placeholder_embedding(config.embedding_dim); + + let node = Node::new( + NodeType::Memory, + embedding, + content.into_bytes(), + MemoryTier::Episodic, + importance, + ); + + match engram_db.put_node(node.clone()) { + Ok(uuid) => { + id_map.insert(neuron_id, uuid); + report.memories_migrated += 1; + } + Err(e) => { + report.errors.push(format!("put_node memory {neuron_id}: {e}")); + } + } + } + Err(e) => { + report.errors.push(format!("read memory row: {e}")); + } + } + } + } + + // ── Import knowledge_entries ────────────────────────────────────────────── + { + let mut stmt = conn + .prepare( + "SELECT id, title, content, tier, created_at \ + FROM knowledge_entries ORDER BY created_at ASC", + ) + .map_err(|e| EngramError::InvalidParam(format!("prepare knowledge_entries: {e}")))?; + + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, // id + row.get::<_, String>(1)?, // title + row.get::<_, String>(2)?, // content + row.get::<_, String>(3)?, // tier + row.get::<_, i64>(4)?, // created_at + )) + }) + .map_err(|e| EngramError::InvalidParam(format!("query knowledge_entries: {e}")))?; + + for row_result in rows { + match row_result { + Ok((neuron_id, title, body, _tier_str, _created_at)) => { + // Combine title + content as the engram node content. + let combined = format!("{title}\n\n{body}"); + let embedding = placeholder_embedding(config.embedding_dim); + + let node = Node::new( + NodeType::Concept, + embedding, + combined.into_bytes(), + MemoryTier::Semantic, + 0.75, // knowledge is moderately important by default + ); + + match engram_db.put_node(node.clone()) { + Ok(uuid) => { + id_map.insert(neuron_id, uuid); + report.knowledge_migrated += 1; + } + Err(e) => { + report.errors.push(format!( + "put_node knowledge {neuron_id}: {e}" + )); + } + } + } + Err(e) => { + report.errors.push(format!("read knowledge row: {e}")); + } + } + } + } + + // ── Import graph_edges ──────────────────────────────────────────────────── + { + // Only import edges where both endpoints ended up in our id_map. + let mut stmt = conn + .prepare( + "SELECT from_id, to_id, edge_type, weight FROM graph_edges", + ) + .map_err(|e| EngramError::InvalidParam(format!("prepare graph_edges: {e}")))?; + + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, // from_id + row.get::<_, String>(1)?, // to_id + row.get::<_, String>(2)?, // edge_type + row.get::<_, f64>(3)?, // weight + )) + }) + .map_err(|e| EngramError::InvalidParam(format!("query graph_edges: {e}")))?; + + for row_result in rows { + match row_result { + Ok((from_str, to_str, edge_type, weight)) => { + let from_uuid = match id_map.get(&from_str) { + Some(u) => *u, + None => continue, // endpoint not migrated, skip + }; + let to_uuid = match id_map.get(&to_str) { + Some(u) => *u, + None => continue, + }; + + let relation = edge_type_to_relation(&edge_type); + let edge = Edge::new(from_uuid, to_uuid, relation, weight as f32); + + match engram_db.put_edge(edge) { + Ok(()) => report.edges_created += 1, + Err(e) => { + report.errors.push(format!( + "put_edge {from_str} → {to_str}: {e}" + )); + } + } + } + Err(e) => { + report.errors.push(format!("read edge row: {e}")); + } + } + } + } + + Ok(report) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/// Convert Neuron's text importance level to a float score. +fn importance_string_to_f32(importance: &str) -> f32 { + match importance.to_lowercase().as_str() { + "critical" => 1.0, + "high" => 0.85, + "normal" | "medium" => 0.5, + "low" => 0.25, + _ => { + // Try parsing directly as a float. + importance.parse::().unwrap_or(0.5).clamp(0.0, 1.0) + } + } +} + +/// Convert a Neuron edge type string to an Engram RelationType. +fn edge_type_to_relation(edge_type: &str) -> RelationType { + match edge_type.to_lowercase().as_str() { + "supersedes" | "superseded_by" => RelationType::Supersedes, + "causes" | "caused_by" => RelationType::Causes, + "contains" | "contained_by" => RelationType::Contains, + "references" | "referenced_by" => RelationType::References, + "contradicts" => RelationType::Contradicts, + "exemplifies" | "exemplified_by" => RelationType::Exemplifies, + "activates" => RelationType::Activates, + "temporally_precedes" | "follows" => RelationType::TemporallyPrecedes, + _ => RelationType::References, // safe default + } +} + +/// Generate a pseudo-random unit vector of the given dimension as a placeholder embedding. +/// +/// Uses a simple xorshift64 PRNG seeded from the current time. The result is +/// semantically meaningless — it only satisfies the schema requirement that +/// every node has an embedding vector. +/// +/// TODO: replace with actual embeddings from all-MiniLM-L6-v2 via ONNX runtime +/// once the embedding engine is wired in. +pub fn placeholder_embedding(dim: usize) -> Vec { + // Seed from subsecond wall time for reasonable entropy across calls. + let mut state: u64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .subsec_nanos() as u64 + | 1; // ensure non-zero + + // xorshift64 — no overflow risk, passes statistical tests well enough for placeholders. + let mut xorshift = || -> f32 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + // Map to [-1, 1] + (state as f32 / u64::MAX as f32) * 2.0 - 1.0 + }; + + let mut raw: Vec = (0..dim).map(|_| xorshift()).collect(); + + // Normalise to unit length. + let norm: f32 = raw.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + for x in &mut raw { + *x /= norm; + } + } + raw +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn importance_string_critical() { + assert!((importance_string_to_f32("critical") - 1.0).abs() < 1e-6); + } + + #[test] + fn importance_string_normal() { + assert!((importance_string_to_f32("normal") - 0.5).abs() < 1e-6); + } + + #[test] + fn importance_string_unknown_defaults_to_half() { + assert!((importance_string_to_f32("???") - 0.5).abs() < 1e-6); + } + + #[test] + fn edge_type_supersedes() { + assert_eq!(edge_type_to_relation("supersedes"), RelationType::Supersedes); + } + + #[test] + fn edge_type_unknown_is_references() { + assert_eq!(edge_type_to_relation("foobar"), RelationType::References); + } + + #[test] + fn placeholder_embedding_correct_length() { + let emb = placeholder_embedding(384); + assert_eq!(emb.len(), 384); + } + + #[test] + fn placeholder_embedding_is_unit_vector() { + let emb = placeholder_embedding(128); + let norm: f32 = emb.iter().map(|x| x * x).sum::().sqrt(); + assert!((norm - 1.0).abs() < 1e-4); + } + + #[test] + fn migrate_from_in_memory_db() { + // Build a minimal SQLite DB in a temp dir and migrate it. + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let engram_path = dir.path().join("engram"); + + // Create minimal Neuron-like schema and insert a couple of rows. + let conn = Connection::open(&db_path).unwrap(); + conn.execute_batch( + "CREATE TABLE memory_nodes ( + id TEXT PRIMARY KEY, content TEXT NOT NULL, importance TEXT NOT NULL DEFAULT 'normal', + superseded_by TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE TABLE knowledge_entries ( + id TEXT PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, + category TEXT NOT NULL DEFAULT '', tier TEXT NOT NULL DEFAULT 'note', + tags TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE TABLE graph_edges ( + from_id TEXT NOT NULL, from_type TEXT NOT NULL, + to_id TEXT NOT NULL, to_type TEXT NOT NULL, + edge_type TEXT NOT NULL, weight REAL NOT NULL DEFAULT 1.0, + PRIMARY KEY (from_id, to_id, edge_type) + );", + ) + .unwrap(); + + conn.execute( + "INSERT INTO memory_nodes (id, content, importance, created_at, updated_at) + VALUES ('mem-1', 'First memory', 'high', 1000, 1000)", + [], + ) + .unwrap(); + + conn.execute( + "INSERT INTO knowledge_entries (id, title, content, created_at, updated_at) + VALUES ('kn-1', 'Some concept', 'Body text.', 2000, 2000)", + [], + ) + .unwrap(); + + drop(conn); // close before migrating + + let config = MigrationConfig { + sqlite_path: db_path, + engram_path, + embedding_dim: 16, + }; + + let report = migrate_from_neuron(&config).unwrap(); + assert_eq!(report.memories_migrated, 1); + assert_eq!(report.knowledge_migrated, 1); + assert_eq!(report.edges_created, 0); // no edges in the test DB + assert!(report.errors.is_empty()); + } +} diff --git a/crates/engram-core/src/salience.rs b/crates/engram-core/src/salience.rs index 99480f0..d06b43e 100644 --- a/crates/engram-core/src/salience.rs +++ b/crates/engram-core/src/salience.rs @@ -9,7 +9,7 @@ /// Things that aren't activated decay toward zero — not because they are lost, but because /// they are no longer relevant to current cognition. This is how biological memory works. /// -/// ``` +/// ```text /// salience = importance × (1 / (1 + days_since_activation)) × ln(activation_count + 1) /// ``` /// diff --git a/crates/engram-core/src/types.rs b/crates/engram-core/src/types.rs index 8ee32a3..a7125ae 100644 --- a/crates/engram-core/src/types.rs +++ b/crates/engram-core/src/types.rs @@ -129,6 +129,12 @@ impl Node { importance, } } + + /// Override the node's UUID. Used in tests and deserialization helpers. + pub fn with_id(mut self, id: Uuid) -> Self { + self.id = id; + self + } } /// A node returned from spreading activation, annotated with how strongly diff --git a/crates/engram-core/src/vector.rs b/crates/engram-core/src/vector.rs index 7f890c0..6b7e401 100644 --- a/crates/engram-core/src/vector.rs +++ b/crates/engram-core/src/vector.rs @@ -1,24 +1,58 @@ /// Vector similarity search over stored node embeddings. /// -/// v0.1 uses a flat cosine scan — O(n) but correct and dependency-free. -/// For < 100k nodes this is adequate. Future versions will layer in HNSW -/// once the graph structure itself is validated. +/// Strategy: +/// - For < HNSW_THRESHOLD indexed nodes: flat O(n) cosine scan (correct, no deps) +/// - For >= HNSW_THRESHOLD nodes: HNSW approximate nearest-neighbour index /// -/// Cosine similarity between two vectors A and B: -/// cos(θ) = (A · B) / (|A| × |B|) +/// The HNSW index is built lazily on first search call when the graph is large +/// enough. A "dirty" flag in sled (`hnsw:dirty`) is set to 1 whenever `put_node` +/// adds an embedding; on the next search the index is rebuilt from the current +/// store. For small graphs (< threshold) the flat scan is always used — it is +/// fast enough and avoids the overhead of HNSW construction. /// -/// We return 0.0 when either vector has zero norm (degenerate case). -use crate::error::{EngramError, EngramResult}; +/// Cosine similarity: cos(θ) = (A · B) / (|A| × |B|) +/// instant-distance expects a *distance* metric (lower = closer), so we expose: +/// distance = 1 − cosine_similarity, clamped to [0, 2] + +#[cfg(feature = "sled-backend")] use crate::storage; -use crate::types::{Node, ScoredNode}; +#[cfg(feature = "sled-backend")] use sled::Db; + +use crate::error::EngramResult; +use crate::types::{Node, ScoredNode}; +use instant_distance::{Builder, HnswMap, Search}; +use serde::{Deserialize, Serialize}; use uuid::Uuid; +/// Minimum number of nodes before we switch from flat scan to HNSW. +const HNSW_THRESHOLD: usize = 100; + +/// sled key used to store the dirty flag (1 = needs rebuild, 0 = clean). +#[cfg(feature = "sled-backend")] +const HNSW_DIRTY_KEY: &[u8] = b"hnsw:dirty"; + +// ── Point wrapper ───────────────────────────────────────────────────────────── + +/// An f32 embedding vector treated as an HNSW point. +/// +/// The distance metric is `1 − cosine_similarity` so that instant-distance +/// (which minimises distance) finds the most similar vectors. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EmbeddingPoint(pub Vec); + +impl instant_distance::Point for EmbeddingPoint { + fn distance(&self, other: &Self) -> f32 { + let sim = cosine_similarity(&self.0, &other.0); + (1.0 - sim).clamp(0.0, 2.0) + } +} + +// ── Public similarity helper ────────────────────────────────────────────────── + /// Compute the cosine similarity between two equal-length f32 slices. /// /// Returns a value in [-1.0, 1.0], where 1.0 means identical direction. -/// For normalized embeddings (unit vectors) the dot product alone is sufficient, -/// but we compute full cosine here to be robust to unnormalized inputs. pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { if a.len() != b.len() || a.is_empty() { return 0.0; @@ -32,42 +66,72 @@ pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { (dot / (norm_a * norm_b)).clamp(-1.0, 1.0) } +// ── sled-backed search ──────────────────────────────────────────────────────── + +#[cfg(feature = "sled-backend")] /// Search all stored embeddings for the `limit` closest nodes to `query`. /// -/// This is a full scan. Every stored vector is loaded and scored. -/// Results are sorted descending by cosine similarity. +/// Falls back to flat scan for stores with < HNSW_THRESHOLD nodes, or when the +/// index has not yet been built. Uses HNSW for large stores. pub fn search_embedding( db: &Db, query: &[f32], limit: usize, - // Loader that retrieves a Node by Uuid — avoids a circular dep on graph.rs node_loader: impl Fn(Uuid) -> EngramResult>, ) -> EngramResult> { let vectors = storage::scan_vectors(db)?; - let mut scored: Vec<(Uuid, f32)> = vectors - .iter() - .map(|(id, emb)| { - let sim = cosine_similarity(query, emb); - (*id, sim) - }) - .collect(); - // Sort descending by score - scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - scored.truncate(limit); - - let mut results = Vec::with_capacity(scored.len()); - for (id, score) in scored { - if let Some(node) = node_loader(id)? { - results.push(ScoredNode { node, score }); - } + if vectors.len() < HNSW_THRESHOLD { + return flat_search(query, limit, &vectors, node_loader); } - Ok(results) + + // Check if the index needs rebuilding. + let dirty = db + .get(HNSW_DIRTY_KEY)? + .map(|v| v.first().copied().unwrap_or(1) != 0) + .unwrap_or(true); + + // We always rebuild if dirty. The index is not serialised to sled because + // HnswMap serialisation size can be large and the rebuild is fast (<10ms + // for typical node counts). The dirty flag is persisted so we skip + // unnecessary rebuilds between searches within the same sled session. + let (map, ids) = build_hnsw_index(&vectors); + + if dirty { + // Clear the dirty flag now that we have a fresh index. + let _ = db.insert(HNSW_DIRTY_KEY, vec![0u8]); + } + + hnsw_search(query, limit, &map, &ids, node_loader) +} + +/// Mark the HNSW index as dirty. Call this after any `put_node`. +#[cfg(feature = "sled-backend")] +pub fn mark_dirty(db: &Db) { + let _ = db.insert(HNSW_DIRTY_KEY, vec![1u8]); +} + +/// Explicitly build and persist the HNSW index. Returns the number of nodes indexed. +/// +/// Not normally needed — the index is built lazily on first search. +/// Call this to pre-warm after a large batch insert. +#[cfg(feature = "sled-backend")] +pub fn build_index(db: &Db) -> EngramResult { + let vectors = storage::scan_vectors(db)?; + let n = vectors.len(); + // Build the index (result is discarded — next search will build from the + // current clean state). + if n > 0 { + let _ = build_hnsw_index(&vectors); + } + let _ = db.insert(HNSW_DIRTY_KEY, vec![0u8]); + Ok(n) } /// Retrieve the stored embedding for a single node by id. -/// Returns an error if the node has no stored vector (shouldn't happen in normal use). +#[cfg(feature = "sled-backend")] pub fn get_embedding(db: &Db, id: Uuid) -> EngramResult> { + use crate::error::EngramError; let key = storage::vector_key(id); match db.get(key)? { Some(bytes) => { @@ -80,3 +144,187 @@ pub fn get_embedding(db: &Db, id: Uuid) -> EngramResult> { None => Err(EngramError::NotFound(id)), } } + +// ── In-memory search (used by wasm / unit tests) ────────────────────────────── + +/// Search a list of (id, embedding) pairs without a database. +pub fn search_embedding_memory( + query: &[f32], + limit: usize, + vectors: &[(Uuid, Vec)], + node_loader: impl Fn(Uuid) -> EngramResult>, +) -> EngramResult> { + if vectors.len() < HNSW_THRESHOLD { + flat_search(query, limit, vectors, node_loader) + } else { + let (map, ids) = build_hnsw_index(vectors); + hnsw_search(query, limit, &map, &ids, node_loader) + } +} + +// ── Internal helpers ────────────────────────────────────────────────────────── + +/// Flat cosine scan — O(n). Used when the graph is small. +fn flat_search( + query: &[f32], + limit: usize, + vectors: &[(Uuid, Vec)], + node_loader: impl Fn(Uuid) -> EngramResult>, +) -> EngramResult> { + let mut scored: Vec<(Uuid, f32)> = vectors + .iter() + .map(|(id, emb)| (*id, cosine_similarity(query, emb))) + .collect(); + + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + scored.truncate(limit); + + let mut results = Vec::with_capacity(scored.len()); + for (id, score) in scored { + if let Some(node) = node_loader(id)? { + results.push(ScoredNode { node, score }); + } + } + Ok(results) +} + +/// Build an HnswMap from a flat vector list. +fn build_hnsw_index(vectors: &[(Uuid, Vec)]) -> (HnswMap, Vec) { + let points: Vec = vectors + .iter() + .map(|(_, emb)| EmbeddingPoint(emb.clone())) + .collect(); + let ids: Vec = vectors.iter().map(|(id, _)| *id).collect(); + let map = Builder::default().build(points, ids.clone()); + (map, ids) +} + +/// Search using an HnswMap. Converts distance back to cosine similarity score. +fn hnsw_search( + query: &[f32], + limit: usize, + map: &HnswMap, + _ids: &[Uuid], + node_loader: impl Fn(Uuid) -> EngramResult>, +) -> EngramResult> { + let query_point = EmbeddingPoint(query.to_vec()); + let mut search = Search::default(); + + let mut results = Vec::new(); + for item in map.search(&query_point, &mut search).take(limit) { + // distance = 1 − cosine_sim → cosine_sim = 1 − distance + let score = (1.0 - item.distance).clamp(-1.0, 1.0); + let node_id = *item.value; + if let Some(node) = node_loader(node_id)? { + results.push(ScoredNode { node, score }); + } + } + // Ensure descending score order (HNSW returns ascending distance order) + results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + Ok(results) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{MemoryTier, Node, NodeType}; + + fn dummy_node(id: Uuid) -> Node { + Node::new( + NodeType::Memory, + vec![0.0; 4], + vec![], + MemoryTier::Episodic, + 0.5, + ) + .with_id(id) + } + + #[test] + fn cosine_identical_vectors() { + let v = vec![1.0_f32, 0.0, 0.0, 1.0]; + assert!((cosine_similarity(&v, &v) - 1.0).abs() < 1e-6); + } + + #[test] + fn cosine_orthogonal_vectors() { + let a = vec![1.0_f32, 0.0]; + let b = vec![0.0_f32, 1.0]; + assert!(cosine_similarity(&a, &b).abs() < 1e-6); + } + + #[test] + fn cosine_opposite_vectors() { + let a = vec![1.0_f32, 0.0]; + let b = vec![-1.0_f32, 0.0]; + let sim = cosine_similarity(&a, &b); + assert!((sim - (-1.0)).abs() < 1e-6); + } + + #[test] + fn flat_search_returns_ordered_results() { + let id_best = Uuid::new_v4(); + let id_mid = Uuid::new_v4(); + let id_low = Uuid::new_v4(); + + let query = vec![1.0_f32, 0.0, 0.0, 0.0]; + let vecs: Vec<(Uuid, Vec)> = vec![ + (id_low, vec![0.0, 1.0, 0.0, 0.0]), // sim=0 + (id_best, vec![1.0, 0.0, 0.0, 0.0]), // sim=1 ← best + (id_mid, vec![0.7, 0.7, 0.0, 0.0]), // sim≈0.7 + ]; + + let results = flat_search(&query, 3, &vecs, |id| Ok(Some(dummy_node(id)))).unwrap(); + + assert_eq!(results.len(), 3); + assert_eq!(results[0].node.id, id_best); + assert!(results[0].score > results[1].score); + assert!(results[1].score > results[2].score); + } + + #[test] + fn flat_search_respects_limit() { + let query = vec![1.0_f32, 0.0]; + let vecs: Vec<(Uuid, Vec)> = (0..10) + .map(|i| (Uuid::new_v4(), vec![i as f32, 0.0])) + .collect(); + let results = flat_search(&query, 3, &vecs, |id| Ok(Some(dummy_node(id)))).unwrap(); + assert_eq!(results.len(), 3); + } + + #[test] + fn embedding_point_distance_self_is_zero() { + use instant_distance::Point; + let p = EmbeddingPoint(vec![0.6_f32, 0.8]); + assert!(p.distance(&p) < 1e-5); + } + + #[test] + fn search_memory_small_falls_back_to_flat() { + let vecs: Vec<(Uuid, Vec)> = (0..10) + .map(|i| { + let mut emb = vec![0.0_f32; 8]; + emb[i % 8] = 1.0; + (Uuid::new_v4(), emb) + }) + .collect(); + + let target = vecs[3].clone(); + let query = target.1.clone(); + + let results = + search_embedding_memory(&query, 1, &vecs, |id| Ok(Some(dummy_node(id)))).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].node.id, target.0); + assert!((results[0].score - 1.0).abs() < 1e-5); + } + + #[test] + fn cosine_zero_vector_returns_zero() { + let a = vec![0.0_f32, 0.0]; + let b = vec![1.0_f32, 0.0]; + assert_eq!(cosine_similarity(&a, &b), 0.0); + } +} diff --git a/crates/engram-ffi/Cargo.toml b/crates/engram-ffi/Cargo.toml index 62a6f60..76c8465 100644 --- a/crates/engram-ffi/Cargo.toml +++ b/crates/engram-ffi/Cargo.toml @@ -11,3 +11,6 @@ crate-type = ["cdylib", "staticlib"] [dependencies] engram-core = { path = "../engram-core" } uuid = { version = "1", features = ["v4", "serde"] } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/engram-ffi/src/lib.rs b/crates/engram-ffi/src/lib.rs index ce98979..7caa184 100644 --- a/crates/engram-ffi/src/lib.rs +++ b/crates/engram-ffi/src/lib.rs @@ -1,32 +1,40 @@ -/// C FFI stubs for engram-core. +/// C FFI for engram-core. /// -/// These are minimal stubs for v0.1 — enough to link from Kotlin, TypeScript (via WASM -/// or Node native addon), and Go. Full binding generation will use cbindgen in v0.2. -/// -/// All pointers passed across the FFI boundary must remain valid for the duration of -/// the call. Strings are null-terminated UTF-8. The caller owns all returned heap memory -/// and must free it via the corresponding `engram_free_*` function. +/// These functions form the stable ABI that Go (via CGo), Python (via ctypes), +/// and other native callers use. All pointers must remain valid for the duration +/// of the call. Strings are null-terminated UTF-8. The caller must free any +/// returned heap-allocated C string with `engram_free_string`. /// /// # Safety -/// All functions in this module are `unsafe` because they accept raw pointers. -/// Callers are responsible for ensuring pointer validity and correct lifetimes. -use engram_core::EngramDb; +/// Every function in this module accepts raw pointers and is therefore `unsafe`. +/// Callers must ensure: +/// - All handle pointers came from `engram_open` and have not been freed. +/// - All string pointers are valid null-terminated UTF-8. +/// - Returned C strings are freed exactly once via `engram_free_string`. +use engram_core::{ + ActivatedNode, EngramDb, MemoryTier, Node, NodeType, +}; use std::ffi::{CStr, CString}; use std::os::raw::c_char; use std::path::Path; +use uuid::Uuid; -/// Opaque handle to an open EngramDb instance. +// ── Handle type ─────────────────────────────────────────────────────────────── + +/// Opaque handle wrapping an open `EngramDb`. pub struct EngramHandle { db: EngramDb, } -/// Open an engram database at the given path. +// ── Lifecycle ───────────────────────────────────────────────────────────────── + +/// Open or create an engram database at `path`. /// -/// Returns a heap-allocated handle on success, or null on failure. -/// The caller must eventually call `engram_close` to free the handle. +/// Returns a heap-allocated `EngramHandle` on success, null on error. +/// Must be freed with `engram_close`. /// /// # Safety -/// `path` must be a valid, null-terminated UTF-8 string. +/// `path` must be a valid, non-null, null-terminated UTF-8 string. #[no_mangle] pub unsafe extern "C" fn engram_open(path: *const c_char) -> *mut EngramHandle { if path.is_null() { @@ -42,9 +50,9 @@ pub unsafe extern "C" fn engram_open(path: *const c_char) -> *mut EngramHandle { } } -/// Close and free an engram database handle. +/// Close and free an engram handle. /// -/// After this call, `handle` is invalid and must not be used. +/// After this call `handle` is invalid. /// /// # Safety /// `handle` must have been returned by `engram_open` and not yet freed. @@ -55,64 +63,407 @@ pub unsafe extern "C" fn engram_close(handle: *mut EngramHandle) { } } -/// Return the number of nodes in the database. -/// -/// Returns -1 on error. +// ── Statistics ──────────────────────────────────────────────────────────────── + +/// Return the total number of nodes. Returns -1 on error. /// /// # Safety -/// `handle` must be a valid, non-null pointer from `engram_open`. +/// `handle` must be a valid non-null pointer from `engram_open`. #[no_mangle] pub unsafe extern "C" fn engram_node_count(handle: *const EngramHandle) -> i64 { if handle.is_null() { return -1; } - match (*handle).db.node_count() { - Ok(n) => n as i64, - Err(_) => -1, - } + (*handle).db.node_count().map(|n| n as i64).unwrap_or(-1) } -/// Return the number of edges in the database. -/// -/// Returns -1 on error. +/// Return the total number of edges. Returns -1 on error. /// /// # Safety -/// `handle` must be a valid, non-null pointer from `engram_open`. +/// `handle` must be a valid non-null pointer from `engram_open`. #[no_mangle] pub unsafe extern "C" fn engram_edge_count(handle: *const EngramHandle) -> i64 { if handle.is_null() { return -1; } - match (*handle).db.edge_count() { - Ok(n) => n as i64, - Err(_) => -1, - } + (*handle).db.edge_count().map(|n| n as i64).unwrap_or(-1) } -/// Apply salience decay across all nodes. +// ── Salience management ─────────────────────────────────────────────────────── + +/// Apply multiplicative decay to all node saliences. /// -/// Returns the number of nodes updated, or -1 on error. +/// `factor` should be in (0.0, 1.0). Returns nodes updated, or -1 on error. /// /// # Safety -/// `handle` must be a valid, non-null pointer from `engram_open`. +/// `handle` must be a valid non-null pointer from `engram_open`. #[no_mangle] pub unsafe extern "C" fn engram_decay(handle: *mut EngramHandle, factor: f32) -> i64 { if handle.is_null() { return -1; } - match (*handle).db.decay(factor) { - Ok(n) => n as i64, - Err(_) => -1, + (*handle).db.decay(factor).map(|n| n as i64).unwrap_or(-1) +} + +// ── Node operations ─────────────────────────────────────────────────────────── + +/// Store a node from a JSON representation. +/// +/// `json` must be a UTF-8 JSON object with at least: +/// `{ "content": "...", "node_type": "Memory"|"Concept"|..., "tier": "Episodic"|..., +/// "importance": 0.8, "embedding": [f32, ...] }` +/// +/// Returns a heap-allocated UUID string on success, null on error. +/// Caller must free with `engram_free_string`. +/// +/// # Safety +/// `handle` and `json` must be valid non-null pointers. +#[no_mangle] +pub unsafe extern "C" fn engram_put_node( + handle: *mut EngramHandle, + json: *const c_char, +) -> *mut c_char { + if handle.is_null() || json.is_null() { + return std::ptr::null_mut(); + } + let json_str = match CStr::from_ptr(json).to_str() { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + + let node = match node_from_json(json_str) { + Some(n) => n, + None => return std::ptr::null_mut(), + }; + + match (*handle).db.put_node(node) { + Ok(id) => match CString::new(id.to_string()) { + Ok(s) => s.into_raw(), + Err(_) => std::ptr::null_mut(), + }, + Err(_) => std::ptr::null_mut(), } } -/// Free a C string returned by engram FFI functions. +/// Retrieve a node by UUID and return it as JSON. +/// +/// `id` must be a UUID string. Returns heap-allocated JSON on success, null if +/// not found or on error. Caller must free with `engram_free_string`. /// /// # Safety -/// `s` must have been allocated by an engram FFI function, not by the caller. +/// `handle` and `id` must be valid non-null pointers. +#[no_mangle] +pub unsafe extern "C" fn engram_get_node( + handle: *const EngramHandle, + id: *const c_char, +) -> *mut c_char { + if handle.is_null() || id.is_null() { + return std::ptr::null_mut(); + } + let id_str = match CStr::from_ptr(id).to_str() { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + let uuid = match id_str.parse::() { + Ok(u) => u, + Err(_) => return std::ptr::null_mut(), + }; + match (*handle).db.get_node(uuid) { + Ok(Some(node)) => match CString::new(node_to_json(&node)) { + Ok(s) => s.into_raw(), + Err(_) => std::ptr::null_mut(), + }, + _ => std::ptr::null_mut(), + } +} + +// ── Spreading activation ────────────────────────────────────────────────────── + +/// Run spreading activation and return results as JSON. +/// +/// `req_json` must be: +/// `{ "seeds": ["uuid", ...], "query_embedding": [f32, ...], +/// "max_depth": 3, "limit": 10 }` +/// +/// Returns heap-allocated JSON array of `ActivatedNode` objects, or null. +/// Caller must free with `engram_free_string`. +/// +/// # Safety +/// `handle` and `req_json` must be valid non-null pointers. +#[no_mangle] +pub unsafe extern "C" fn engram_activate( + handle: *const EngramHandle, + req_json: *const c_char, +) -> *mut c_char { + if handle.is_null() || req_json.is_null() { + return std::ptr::null_mut(); + } + let json_str = match CStr::from_ptr(req_json).to_str() { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + + let (seeds, query_emb, max_depth, limit) = match parse_activate_request(json_str) { + Some(r) => r, + None => return std::ptr::null_mut(), + }; + + match (*handle).db.activate(&seeds, &query_emb, max_depth, limit) { + Ok(results) => { + let json = activated_nodes_to_json(&results); + match CString::new(json) { + Ok(s) => s.into_raw(), + Err(_) => std::ptr::null_mut(), + } + } + Err(_) => std::ptr::null_mut(), + } +} + +/// Free a C string returned by any engram FFI function. +/// +/// # Safety +/// `s` must have been allocated by an engram FFI function. Do not call twice. #[no_mangle] pub unsafe extern "C" fn engram_free_string(s: *mut c_char) { if !s.is_null() { drop(CString::from_raw(s)); } } + +// ── JSON helpers ────────────────────────────────────────────────────────────── +// Minimal hand-rolled JSON to avoid adding serde_json as a dependency. +// These are intentionally simple — they handle the subset we need. + +fn node_from_json(json: &str) -> Option { + // Extract fields with simple string scanning. + let content = extract_string(json, "content").unwrap_or_default(); + let node_type_str = extract_string(json, "node_type").unwrap_or_else(|| "Memory".into()); + let tier_str = extract_string(json, "tier").unwrap_or_else(|| "Episodic".into()); + let importance: f32 = extract_number(json, "importance").unwrap_or(0.5); + let embedding = extract_float_array(json, "embedding").unwrap_or_default(); + + let node_type = match node_type_str.as_str() { + "Concept" => NodeType::Concept, + "Event" => NodeType::Event, + "Entity" => NodeType::Entity, + "Process" => NodeType::Process, + "InternalState" => NodeType::InternalState, + _ => NodeType::Memory, + }; + + let tier = match tier_str.as_str() { + "Working" => MemoryTier::Working, + "Semantic" => MemoryTier::Semantic, + "Procedural" => MemoryTier::Procedural, + _ => MemoryTier::Episodic, + }; + + Some(Node::new(node_type, embedding, content.into_bytes(), tier, importance)) +} + +fn node_to_json(node: &Node) -> String { + let content = String::from_utf8_lossy(&node.content); + let node_type = format!("{:?}", node.node_type); + let tier = format!("{:?}", node.tier); + let emb_str = node + .embedding + .iter() + .map(|f| format!("{:.6}", f)) + .collect::>() + .join(","); + format!( + r#"{{"id":"{}","node_type":"{}","tier":"{}","content":"{}","salience":{:.6},"importance":{:.6},"activation_count":{},"embedding":[{}]}}"#, + node.id, + node_type, + tier, + content.replace('"', "\\\""), + node.salience, + node.importance, + node.activation_count, + emb_str, + ) +} + +fn activated_nodes_to_json(nodes: &[ActivatedNode]) -> String { + let items: Vec = nodes + .iter() + .map(|a| { + format!( + r#"{{"node":{},"activation_strength":{:.6},"hops":{}}}"#, + node_to_json(&a.node), + a.activation_strength, + a.hops, + ) + }) + .collect(); + format!("[{}]", items.join(",")) +} + +fn parse_activate_request(json: &str) -> Option<(Vec, Vec, u8, usize)> { + let seeds_raw = extract_string_array(json, "seeds")?; + let seeds: Vec = seeds_raw + .iter() + .filter_map(|s| s.parse::().ok()) + .collect(); + let query_emb = extract_float_array(json, "query_embedding")?; + let max_depth = extract_number(json, "max_depth").unwrap_or(3.0) as u8; + let limit = extract_number(json, "limit").unwrap_or(10.0) as usize; + Some((seeds, query_emb, max_depth, limit)) +} + +// ── Tiny JSON field extractors ──────────────────────────────────────────────── + +fn extract_string(json: &str, key: &str) -> Option { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + if !rest.starts_with('"') { + return None; + } + let inner = &rest[1..]; + let end = inner.find('"')?; + Some(inner[..end].to_string()) +} + +fn extract_number(json: &str, key: &str) -> Option { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + let end = rest + .find(|c: char| c == ',' || c == '}' || c == ']') + .unwrap_or(rest.len()); + rest[..end].trim().parse::().ok() +} + +fn extract_float_array(json: &str, key: &str) -> Option> { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + if !rest.starts_with('[') { + return None; + } + let end = rest.find(']')?; + let inner = &rest[1..end]; + let floats: Vec = inner + .split(',') + .filter_map(|s| s.trim().parse::().ok()) + .collect(); + Some(floats) +} + +fn extract_string_array(json: &str, key: &str) -> Option> { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + if !rest.starts_with('[') { + return None; + } + let end = rest.find(']')?; + let inner = &rest[1..end]; + let strings: Vec = inner + .split(',') + .filter_map(|s| { + let s = s.trim(); + if s.starts_with('"') && s.ends_with('"') { + Some(s[1..s.len() - 1].to_string()) + } else { + None + } + }) + .collect(); + Some(strings) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + #[test] + fn open_and_close() { + let dir = tempfile::tempdir().unwrap(); + let path = CString::new(dir.path().to_str().unwrap()).unwrap(); + unsafe { + let handle = engram_open(path.as_ptr()); + assert!(!handle.is_null()); + engram_close(handle); + } + } + + #[test] + fn null_path_returns_null() { + unsafe { + let handle = engram_open(std::ptr::null()); + assert!(handle.is_null()); + } + } + + #[test] + fn put_and_get_node_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let path = CString::new(dir.path().to_str().unwrap()).unwrap(); + let json = CString::new( + r#"{"content":"hello","node_type":"Memory","tier":"Episodic","importance":0.8,"embedding":[0.1,0.2,0.3]}"#, + ) + .unwrap(); + + unsafe { + let handle = engram_open(path.as_ptr()); + assert!(!handle.is_null()); + + let uuid_ptr = engram_put_node(handle, json.as_ptr()); + assert!(!uuid_ptr.is_null()); + + let uuid_str = CStr::from_ptr(uuid_ptr).to_str().unwrap().to_string(); + engram_free_string(uuid_ptr); + + // Now get the node back. + let id_cstr = CString::new(uuid_str).unwrap(); + let node_json_ptr = engram_get_node(handle, id_cstr.as_ptr()); + assert!(!node_json_ptr.is_null()); + + let node_json = CStr::from_ptr(node_json_ptr).to_str().unwrap().to_string(); + assert!(node_json.contains("hello")); + engram_free_string(node_json_ptr); + + assert_eq!(engram_node_count(handle), 1); + engram_close(handle); + } + } + + #[test] + fn node_count_and_edge_count() { + let dir = tempfile::tempdir().unwrap(); + let path = CString::new(dir.path().to_str().unwrap()).unwrap(); + unsafe { + let handle = engram_open(path.as_ptr()); + assert_eq!(engram_node_count(handle), 0); + assert_eq!(engram_edge_count(handle), 0); + engram_close(handle); + } + } + + #[test] + fn extract_string_works() { + let json = r#"{"content":"hello world","importance":0.5}"#; + assert_eq!(extract_string(json, "content"), Some("hello world".into())); + } + + #[test] + fn extract_number_works() { + let json = r#"{"importance":0.75,"other":1}"#; + let v = extract_number(json, "importance").unwrap(); + assert!((v - 0.75).abs() < 1e-4); + } + + #[test] + fn extract_float_array_works() { + let json = r#"{"embedding":[0.1,0.2,0.3]}"#; + let arr = extract_float_array(json, "embedding").unwrap(); + assert_eq!(arr.len(), 3); + assert!((arr[0] - 0.1).abs() < 1e-4); + } +} diff --git a/crates/engram-jni/Cargo.toml b/crates/engram-jni/Cargo.toml new file mode 100644 index 0000000..2ad0926 --- /dev/null +++ b/crates/engram-jni/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "engram-jni" +version = "0.1.0" +edition = "2021" +description = "JNI bindings for engram-core (Kotlin/Android)" +license = "MIT" + +[lib] +name = "engram_jni" +crate-type = ["cdylib"] + +[dependencies] +engram-core = { path = "../engram-core" } +jni = "0.21" +uuid = { version = "1", features = ["v4", "serde"] } +serde_json = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/engram-jni/src/lib.rs b/crates/engram-jni/src/lib.rs new file mode 100644 index 0000000..6dc2523 --- /dev/null +++ b/crates/engram-jni/src/lib.rs @@ -0,0 +1,496 @@ +/// JNI bindings for engram-core. +/// +/// These functions expose the Engram API to Kotlin/JVM callers via Java Native Interface. +/// The convention is: +/// +/// Java___ +/// → Java_ai_neuron_engram_EngramDb_ +/// +/// The `EngramDb` handle is stored as a Java `long` (native pointer). The Kotlin +/// wrapper class casts it to/from `Long` and keeps it private. +/// +/// # Memory model +/// - `open` allocates an `EngramHandle` on the Rust heap and returns its address as `jlong`. +/// - `close` takes that `jlong`, reconstructs the Box, and drops it. +/// - All other methods borrow the handle via `&*ptr`. +/// +/// # JSON wire format +/// Nodes and results are passed as JSON strings to avoid bespoke JNI object marshalling. +/// The Kotlin layer converts between the data classes and JSON. +use engram_core::{ActivatedNode, EngramDb, MemoryTier, Node, NodeType, ScoredNode}; +use jni::objects::{JClass, JString}; +use jni::sys::{jfloatArray, jint, jlong, jstring}; +use jni::JNIEnv; +use std::path::Path; +use uuid::Uuid; + +// ── Handle ──────────────────────────────────────────────────────────────────── + +struct EngramHandle { + db: EngramDb, +} + +// ── Helper macros ───────────────────────────────────────────────────────────── + +macro_rules! handle_ref { + ($handle:expr) => { + unsafe { &*($handle as *const EngramHandle) } + }; +} + +// ── JNI methods: lifecycle ──────────────────────────────────────────────────── + +/// Open an engram database and return a native handle as jlong. +/// +/// Kotlin: `external fun open(path: String): Long` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_open( + mut env: JNIEnv, + _class: JClass, + path: JString, +) -> jlong { + let path_str: String = match env.get_string(&path) { + Ok(s) => s.into(), + Err(_) => return 0, + }; + match EngramDb::open(Path::new(&path_str)) { + Ok(db) => { + let handle = Box::new(EngramHandle { db }); + Box::into_raw(handle) as jlong + } + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + 0 + } + } +} + +/// Close and free a database handle. +/// +/// Kotlin: `external fun close(handle: Long)` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_close( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + if handle != 0 { + unsafe { + drop(Box::from_raw(handle as *mut EngramHandle)); + } + } +} + +// ── JNI methods: statistics ─────────────────────────────────────────────────── + +/// Kotlin: `external fun nodeCount(handle: Long): Long` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_nodeCount( + mut env: JNIEnv, + _class: JClass, + handle: jlong, +) -> jlong { + let h = handle_ref!(handle); + match h.db.node_count() { + Ok(n) => n as jlong, + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + -1 + } + } +} + +/// Kotlin: `external fun edgeCount(handle: Long): Long` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_edgeCount( + mut env: JNIEnv, + _class: JClass, + handle: jlong, +) -> jlong { + let h = handle_ref!(handle); + match h.db.edge_count() { + Ok(n) => n as jlong, + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + -1 + } + } +} + +// ── JNI methods: nodes ──────────────────────────────────────────────────────── + +/// Store a node from JSON and return the assigned UUID string. +/// +/// Kotlin: `external fun putNode(handle: Long, nodeJson: String): String` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_putNode( + mut env: JNIEnv, + _class: JClass, + handle: jlong, + node_json: JString, +) -> jstring { + let json: String = match env.get_string(&node_json) { + Ok(s) => s.into(), + Err(_) => return std::ptr::null_mut(), + }; + + let node = match node_from_json(&json) { + Some(n) => n, + None => { + let _ = env.throw_new("java/lang/IllegalArgumentException", "Invalid node JSON"); + return std::ptr::null_mut(); + } + }; + + let h = handle_ref!(handle); + match h.db.put_node(node) { + Ok(id) => { + let id_str = id.to_string(); + env.new_string(&id_str) + .map(|s| s.into_raw()) + .unwrap_or(std::ptr::null_mut()) + } + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + std::ptr::null_mut() + } + } +} + +/// Retrieve a node by UUID, returned as JSON, or null if not found. +/// +/// Kotlin: `external fun getNode(handle: Long, id: String): String?` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_getNode( + mut env: JNIEnv, + _class: JClass, + handle: jlong, + id: JString, +) -> jstring { + let id_str: String = match env.get_string(&id) { + Ok(s) => s.into(), + Err(_) => return std::ptr::null_mut(), + }; + let uuid = match id_str.parse::() { + Ok(u) => u, + Err(_) => return std::ptr::null_mut(), + }; + + let h = handle_ref!(handle); + match h.db.get_node(uuid) { + Ok(Some(node)) => { + let json = node_to_json(&node); + env.new_string(&json) + .map(|s| s.into_raw()) + .unwrap_or(std::ptr::null_mut()) + } + Ok(None) => std::ptr::null_mut(), + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + std::ptr::null_mut() + } + } +} + +// ── JNI methods: search ─────────────────────────────────────────────────────── + +/// Search for similar nodes by embedding vector. +/// Returns a JSON array of scored nodes. +/// +/// Kotlin: `external fun searchEmbedding(handle: Long, embedding: FloatArray, limit: Int): String` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_searchEmbedding( + mut env: JNIEnv, + _class: JClass, + handle: jlong, + embedding: jfloatArray, + limit: jint, +) -> jstring { + let emb = match float_array_from_jni(&mut env, embedding) { + Some(v) => v, + None => return std::ptr::null_mut(), + }; + + let h = handle_ref!(handle); + match h.db.search_embedding(&emb, limit as usize) { + Ok(results) => { + let json = scored_nodes_to_json(&results); + env.new_string(&json) + .map(|s| s.into_raw()) + .unwrap_or(std::ptr::null_mut()) + } + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + std::ptr::null_mut() + } + } +} + +/// Run spreading activation. +/// `seeds_json` is a JSON array of UUID strings. +/// Returns a JSON array of activated nodes. +/// +/// Kotlin: `external fun activate(handle: Long, seedsJson: String, queryEmbedding: FloatArray, maxDepth: Int, limit: Int): String` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_activate( + mut env: JNIEnv, + _class: JClass, + handle: jlong, + seeds_json: JString, + query_embedding: jfloatArray, + max_depth: jint, + limit: jint, +) -> jstring { + let seeds_str: String = match env.get_string(&seeds_json) { + Ok(s) => s.into(), + Err(_) => return std::ptr::null_mut(), + }; + + let seeds: Vec = parse_uuid_array(&seeds_str); + let query_emb = match float_array_from_jni(&mut env, query_embedding) { + Some(v) => v, + None => return std::ptr::null_mut(), + }; + + let h = handle_ref!(handle); + match h.db.activate(&seeds, &query_emb, max_depth as u8, limit as usize) { + Ok(results) => { + let json = activated_nodes_to_json(&results); + env.new_string(&json) + .map(|s| s.into_raw()) + .unwrap_or(std::ptr::null_mut()) + } + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + std::ptr::null_mut() + } + } +} + +// ── JNI methods: salience ───────────────────────────────────────────────────── + +/// Touch a node (increment activation count and update salience). +/// +/// Kotlin: `external fun touch(handle: Long, id: String)` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_touch( + mut env: JNIEnv, + _class: JClass, + handle: jlong, + id: JString, +) { + let id_str: String = match env.get_string(&id) { + Ok(s) => s.into(), + Err(_) => return, + }; + let uuid = match id_str.parse::() { + Ok(u) => u, + Err(_) => return, + }; + let h = handle_ref!(handle); + if let Err(e) = h.db.touch(uuid) { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + } +} + +/// Apply salience decay. Returns the number of nodes updated. +/// +/// Kotlin: `external fun decay(handle: Long, factor: Float): Int` +#[no_mangle] +pub extern "system" fn Java_ai_neuron_engram_EngramDb_decay( + mut env: JNIEnv, + _class: JClass, + handle: jlong, + factor: f32, +) -> jint { + let h = handle_ref!(handle); + match h.db.decay(factor) { + Ok(n) => n as jint, + Err(e) => { + let _ = env.throw_new("java/lang/RuntimeException", e.to_string()); + -1 + } + } +} + +// ── JNI helpers ─────────────────────────────────────────────────────────────── + +fn float_array_from_jni(env: &mut JNIEnv, arr: jfloatArray) -> Option> { + if arr.is_null() { + return None; + } + let arr_obj = unsafe { jni::objects::JFloatArray::from_raw(arr) }; + let len = env.get_array_length(&arr_obj).ok()? as usize; + let mut buf = vec![0f32; len]; + env.get_float_array_region(&arr_obj, 0, &mut buf).ok()?; + Some(buf) +} + +fn parse_uuid_array(json: &str) -> Vec { + // Minimal parser: `["uuid1","uuid2",...]` + json.trim_matches(|c| c == '[' || c == ']') + .split(',') + .filter_map(|s| { + let s = s.trim().trim_matches('"'); + s.parse::().ok() + }) + .collect() +} + +// ── JSON helpers ────────────────────────────────────────────────────────────── + +fn node_from_json(json: &str) -> Option { + let content = extract_string_field(json, "content").unwrap_or_default(); + let node_type_str = + extract_string_field(json, "node_type").unwrap_or_else(|| "Memory".into()); + let tier_str = extract_string_field(json, "tier").unwrap_or_else(|| "Episodic".into()); + let importance: f32 = extract_f32_field(json, "importance").unwrap_or(0.5); + let embedding = extract_f32_array(json, "embedding").unwrap_or_default(); + + let node_type = match node_type_str.as_str() { + "Concept" => NodeType::Concept, + "Event" => NodeType::Event, + "Entity" => NodeType::Entity, + "Process" => NodeType::Process, + "InternalState" => NodeType::InternalState, + _ => NodeType::Memory, + }; + let tier = match tier_str.as_str() { + "Working" => MemoryTier::Working, + "Semantic" => MemoryTier::Semantic, + "Procedural" => MemoryTier::Procedural, + _ => MemoryTier::Episodic, + }; + Some(Node::new(node_type, embedding, content.into_bytes(), tier, importance)) +} + +fn node_to_json(node: &Node) -> String { + let content = String::from_utf8_lossy(&node.content) + .replace('\\', "\\\\") + .replace('"', "\\\""); + let emb_str = node + .embedding + .iter() + .map(|f| format!("{:.6}", f)) + .collect::>() + .join(","); + format!( + r#"{{"id":"{}","node_type":"{:?}","tier":"{:?}","content":"{}","salience":{:.6},"importance":{:.6},"activation_count":{},"embedding":[{}]}}"#, + node.id, node.node_type, node.tier, content, node.salience, node.importance, + node.activation_count, emb_str, + ) +} + +fn scored_nodes_to_json(nodes: &[ScoredNode]) -> String { + let items: Vec = nodes + .iter() + .map(|s| format!(r#"{{"node":{},"score":{:.6}}}"#, node_to_json(&s.node), s.score)) + .collect(); + format!("[{}]", items.join(",")) +} + +fn activated_nodes_to_json(nodes: &[ActivatedNode]) -> String { + let items: Vec = nodes + .iter() + .map(|a| { + format!( + r#"{{"node":{},"activation_strength":{:.6},"hops":{}}}"#, + node_to_json(&a.node), a.activation_strength, a.hops, + ) + }) + .collect(); + format!("[{}]", items.join(",")) +} + +fn extract_string_field(json: &str, key: &str) -> Option { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + if !rest.starts_with('"') { + return None; + } + let inner = &rest[1..]; + let end = inner.find('"')?; + Some(inner[..end].to_string()) +} + +fn extract_f32_field(json: &str, key: &str) -> Option { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + let end = rest + .find(|c: char| c == ',' || c == '}') + .unwrap_or(rest.len()); + rest[..end].trim().parse::().ok() +} + +fn extract_f32_array(json: &str, key: &str) -> Option> { + let needle = format!("\"{}\":", key); + let start = json.find(&needle)? + needle.len(); + let rest = json[start..].trim_start(); + if !rest.starts_with('[') { + return None; + } + let end = rest.find(']')?; + let inner = &rest[1..end]; + Some( + inner + .split(',') + .filter_map(|s| s.trim().parse::().ok()) + .collect(), + ) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn node_json_roundtrip() { + let node = Node::new( + NodeType::Memory, + vec![0.1, 0.2, 0.3], + b"test content".to_vec(), + MemoryTier::Episodic, + 0.8, + ); + let json = node_to_json(&node); + assert!(json.contains("test content")); + assert!(json.contains("Memory")); + assert!(json.contains("Episodic")); + } + + #[test] + fn parse_uuid_array_valid() { + let uuids = parse_uuid_array(r#"["550e8400-e29b-41d4-a716-446655440000"]"#); + assert_eq!(uuids.len(), 1); + } + + #[test] + fn parse_uuid_array_empty() { + let uuids = parse_uuid_array("[]"); + assert_eq!(uuids.len(), 0); + } + + #[test] + fn extract_string_field_works() { + let json = r#"{"content":"hello","type":"Memory"}"#; + assert_eq!(extract_string_field(json, "content"), Some("hello".into())); + assert_eq!(extract_string_field(json, "type"), Some("Memory".into())); + } + + #[test] + fn extract_f32_array_works() { + let json = r#"{"embedding":[0.1,0.2,0.3]}"#; + let arr = extract_f32_array(json, "embedding").unwrap(); + assert_eq!(arr.len(), 3); + } + + #[test] + fn activated_nodes_json_is_array() { + let json = activated_nodes_to_json(&[]); + assert_eq!(json, "[]"); + } +} diff --git a/crates/engram-migrate/Cargo.toml b/crates/engram-migrate/Cargo.toml new file mode 100644 index 0000000..3986ae4 --- /dev/null +++ b/crates/engram-migrate/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "engram-migrate" +version = "0.1.0" +edition = "2021" +description = "CLI tool: migrate a Neuron SQLite database into an Engram sled store" +license = "MIT" + +[[bin]] +name = "engram-migrate" +path = "src/main.rs" + +[dependencies] +engram-core = { path = "../engram-core", features = ["sled-backend", "migration"] } diff --git a/crates/engram-migrate/src/main.rs b/crates/engram-migrate/src/main.rs new file mode 100644 index 0000000..d9520d3 --- /dev/null +++ b/crates/engram-migrate/src/main.rs @@ -0,0 +1,105 @@ +/// engram-migrate — import a Neuron SQLite database into an Engram sled store. +/// +/// Usage: +/// engram-migrate --sqlite ~/.neuron/neuron.db --output ~/.engram/neuron +/// +/// The tool reads memory_nodes, knowledge_entries, and graph_edges from the +/// Neuron SQLite database and writes them to a new Engram sled store. +/// +/// Embeddings are placeholder random unit vectors (dimension 384 by default). +/// Re-run with a real embedding model once the ONNX engine is available. +use engram_core::migration::{migrate_from_neuron, MigrationConfig}; +use std::path::PathBuf; +use std::process; + +fn main() { + let args: Vec = std::env::args().collect(); + + if args.len() < 5 { + eprintln!("Usage: engram-migrate --sqlite --output "); + eprintln!(" --sqlite Path to the Neuron SQLite database (e.g. ~/.neuron/neuron.db)"); + eprintln!(" --output Path for the new Engram sled store (e.g. ~/.engram/neuron)"); + process::exit(1); + } + + let mut sqlite_path: Option = None; + let mut output_path: Option = None; + let mut embedding_dim: usize = 384; + + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "--sqlite" => { + i += 1; + sqlite_path = Some(PathBuf::from(&args[i])); + } + "--output" => { + i += 1; + output_path = Some(PathBuf::from(&args[i])); + } + "--embedding-dim" => { + i += 1; + embedding_dim = args[i].parse().unwrap_or(384); + } + _ => { + eprintln!("Unknown argument: {}", args[i]); + process::exit(1); + } + } + i += 1; + } + + let sqlite_path = match sqlite_path { + Some(p) => p, + None => { + eprintln!("Missing --sqlite argument"); + process::exit(1); + } + }; + + let output_path = match output_path { + Some(p) => p, + None => { + eprintln!("Missing --output argument"); + process::exit(1); + } + }; + + if !sqlite_path.exists() { + eprintln!("SQLite file not found: {}", sqlite_path.display()); + process::exit(1); + } + + println!("Migrating Neuron database..."); + println!(" Source: {}", sqlite_path.display()); + println!(" Output: {}", output_path.display()); + println!(" Embedding dim: {}", embedding_dim); + println!(); + + let config = MigrationConfig { + sqlite_path, + engram_path: output_path, + embedding_dim, + }; + + match migrate_from_neuron(&config) { + Ok(report) => { + println!("Migration complete."); + println!(" Memories migrated: {}", report.memories_migrated); + println!(" Knowledge migrated: {}", report.knowledge_migrated); + println!(" Edges created: {}", report.edges_created); + + if !report.errors.is_empty() { + println!(); + println!("Non-fatal errors ({}):", report.errors.len()); + for e in &report.errors { + println!(" - {}", e); + } + } + } + Err(e) => { + eprintln!("Migration failed: {}", e); + process::exit(1); + } + } +} diff --git a/examples/basic.rs b/examples/basic.rs index 40fb436..a66713d 100644 --- a/examples/basic.rs +++ b/examples/basic.rs @@ -1,11 +1,14 @@ /// Basic engram demonstration. /// /// This example builds a small memory graph, runs spreading activation, -/// performs a vector search, and shows salience decay in action. +/// performs a vector search, shows salience decay, and demonstrates +/// the consolidation engine promoting Episodic nodes to Semantic. /// /// The nodes represent a tiny knowledge graph about the spreading activation /// model itself — somewhat recursive, intentionally. -use engram_core::{ActivatedNode, Edge, EngramDb, MemoryTier, Node, NodeType, RelationType}; +use engram_core::{ + ActivatedNode, ConsolidationConfig, Edge, EngramDb, MemoryTier, Node, NodeType, RelationType, +}; use std::path::Path; fn main() -> Result<(), Box> { @@ -219,6 +222,43 @@ fn main() -> Result<(), Box> { } println!(); - println!("Done. Engram v0.1."); + + // ── 8. Consolidation ────────────────────────────────────────────────────── + // + // Touch node2 (Hebbian learning — Episodic) several times to simulate it + // being frequently recalled, then run consolidation to promote it. + + println!("=== Memory Consolidation ==="); + + // Simulate repeated activation: touch node2 six times so it crosses the + // default threshold of 5 activations. + for _ in 0..6 { + db.touch(id2)?; + } + + // Confirm the node's activation count has increased. + if let Some(n) = db.get_node(id2)? { + println!(" node2 (Episodic) activation_count before consolidation: {}", n.activation_count); + println!(" node2 tier before consolidation: {:?}", n.tier); + } + + let config = ConsolidationConfig { + episodic_to_semantic_threshold: 5, + salience_floor: 0.0, // no salience floor — accept any salient node + max_promotions_per_run: 10, + decay_factor: 0.98, + }; + + let report = db.consolidate(&config)?; + println!(" Promoted {} Episodic → Semantic", report.promoted); + println!(" Decayed {} nodes", report.decayed); + + // Confirm node2 has been promoted. + if let Some(n) = db.get_node(id2)? { + println!(" node2 tier after consolidation: {:?}", n.tier); + } + + println!(); + println!("Done. Engram v0.1.1."); Ok(()) } diff --git a/examples/migrate.rs b/examples/migrate.rs new file mode 100644 index 0000000..d793de1 --- /dev/null +++ b/examples/migrate.rs @@ -0,0 +1,96 @@ +/// Migration example — shows the migrate_from_neuron API. +/// +/// This example creates a tiny in-memory SQLite database that mimics the +/// Neuron schema, then migrates it into an Engram sled store and prints +/// the resulting node count. +/// +/// In production: +/// use engram_core::migration::{migrate_from_neuron, MigrationConfig}; +/// let config = MigrationConfig::new( +/// PathBuf::from(shellexpand::tilde("~/.neuron/neuron.db").as_ref()), +/// PathBuf::from(shellexpand::tilde("~/.engram/neuron").as_ref()), +/// ); +/// let report = migrate_from_neuron(&config)?; +#[cfg(feature = "migration")] +fn main() -> Result<(), Box> { + use engram_core::migration::{migrate_from_neuron, MigrationConfig}; + use rusqlite::Connection; + use std::path::PathBuf; + + // 1. Create a temp Neuron-like SQLite database. + let tmp = tempfile::tempdir()?; + let sqlite_path = tmp.path().join("neuron.db"); + let engram_path = tmp.path().join("engram"); + + let conn = Connection::open(&sqlite_path)?; + conn.execute_batch( + "CREATE TABLE memory_nodes ( + id TEXT PRIMARY KEY, content TEXT NOT NULL, + importance TEXT NOT NULL DEFAULT 'normal', + superseded_by TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE TABLE knowledge_entries ( + id TEXT PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, + category TEXT NOT NULL DEFAULT '', tier TEXT NOT NULL DEFAULT 'note', + tags TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE TABLE graph_edges ( + from_id TEXT NOT NULL, from_type TEXT NOT NULL, + to_id TEXT NOT NULL, to_type TEXT NOT NULL, + edge_type TEXT NOT NULL, weight REAL NOT NULL DEFAULT 1.0, + PRIMARY KEY (from_id, to_id, edge_type) + );", + )?; + + // Insert sample data. + for i in 0..5 { + conn.execute( + "INSERT INTO memory_nodes (id, content, importance, created_at, updated_at) + VALUES (?1, ?2, 'normal', 1000, 1000)", + rusqlite::params![format!("mem-{i}"), format!("Memory node {i}")], + )?; + } + for i in 0..3 { + conn.execute( + "INSERT INTO knowledge_entries (id, title, content, created_at, updated_at) + VALUES (?1, ?2, ?3, 2000, 2000)", + rusqlite::params![ + format!("kn-{i}"), + format!("Concept {i}"), + format!("Body of knowledge entry {i}"), + ], + )?; + } + drop(conn); + + // 2. Run the migration. + println!("Running migration..."); + let config = MigrationConfig { + sqlite_path, + engram_path, + embedding_dim: 64, + }; + + let report = migrate_from_neuron(&config)?; + + println!("Migration complete."); + println!(" Memories migrated: {}", report.memories_migrated); + println!(" Knowledge migrated: {}", report.knowledge_migrated); + println!(" Edges created: {}", report.edges_created); + if !report.errors.is_empty() { + println!(" Errors: {:?}", report.errors); + } + + // 3. Open the result and check counts. + let db = engram_core::EngramDb::open(&config.engram_path)?; + println!(); + println!("Engram node count: {}", db.node_count()?); + + Ok(()) +} + +#[cfg(not(feature = "migration"))] +fn main() { + eprintln!("This example requires the 'migration' feature."); + eprintln!("Run with: cargo run --example migrate --features migration"); +}