From 6601761cd99787f4b74a97406dde60b045b1ebe8 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Mon, 27 Apr 2026 17:18:51 -0500 Subject: [PATCH] =?UTF-8?q?feat:=20Engram=20sync=20layer=20=E2=80=94=20swa?= =?UTF-8?q?rm=20memory=20protocol,=20peer=20delta=20sync,=20distributed=20?= =?UTF-8?q?activation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engram/Cargo.lock | 1643 ++++++++- engram/Cargo.toml | 2 + engram/crates/engram-core/src/db.rs | 28 + engram/crates/engram-server/Cargo.toml | 30 + engram/crates/engram-server/src/auth.rs | 31 + engram/crates/engram-server/src/main.rs | 165 + .../crates/engram-server/src/routes/core.rs | 218 ++ engram/crates/engram-server/src/routes/mod.rs | 3 + .../crates/engram-server/src/routes/swarm.rs | 124 + .../crates/engram-server/src/routes/sync.rs | 133 + engram/crates/engram-server/src/state.rs | 13 + engram/crates/engram-sync/Cargo.toml | 24 + engram/crates/engram-sync/src/client.rs | 122 + engram/crates/engram-sync/src/engine.rs | 423 +++ engram/crates/engram-sync/src/lib.rs | 265 ++ engram/crates/engram-sync/src/types.rs | 166 + engram/studio/index.html | 3258 +++++++++++++++++ 17 files changed, 6637 insertions(+), 11 deletions(-) create mode 100644 engram/crates/engram-server/Cargo.toml create mode 100644 engram/crates/engram-server/src/auth.rs create mode 100644 engram/crates/engram-server/src/main.rs create mode 100644 engram/crates/engram-server/src/routes/core.rs create mode 100644 engram/crates/engram-server/src/routes/mod.rs create mode 100644 engram/crates/engram-server/src/routes/swarm.rs create mode 100644 engram/crates/engram-server/src/routes/sync.rs create mode 100644 engram/crates/engram-server/src/state.rs create mode 100644 engram/crates/engram-sync/Cargo.toml create mode 100644 engram/crates/engram-sync/src/client.rs create mode 100644 engram/crates/engram-sync/src/engine.rs create mode 100644 engram/crates/engram-sync/src/lib.rs create mode 100644 engram/crates/engram-sync/src/types.rs create mode 100644 engram/studio/index.html diff --git a/engram/Cargo.lock b/engram/Cargo.lock index c6ece74..5c26306 100644 --- a/engram/Cargo.lock +++ b/engram/Cargo.lock @@ -14,18 +14,140 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "auto-future" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1e7e457ea78e524f48639f551fd79703ac3f2237f5ecccdf4708f8a75ad373" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.4.0", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.4.0", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-test" +version = "14.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167294800740b4b6bc7bfbccbf3a1d50a6c6e097342580ec4c11d1672e456292" +dependencies = [ + "anyhow", + "async-trait", + "auto-future", + "axum", + "bytes", + "cookie", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", + "mime", + "pretty_assertions", + "reserve-port", + "rust-multipart-rfc7578_2", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "tokio", + "tower 0.4.13", + "url", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bincode" version = "1.3.3" @@ -65,6 +187,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cesu8" version = "1.1.0" @@ -87,6 +219,42 @@ dependencies = [ "memchr", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "crc32fast" version = "1.5.0" @@ -121,12 +289,47 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "engram" version = "0.1.1" @@ -145,7 +348,7 @@ dependencies = [ "serde", "sled", "tempfile", - "thiserror", + "thiserror 1.0.69", "uuid", ] @@ -176,6 +379,42 @@ dependencies = [ "engram-core", ] +[[package]] +name = "engram-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "axum-test", + "engram-core", + "engram-sync", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tower 0.4.13", + "tower-http 0.5.2", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "engram-sync" +version = "0.1.0" +dependencies = [ + "anyhow", + "engram-core", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "uuid", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -210,12 +449,48 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs2" version = "0.4.3" @@ -226,6 +501,53 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -259,6 +581,25 @@ dependencies = [ "wasip3", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -304,12 +645,255 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http 1.4.0", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http 1.4.0", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.4.0", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -346,6 +930,22 @@ dependencies = [ "serde-big-array", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itoa" version = "1.0.18" @@ -363,7 +963,7 @@ dependencies = [ "combine", "jni-sys 0.3.1", "log", - "thiserror", + "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] @@ -402,10 +1002,18 @@ version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -434,6 +1042,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -449,12 +1063,86 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + [[package]] name = "num-traits" version = "0.2.19" @@ -480,6 +1168,50 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl" +version = "0.10.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-float" version = "3.9.2" @@ -537,12 +1269,59 @@ dependencies = [ "windows-link", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -552,6 +1331,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -654,6 +1443,86 @@ dependencies = [ "bitflags 2.11.1", ] +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http 1.4.0", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower 0.5.3", + "tower-http 0.6.8", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "reserve-port" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94070964579245eb2f76e62a7668fe87bd9969ed6c41256f3bf614e3323dd3cc" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rusqlite" version = "0.31.0" @@ -668,6 +1537,22 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust-multipart-rfc7578_2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b748410c0afdef2ebbe3685a6a862e2ee937127cdaae623336a459451c8d57" +dependencies = [ + "bytes", + "futures-core", + "futures-util", + "http 0.2.12", + "mime", + "mime_guess", + "rand", + "thiserror 1.0.69", +] + [[package]] name = "rustix" version = "1.1.4" @@ -681,12 +1566,51 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -696,12 +1620,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -760,6 +1716,60 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "sled" version = "0.34.7" @@ -782,6 +1792,28 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -793,6 +1825,47 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -812,7 +1885,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -826,6 +1908,289 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot 0.12.5", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.11.1", + "bytes", + "futures-util", + "http 1.4.0", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.11.1", + "bytes", + "futures-util", + "http 1.4.0", + "http-body", + "iri-string", + "pin-project-lite", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -838,6 +2203,30 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.23.1" @@ -850,6 +2239,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -872,6 +2267,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -909,6 +2313,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.118" @@ -975,6 +2389,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1012,13 +2436,51 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -1036,13 +2498,29 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -1051,42 +2529,90 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -1181,6 +2707,41 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.48" @@ -1201,6 +2762,66 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/engram/Cargo.toml b/engram/Cargo.toml index f0c64c9..1e2a64f 100644 --- a/engram/Cargo.toml +++ b/engram/Cargo.toml @@ -5,6 +5,8 @@ members = [ "crates/engram-ffi", "crates/engram-jni", "crates/engram-migrate", + "crates/engram-sync", + "crates/engram-server", # engram-wasm is in bindings/ and compiled separately via wasm-pack # (wasm targets can't be in the same workspace build as native targets) ] diff --git a/engram/crates/engram-core/src/db.rs b/engram/crates/engram-core/src/db.rs index f499a62..e42b75c 100644 --- a/engram/crates/engram-core/src/db.rs +++ b/engram/crates/engram-core/src/db.rs @@ -187,6 +187,34 @@ mod sled_impl { pub fn edge_count(&self) -> EngramResult { graph::edge_count(&self.db) } + + // ── Bulk scan (for sync) ─────────────────────────────────────────────── + + /// Scan all nodes in the store. + /// + /// Used by the sync engine to generate delta snapshots. + pub fn scan_nodes(&self) -> EngramResult> { + storage::scan_nodes(&self.db) + } + + /// Scan all edges in the store (forward index only). + pub fn scan_edges(&self) -> EngramResult> { + let prefix = b"edges:from:"; + let mut edges = Vec::new(); + for result in self.db.scan_prefix(prefix) { + let (_k, v) = result?; + let edge: Edge = bincode::deserialize(&v)?; + edges.push(edge); + } + Ok(edges) + } + + /// Delete a node by UUID (tombstone support for sync). + pub fn delete_node(&self, id: Uuid) -> EngramResult<()> { + let key = storage::node_key(id); + self.db.remove(key)?; + Ok(()) + } } } diff --git a/engram/crates/engram-server/Cargo.toml b/engram/crates/engram-server/Cargo.toml new file mode 100644 index 0000000..5edada8 --- /dev/null +++ b/engram/crates/engram-server/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "engram-server" +version = "0.1.0" +edition = "2021" +description = "HTTP server for Engram — REST API + sync endpoints + swarm activation" +license = "MIT" + +[[bin]] +name = "engram-server" +path = "src/main.rs" + +[dependencies] +engram-core = { path = "../engram-core" } +engram-sync = { path = "../engram-sync" } +axum = { version = "0.7", features = ["json"] } +tokio = { version = "1", features = ["full"] } +tower = "0.4" +tower-http = { version = "0.5", features = ["cors", "fs"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +uuid = { version = "1", features = ["v4", "serde"] } +anyhow = "1" +thiserror = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[dev-dependencies] +axum-test = "14" +tokio = { version = "1", features = ["full"] } +tempfile = "3" diff --git a/engram/crates/engram-server/src/auth.rs b/engram/crates/engram-server/src/auth.rs new file mode 100644 index 0000000..8c92db8 --- /dev/null +++ b/engram/crates/engram-server/src/auth.rs @@ -0,0 +1,31 @@ +/// Auth middleware for sync endpoints. +/// +/// Sync and swarm endpoints require `Authorization: Bearer {api_key}`. +/// The API key is configured at server startup and stored in AppState. +use axum::{ + extract::{Request, State}, + http::StatusCode, + middleware::Next, + response::Response, +}; +use std::sync::Arc; + +use crate::state::AppState; + +/// Axum middleware that checks the Authorization header on sync/swarm routes. +pub async fn require_auth( + State(state): State>, + req: Request, + next: Next, +) -> Result { + let api_key = req + .headers() + .get("Authorization") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")); + + match api_key { + Some(key) if key == state.api_key => Ok(next.run(req).await), + _ => Err(StatusCode::UNAUTHORIZED), + } +} diff --git a/engram/crates/engram-server/src/main.rs b/engram/crates/engram-server/src/main.rs new file mode 100644 index 0000000..7825301 --- /dev/null +++ b/engram/crates/engram-server/src/main.rs @@ -0,0 +1,165 @@ +/// Engram Server — HTTP API for Engram with sync and swarm activation. +/// +/// # Endpoints +/// +/// ## Core +/// GET /stats — node/edge counts +/// POST /nodes — create a node +/// GET /nodes/{id} — get a node +/// POST /edges — create an edge +/// GET /nodes/{id}/edges — list edges from a node +/// POST /activate — spreading activation +/// POST /search — embedding search +/// POST /decay — apply salience decay +/// POST /consolidate — promote Episodic → Semantic +/// +/// ## Sync (auth required) +/// GET /sync/delta?since={ms}&peer_id={uuid} — generate delta +/// POST /sync/push — receive incoming delta +/// POST /sync/peers — register peer +/// GET /sync/peers — list peers +/// DELETE /sync/peers/{id} — remove peer +/// +/// ## Swarm +/// POST /swarm/activate — distributed activation (auth required) +/// GET /swarm/status — peer health (auth required) +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::{ + middleware, + routing::{delete, get, post}, + Router, +}; +use engram_core::EngramDb; +use engram_sync::{SyncConfig, SyncEngine}; +use tokio::time::interval; +use tower_http::cors::CorsLayer; +use tracing::info; +use tracing_subscriber::EnvFilter; + +mod auth; +mod routes; +mod state; + +use auth::require_auth; +use state::AppState; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Logging + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + // Configuration from environment (with sensible defaults) + let db_path = std::env::var("ENGRAM_DB_PATH").unwrap_or_else(|_| "./engram-data".to_string()); + let bind_addr = std::env::var("ENGRAM_BIND").unwrap_or_else(|_| "0.0.0.0:8742".to_string()); + let api_key = std::env::var("ENGRAM_API_KEY").unwrap_or_else(|_| { + let key = uuid::Uuid::new_v4().to_string(); + eprintln!("No ENGRAM_API_KEY set — generated key: {}", key); + key + }); + + // Open database + let db = EngramDb::open(&PathBuf::from(&db_path))?; + let db = Arc::new(Mutex::new(db)); + + info!("Database opened at {}", db_path); + + // Sync engine — wrapped in tokio::sync::Mutex so it can be held across .await + let sync_config = SyncConfig { + our_id: uuid::Uuid::new_v4(), + our_name: std::env::var("ENGRAM_PEER_NAME").unwrap_or_else(|_| "engram-local".to_string()), + api_key: api_key.clone(), + default_sync_tiers: vec![engram_core::types::MemoryTier::Semantic], + sync_interval_secs: std::env::var("ENGRAM_SYNC_INTERVAL_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(300), + }; + let sync_interval_secs = sync_config.sync_interval_secs; + let sync_engine = Arc::new(tokio::sync::Mutex::new(SyncEngine::new(db.clone(), sync_config))); + + { + let e = sync_engine.lock().await; + info!( + peer_name = e.our_name(), + peer_id = %e.our_id(), + sync_interval_secs, + "Sync engine ready" + ); + } + + // Background sync task — tokio::sync::Mutex guard is Send-safe + { + let engine_arc = sync_engine.clone(); + tokio::spawn(async move { + let mut ticker = interval(Duration::from_secs(sync_interval_secs)); + loop { + ticker.tick().await; + let report = { + let mut e = engine_arc.lock().await; + e.sync_all().await + }; + if report.peers_synced > 0 || !report.errors.is_empty() { + info!( + peers_synced = report.peers_synced, + nodes_received = report.nodes_received, + nodes_sent = report.nodes_sent, + errors = report.errors.len(), + "Sync cycle complete" + ); + } + } + }); + } + + // Shared state + let state = Arc::new(AppState { + db: db.clone(), + sync_engine: sync_engine.clone(), + api_key: api_key.clone(), + }); + + // Protected sync/swarm routes (auth middleware applied) + let sync_routes = Router::new() + .route("/sync/delta", get(routes::sync::get_delta)) + .route("/sync/push", post(routes::sync::push_delta)) + .route("/sync/peers", get(routes::sync::list_peers)) + .route("/sync/peers", post(routes::sync::register_peer)) + .route("/sync/peers/{id}", delete(routes::sync::delete_peer)) + .route("/swarm/activate", post(routes::swarm::swarm_activate)) + .route("/swarm/status", get(routes::swarm::swarm_status)) + .layer(middleware::from_fn_with_state( + state.clone(), + require_auth, + )); + + // Open core routes (no auth) + let core_routes = Router::new() + .route("/stats", get(routes::core::get_stats)) + .route("/nodes", post(routes::core::create_node)) + .route("/nodes/{id}", get(routes::core::get_node)) + .route("/edges", post(routes::core::create_edge)) + .route("/nodes/{id}/edges", get(routes::core::get_edges_from)) + .route("/activate", post(routes::core::activate)) + .route("/search", post(routes::core::search_embedding)) + .route("/decay", post(routes::core::decay)) + .route("/consolidate", post(routes::core::consolidate)); + + let app = Router::new() + .merge(core_routes) + .merge(sync_routes) + .layer(CorsLayer::permissive()) + .with_state(state); + + let listener = tokio::net::TcpListener::bind(&bind_addr).await?; + info!("Engram server listening on {}", bind_addr); + axum::serve(listener, app).await?; + + Ok(()) +} diff --git a/engram/crates/engram-server/src/routes/core.rs b/engram/crates/engram-server/src/routes/core.rs new file mode 100644 index 0000000..d9ecaea --- /dev/null +++ b/engram/crates/engram-server/src/routes/core.rs @@ -0,0 +1,218 @@ +/// Core Engram API routes — nodes, edges, activation, search. +use axum::{ + extract::{Path, State}, + http::StatusCode, + Json, +}; +use engram_core::types::{Edge, MemoryTier, Node, NodeType, RelationType}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use uuid::Uuid; + +use crate::state::AppState; + +// ── Stats ───────────────────────────────────────────────────────────────────── + +#[derive(Serialize)] +pub struct StatsResponse { + pub nodes: usize, + pub edges: usize, +} + +pub async fn get_stats( + State(state): State>, +) -> Result, StatusCode> { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let nodes = db.node_count().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let edges = db.edge_count().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(StatsResponse { nodes, edges })) +} + +// ── Nodes ───────────────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct CreateNodeRequest { + pub node_type: NodeType, + pub embedding: Vec, + pub content: Vec, + pub tier: MemoryTier, + pub importance: f32, +} + +#[derive(Serialize)] +pub struct CreateNodeResponse { + pub id: Uuid, +} + +pub async fn create_node( + State(state): State>, + Json(req): Json, +) -> Result, StatusCode> { + let node = Node::new(req.node_type, req.embedding, req.content, req.tier, req.importance); + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let id = db.put_node(node).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(CreateNodeResponse { id })) +} + +pub async fn get_node( + State(state): State>, + Path(id): Path, +) -> Result, StatusCode> { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + match db.get_node(id).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? { + Some(node) => Ok(Json(node)), + None => Err(StatusCode::NOT_FOUND), + } +} + +// ── Edges ───────────────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct CreateEdgeRequest { + pub from_id: Uuid, + pub to_id: Uuid, + pub relation: RelationType, + pub weight: f32, +} + +pub async fn create_edge( + State(state): State>, + Json(req): Json, +) -> Result { + let edge = Edge::new(req.from_id, req.to_id, req.relation, req.weight); + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + db.put_edge(edge).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(StatusCode::CREATED) +} + +pub async fn get_edges_from( + State(state): State>, + Path(id): Path, +) -> Result>, StatusCode> { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let edges = db.get_edges_from(id).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(edges)) +} + +// ── Activation ──────────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct ActivateRequest { + pub seeds: Vec, + pub query_embedding: Vec, + #[serde(default = "default_depth")] + pub max_depth: u8, + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_depth() -> u8 { 3 } +fn default_limit() -> usize { 10 } + +#[derive(Serialize)] +pub struct ActivateResponse { + pub results: Vec, +} + +#[derive(Serialize)] +pub struct ActivatedNodeJson { + pub node: Node, + pub activation_strength: f32, + pub hops: u8, +} + +pub async fn activate( + State(state): State>, + Json(req): Json, +) -> Result, StatusCode> { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let results = db + .activate(&req.seeds, &req.query_embedding, req.max_depth, req.limit) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + Ok(Json(ActivateResponse { + results: results + .into_iter() + .map(|a| ActivatedNodeJson { + node: a.node, + activation_strength: a.activation_strength, + hops: a.hops, + }) + .collect(), + })) +} + +// ── Search ──────────────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct SearchRequest { + pub embedding: Vec, + #[serde(default = "default_limit")] + pub limit: usize, +} + +#[derive(Serialize)] +pub struct SearchResponse { + pub results: Vec, +} + +#[derive(Serialize)] +pub struct ScoredNodeJson { + pub node: Node, + pub score: f32, +} + +pub async fn search_embedding( + State(state): State>, + Json(req): Json, +) -> Result, StatusCode> { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let results = db + .search_embedding(&req.embedding, req.limit) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + Ok(Json(SearchResponse { + results: results + .into_iter() + .map(|s| ScoredNodeJson { node: s.node, score: s.score }) + .collect(), + })) +} + +// ── Decay / Consolidate ─────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct DecayRequest { + pub factor: f32, +} + +#[derive(Serialize)] +pub struct DecayResponse { + pub nodes_updated: usize, +} + +pub async fn decay( + State(state): State>, + Json(req): Json, +) -> Result, StatusCode> { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let nodes_updated = db.decay(req.factor).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(DecayResponse { nodes_updated })) +} + +#[derive(Serialize)] +pub struct ConsolidateResponse { + pub promoted: usize, +} + +pub async fn consolidate( + State(state): State>, +) -> Result, StatusCode> { + use engram_core::ConsolidationConfig; + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let report = db + .consolidate(&ConsolidationConfig::default()) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(Json(ConsolidateResponse { promoted: report.promoted })) +} + diff --git a/engram/crates/engram-server/src/routes/mod.rs b/engram/crates/engram-server/src/routes/mod.rs new file mode 100644 index 0000000..d61ad46 --- /dev/null +++ b/engram/crates/engram-server/src/routes/mod.rs @@ -0,0 +1,3 @@ +pub mod core; +pub mod sync; +pub mod swarm; diff --git a/engram/crates/engram-server/src/routes/swarm.rs b/engram/crates/engram-server/src/routes/swarm.rs new file mode 100644 index 0000000..62761f2 --- /dev/null +++ b/engram/crates/engram-server/src/routes/swarm.rs @@ -0,0 +1,124 @@ +/// Swarm routes — distributed activation across the peer network. +/// +/// POST /swarm/activate — SwarmActivateRequest → SwarmActivateResponse +/// GET /swarm/status — peer health check and last sync times +use axum::{ + extract::State, + http::StatusCode, + Json, +}; +use engram_sync::{ + client::SyncClient, merge_activation_results, Peer, PeerActivationResult, PeerStatus, + SerializableActivatedNode, SwarmActivateRequest, SwarmActivateResponse, +}; +use std::sync::Arc; +use uuid::Uuid; + +use crate::state::AppState; + +/// POST /swarm/activate +/// +/// Runs spreading activation locally, then (if include_peers=true) fans out +/// to all trusted peers in parallel and returns merged results. +pub async fn swarm_activate( + State(state): State>, + Json(req): Json, +) -> Result, StatusCode> { + // Step 1: Run local activation. Lock db, compute, drop immediately. + let local_results: Vec = { + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let activated = db + .activate(&req.seeds, &req.query_embedding, req.max_depth, req.limit) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + activated.into_iter().map(Into::into).collect() + // db lock released here + }; + + // Step 2: Snapshot peer list (lock, read, drop). + let (our_id, trusted_peers): (Uuid, Vec) = { + let engine = state.sync_engine.lock().await; + let id = engine.our_id(); + let peers = engine.list_peers().iter().filter(|p| p.trusted).cloned().collect(); + (id, peers) + // engine lock released here + }; + + // Step 3: Fan out to peers — no locks held across these awaits. + let mut peer_results: Vec = Vec::new(); + + if req.include_peers { + let mut handles = Vec::new(); + for peer in trusted_peers { + let seeds = req.seeds.clone(); + let embedding = req.query_embedding.clone(); + let max_depth = req.max_depth; + let limit = req.limit; + + handles.push(tokio::spawn(async move { + let client = SyncClient::new(peer.clone(), our_id); + match client.remote_activate(&seeds, &embedding, max_depth, limit).await { + Ok(results) => PeerActivationResult { + peer_id: peer.id, + peer_name: peer.name, + results, + error: None, + }, + Err(e) => PeerActivationResult { + peer_id: peer.id, + peer_name: peer.name, + results: Vec::new(), + error: Some(e.to_string()), + }, + } + })); + } + + for handle in handles { + if let Ok(result) = handle.await { + peer_results.push(result); + } + } + } + + // Step 4: Merge and return. + let merged = merge_activation_results(&local_results, &peer_results, req.limit); + + Ok(Json(SwarmActivateResponse { + local_results, + peer_results, + merged, + })) +} + +/// GET /swarm/status +/// +/// Returns peer list with reachability status and last sync times. +pub async fn swarm_status( + State(state): State>, +) -> Result>, StatusCode> { + // Snapshot peer list and our_id without holding the lock across awaits. + let (our_id, peers): (Uuid, Vec) = { + let engine = state.sync_engine.lock().await; + let id = engine.our_id(); + let peers = engine.list_peers().to_vec(); + (id, peers) + }; + + let mut statuses: Vec = Vec::new(); + for peer in peers { + use engram_core::types::now_ms; + let client = SyncClient::new(peer.clone(), our_id); + let reachable = client.pull_delta(now_ms()).await.is_ok(); + statuses.push(PeerStatus { + peer_id: peer.id, + peer_name: peer.name, + address: peer.address, + last_sync_at: peer.last_sync_at, + reachable, + sync_tiers: peer.sync_tiers, + trusted: peer.trusted, + }); + } + + Ok(Json(statuses)) +} diff --git a/engram/crates/engram-server/src/routes/sync.rs b/engram/crates/engram-server/src/routes/sync.rs new file mode 100644 index 0000000..febf541 --- /dev/null +++ b/engram/crates/engram-server/src/routes/sync.rs @@ -0,0 +1,133 @@ +/// Sync routes — peer delta exchange and peer registry. +/// +/// All routes under /sync require Authorization: Bearer {api_key}. +/// +/// GET /sync/delta?since={ms}&peer_id={uuid} — generate delta for caller +/// POST /sync/push — receive delta from peer +/// POST /sync/peers — register a new peer +/// GET /sync/peers — list peers +/// DELETE /sync/peers/{id} — remove peer +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + Json, +}; +use engram_core::types::MemoryTier; +use engram_sync::{Peer, SyncDelta}; +use serde::Deserialize; +use std::sync::Arc; +use uuid::Uuid; + +use crate::state::AppState; + +// ── GET /sync/delta ─────────────────────────────────────────────────────────── + +#[derive(Deserialize)] +pub struct DeltaParams { + pub since: Option, + pub peer_id: Option, +} + +pub async fn get_delta( + State(state): State>, + Query(params): Query, +) -> Result, StatusCode> { + let since = params.since.unwrap_or(0); + + // Determine which tiers to expose based on caller's peer_id + let tiers: Vec = { + let engine = state.sync_engine.lock().await; + if let Some(peer_id) = params.peer_id { + if let Some(peer) = engine.get_peer(peer_id) { + if peer.trusted { + peer.sync_tiers.clone() + } else { + vec![MemoryTier::Semantic] + } + } else { + vec![MemoryTier::Semantic] + } + } else { + vec![MemoryTier::Semantic] + } + // engine lock released here + }; + + let engine = state.sync_engine.lock().await; + let delta = engine + .generate_delta(since, &tiers) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + Ok(Json(delta)) +} + +// ── POST /sync/push ─────────────────────────────────────────────────────────── + +pub async fn push_delta( + State(state): State>, + Json(delta): Json, +) -> Result { + // Accepted tiers for incoming pushes (Semantic and Procedural by default) + let accepted_tiers = vec![MemoryTier::Semantic, MemoryTier::Procedural]; + + // Apply the delta using the DB handle directly (no async needed) + let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + // Tombstones + for id in &delta.tombstones { + let _ = db.delete_node(*id); + } + + // Nodes + for node in delta.nodes { + if !accepted_tiers.contains(&node.tier) { + continue; + } + if db.get_node(node.id).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?.is_some() { + continue; + } + db.put_node(node).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + } + + // Edges + for edge in delta.edges { + let from_ok = db.get_node(edge.from_id) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + .is_some(); + let to_ok = db.get_node(edge.to_id) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + .is_some(); + if from_ok && to_ok { + let _ = db.put_edge(edge); + } + } + + Ok(StatusCode::OK) +} + +// ── Peer registry ───────────────────────────────────────────────────────────── + +pub async fn list_peers( + State(state): State>, +) -> Result>, StatusCode> { + let engine = state.sync_engine.lock().await; + Ok(Json(engine.list_peers().to_vec())) +} + +pub async fn register_peer( + State(state): State>, + Json(peer): Json, +) -> Result { + let mut engine = state.sync_engine.lock().await; + engine.add_peer(peer); + Ok(StatusCode::CREATED) +} + +pub async fn delete_peer( + State(state): State>, + Path(id): Path, +) -> Result { + let mut engine = state.sync_engine.lock().await; + engine.remove_peer(id); + Ok(StatusCode::NO_CONTENT) +} diff --git a/engram/crates/engram-server/src/state.rs b/engram/crates/engram-server/src/state.rs new file mode 100644 index 0000000..b32e9fa --- /dev/null +++ b/engram/crates/engram-server/src/state.rs @@ -0,0 +1,13 @@ +/// Shared application state for all request handlers. +use engram_core::EngramDb; +use engram_sync::SyncEngine; +use std::sync::{Arc, Mutex}; + +pub struct AppState { + /// Local database — uses std::sync::Mutex (sync ops only, fast) + pub db: Arc>, + /// Sync engine — uses tokio::sync::Mutex so it can be held across .await + pub sync_engine: Arc>, + /// API key used to authenticate incoming sync requests + pub api_key: String, +} diff --git a/engram/crates/engram-sync/Cargo.toml b/engram/crates/engram-sync/Cargo.toml new file mode 100644 index 0000000..8952193 --- /dev/null +++ b/engram/crates/engram-sync/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "engram-sync" +version = "0.1.0" +edition = "2021" +description = "Swarm memory sync layer for Engram — peer delta sync and distributed activation" +license = "MIT" + +[lib] +name = "engram_sync" +path = "src/lib.rs" + +[dependencies] +engram-core = { path = "../engram-core" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +uuid = { version = "1", features = ["v4", "serde"] } +reqwest = { version = "0.12", features = ["json"] } +tokio = { version = "1", features = ["full"] } +anyhow = "1" +thiserror = "1" + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1", features = ["full", "test-util"] } diff --git a/engram/crates/engram-sync/src/client.rs b/engram/crates/engram-sync/src/client.rs new file mode 100644 index 0000000..4a1bca1 --- /dev/null +++ b/engram/crates/engram-sync/src/client.rs @@ -0,0 +1,122 @@ +/// HTTP client for talking to a remote Engram peer. +/// +/// All peer-to-peer communication goes through this client. +/// Authentication is via `Authorization: Bearer {api_key}` on every request. +use crate::types::{Peer, SerializableActivatedNode, SyncDelta}; +use anyhow::Context; +use uuid::Uuid; + +pub struct SyncClient { + peer: Peer, + http: reqwest::Client, + our_id: Uuid, +} + +impl SyncClient { + pub fn new(peer: Peer, our_id: Uuid) -> Self { + let http = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("failed to build reqwest client"); + Self { peer, http, our_id } + } + + /// Pull a delta from the remote peer containing everything since `since` (Unix ms). + /// + /// GET {address}/sync/delta?since={since}&peer_id={our_id} + pub async fn pull_delta(&self, since: i64) -> anyhow::Result { + let url = format!( + "{}/sync/delta?since={}&peer_id={}", + self.peer.address, since, self.our_id + ); + let resp = self + .http + .get(&url) + .header("Authorization", format!("Bearer {}", self.peer.api_key)) + .send() + .await + .context("pull_delta: request failed")?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("pull_delta: peer returned {}: {}", status, body); + } + + resp.json::() + .await + .context("pull_delta: failed to decode response") + } + + /// Push our delta to the remote peer. + /// + /// POST {address}/sync/push + pub async fn push_delta(&self, delta: &SyncDelta) -> anyhow::Result<()> { + let url = format!("{}/sync/push", self.peer.address); + let resp = self + .http + .post(&url) + .header("Authorization", format!("Bearer {}", self.peer.api_key)) + .json(delta) + .send() + .await + .context("push_delta: request failed")?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("push_delta: peer returned {}: {}", status, body); + } + + Ok(()) + } + + /// Fan spreading activation out to a remote peer. + /// + /// POST {address}/swarm/activate — the remote peer runs activation locally + /// (include_peers=false so it does not fan out further, preventing cycles). + pub async fn remote_activate( + &self, + seeds: &[Uuid], + query_embedding: &[f32], + max_depth: u8, + limit: usize, + ) -> anyhow::Result> { + use crate::types::SwarmActivateRequest; + + let url = format!("{}/swarm/activate", self.peer.address); + let req = SwarmActivateRequest { + seeds: seeds.to_vec(), + query_embedding: query_embedding.to_vec(), + max_depth, + limit, + include_peers: false, // no further fan-out — prevents cycles + }; + + let resp = self + .http + .post(&url) + .header("Authorization", format!("Bearer {}", self.peer.api_key)) + .json(&req) + .send() + .await + .context("remote_activate: request failed")?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("remote_activate: peer returned {}: {}", status, body); + } + + // The remote returns a SwarmActivateResponse; we only want local_results + let response: crate::types::SwarmActivateResponse = resp + .json() + .await + .context("remote_activate: failed to decode response")?; + Ok(response.local_results) + } + + pub fn peer(&self) -> &Peer { + &self.peer + } +} diff --git a/engram/crates/engram-sync/src/engine.rs b/engram/crates/engram-sync/src/engine.rs new file mode 100644 index 0000000..abeaacb --- /dev/null +++ b/engram/crates/engram-sync/src/engine.rs @@ -0,0 +1,423 @@ +/// SyncEngine — orchestrates peer sync and swarm activation. +/// +/// The engine holds a list of known peers, a handle to the local database, +/// and owns our identity (UUID + API key). It drives two workflows: +/// +/// 1. **Delta sync**: periodic pull-then-push with each peer, filtering by +/// tier allowlist and skipping nodes we already have. +/// +/// 2. **Swarm activation**: run spreading activation locally, then fan out +/// to all trusted peers in parallel, and merge results by strength. +use crate::client::SyncClient; +use crate::types::{ + MergedActivatedNode, Peer, PeerActivationResult, PeerStatus, PeerSyncResult, + SerializableActivatedNode, SyncConfig, SyncDelta, SyncReport, SwarmActivateRequest, + SwarmActivateResponse, +}; +use engram_core::types::{now_ms, MemoryTier}; +use engram_core::EngramDb; +use std::sync::{Arc, Mutex}; +use uuid::Uuid; + +pub struct SyncEngine { + db: Arc>, + peers: Vec, + our_id: Uuid, + our_name: String, + api_key: String, + default_sync_tiers: Vec, +} + +impl SyncEngine { + pub fn new(db: Arc>, config: SyncConfig) -> Self { + Self { + db, + peers: Vec::new(), + our_id: config.our_id, + our_name: config.our_name, + api_key: config.api_key, + default_sync_tiers: config.default_sync_tiers, + } + } + + // ── Peer registry ───────────────────────────────────────────────────────── + + pub fn add_peer(&mut self, peer: Peer) { + // Replace if already registered + self.peers.retain(|p| p.id != peer.id); + self.peers.push(peer); + } + + pub fn remove_peer(&mut self, peer_id: Uuid) { + self.peers.retain(|p| p.id != peer_id); + } + + pub fn list_peers(&self) -> &[Peer] { + &self.peers + } + + pub fn get_peer(&self, peer_id: Uuid) -> Option<&Peer> { + self.peers.iter().find(|p| p.id == peer_id) + } + + pub fn our_id(&self) -> Uuid { + self.our_id + } + + pub fn our_name(&self) -> &str { + &self.our_name + } + + pub fn api_key(&self) -> &str { + &self.api_key + } + + // ── Full sync cycle ─────────────────────────────────────────────────────── + + /// Sync with every registered peer. Returns a report summarising what moved. + pub async fn sync_all(&mut self) -> SyncReport { + let mut report = SyncReport { + peers_synced: 0, + nodes_received: 0, + nodes_sent: 0, + errors: Vec::new(), + }; + + // Clone the peer list so we can mutate self afterwards + let peers: Vec = self.peers.clone(); + for peer in peers { + match self.sync_peer(&peer).await { + Ok(result) => { + report.peers_synced += 1; + report.nodes_received += result.nodes_received; + report.nodes_sent += result.nodes_sent; + // Update last_sync_at + if let Some(p) = self.peers.iter_mut().find(|p| p.id == peer.id) { + p.last_sync_at = now_ms(); + } + } + Err(e) => { + report.errors.push(format!("peer {}: {}", peer.name, e)); + } + } + } + + report + } + + /// Sync a single peer: pull delta, apply it, then push our delta. + pub async fn sync_peer(&self, peer: &Peer) -> anyhow::Result { + let client = SyncClient::new(peer.clone(), self.our_id); + let tiers = effective_tiers(peer, &self.default_sync_tiers); + + // Pull: get everything from the peer since their last known sync time + let remote_delta = client.pull_delta(peer.last_sync_at).await?; + let nodes_received = self.apply_delta(remote_delta, &tiers).await?; + + // Push: send everything we have that the peer hasn't seen + let our_delta = self.generate_delta(peer.last_sync_at, &tiers)?; + let nodes_sent = our_delta.nodes.len(); + client.push_delta(&our_delta).await?; + + Ok(PeerSyncResult { + peer_id: peer.id, + nodes_received, + nodes_sent, + }) + } + + // ── Delta generation ────────────────────────────────────────────────────── + + /// Build a delta containing all nodes/edges modified since `since` + /// that belong to one of the allowed `tiers`. + pub fn generate_delta( + &self, + since: i64, + tiers: &[MemoryTier], + ) -> anyhow::Result { + let db = self + .db + .lock() + .map_err(|_| anyhow::anyhow!("db lock poisoned"))?; + + // Scan all nodes; filter by tier and modification time + let all_nodes = db.scan_nodes()?; + let nodes: Vec<_> = all_nodes + .into_iter() + .filter(|n| tiers.contains(&n.tier) && n.last_activated >= since) + .collect(); + + // Edges: include all edges between nodes in our set + let node_ids: std::collections::HashSet = nodes.iter().map(|n| n.id).collect(); + let all_edges = db.scan_edges()?; + let edges: Vec<_> = all_edges + .into_iter() + .filter(|e| node_ids.contains(&e.from_id) && node_ids.contains(&e.to_id)) + .collect(); + + Ok(SyncDelta { + peer_id: self.our_id, + since, + nodes, + edges, + tombstones: Vec::new(), // tombstone tracking requires a separate log; not yet implemented + generated_at: now_ms(), + }) + } + + // ── Delta application ───────────────────────────────────────────────────── + + /// Merge an incoming delta into the local database. + /// + /// - Nodes/edges we already have (same UUID) are skipped — local wins. + /// - Tombstones cause deletion. + /// - Only nodes in the allowed tiers are accepted. + /// + /// Returns the number of nodes actually written. + pub async fn apply_delta( + &self, + delta: SyncDelta, + allowed_tiers: &[MemoryTier], + ) -> anyhow::Result { + let db = self + .db + .lock() + .map_err(|_| anyhow::anyhow!("db lock poisoned"))?; + + let mut written = 0usize; + + // Apply tombstones first + for id in &delta.tombstones { + let _ = db.delete_node(*id); + } + + // Merge nodes + for node in delta.nodes { + if !allowed_tiers.contains(&node.tier) { + continue; + } + // Skip if we already have this UUID + if db.get_node(node.id)?.is_some() { + continue; + } + db.put_node(node)?; + written += 1; + } + + // Merge edges (if both endpoints exist) + for edge in delta.edges { + let from_exists = db.get_node(edge.from_id)?.is_some(); + let to_exists = db.get_node(edge.to_id)?.is_some(); + if from_exists && to_exists { + db.put_edge(edge)?; + } + } + + Ok(written) + } + + // ── Swarm activation ────────────────────────────────────────────────────── + + /// Run spreading activation locally, then fan out to all trusted peers, + /// and merge all results into a unified ranked list. + pub async fn swarm_activate( + &self, + req: SwarmActivateRequest, + ) -> anyhow::Result { + // Local activation + let local_results: Vec = { + let db = self + .db + .lock() + .map_err(|_| anyhow::anyhow!("db lock poisoned"))?; + let activated = db.activate( + &req.seeds, + &req.query_embedding, + req.max_depth, + req.limit, + )?; + activated.into_iter().map(Into::into).collect() + }; + + let mut peer_results: Vec = Vec::new(); + + if req.include_peers { + // Fan out to all trusted peers in parallel + let trusted_peers: Vec = self + .peers + .iter() + .filter(|p| p.trusted) + .cloned() + .collect(); + + let mut handles = Vec::new(); + for peer in trusted_peers { + let seeds = req.seeds.clone(); + let embedding = req.query_embedding.clone(); + let max_depth = req.max_depth; + let limit = req.limit; + let our_id = self.our_id; + + handles.push(tokio::spawn(async move { + let client = SyncClient::new(peer.clone(), our_id); + match client + .remote_activate(&seeds, &embedding, max_depth, limit) + .await + { + Ok(results) => PeerActivationResult { + peer_id: peer.id, + peer_name: peer.name, + results, + error: None, + }, + Err(e) => PeerActivationResult { + peer_id: peer.id, + peer_name: peer.name, + results: Vec::new(), + error: Some(e.to_string()), + }, + } + })); + } + + for handle in handles { + match handle.await { + Ok(result) => peer_results.push(result), + Err(e) => peer_results.push(PeerActivationResult { + peer_id: Uuid::nil(), + peer_name: "unknown".into(), + results: Vec::new(), + error: Some(format!("task error: {}", e)), + }), + } + } + } + + let merged = merge_activation_results(&local_results, &peer_results, req.limit); + + Ok(SwarmActivateResponse { + local_results, + peer_results, + merged, + }) + } + + // ── Peer health check ───────────────────────────────────────────────────── + + /// Check which peers are reachable and return their status. + pub async fn peer_statuses(&self) -> Vec { + let mut statuses = Vec::new(); + for peer in &self.peers { + let client = SyncClient::new(peer.clone(), self.our_id); + // We attempt a health check by pulling an empty delta (since=now) + let reachable = client.pull_delta(now_ms()).await.is_ok(); + statuses.push(PeerStatus { + peer_id: peer.id, + peer_name: peer.name.clone(), + address: peer.address.clone(), + last_sync_at: peer.last_sync_at, + reachable, + sync_tiers: peer.sync_tiers.clone(), + trusted: peer.trusted, + }); + } + statuses + } +} + +// ── Merge logic ─────────────────────────────────────────────────────────────── + +/// Merge local and peer activation results into a unified ranked list. +/// +/// Deduplication: two nodes are considered duplicates if they have the same UUID. +/// When duplicates occur, the one with the highest activation strength is kept. +/// The merged list is sorted by activation_strength descending and truncated to `limit`. +pub fn merge_activation_results( + local: &[SerializableActivatedNode], + peer_results: &[PeerActivationResult], + limit: usize, +) -> Vec { + use std::collections::HashMap; + + // (uuid -> MergedActivatedNode) — we keep the strongest instance of each node + let mut map: HashMap = HashMap::new(); + + // Process local results first (source_peer = None) + for item in local { + let id = item.node.id; + let candidate = MergedActivatedNode { + content: String::from_utf8_lossy(&item.node.content).to_string(), + node_type: item.node.node_type.clone(), + tier: item.node.tier.clone(), + activation_strength: item.activation_strength, + source_peer: None, + hops: item.hops, + node: item.node.clone(), + }; + map.entry(id) + .and_modify(|existing| { + if item.activation_strength > existing.activation_strength { + *existing = candidate.clone(); + } + }) + .or_insert(candidate); + } + + // Process peer results + for peer_result in peer_results { + if peer_result.error.is_some() { + continue; + } + for item in &peer_result.results { + let id = item.node.id; + let candidate = MergedActivatedNode { + content: String::from_utf8_lossy(&item.node.content).to_string(), + node_type: item.node.node_type.clone(), + tier: item.node.tier.clone(), + activation_strength: item.activation_strength, + source_peer: Some(peer_result.peer_id), + hops: item.hops, + node: item.node.clone(), + }; + map.entry(id) + .and_modify(|existing| { + if item.activation_strength > existing.activation_strength { + *existing = candidate.clone(); + } + }) + .or_insert(candidate); + } + } + + // Sort by strength descending, take top N + let mut merged: Vec = map.into_values().collect(); + merged.sort_by(|a, b| { + b.activation_strength + .partial_cmp(&a.activation_strength) + .unwrap_or(std::cmp::Ordering::Equal) + }); + merged.truncate(limit); + merged +} + +// ── Tier helpers ────────────────────────────────────────────────────────────── + +/// Compute the effective sync tiers for a peer — intersection of the peer's +/// own allowlist and our defaults. Untrusted peers are further restricted +/// to Semantic only. +fn effective_tiers<'a>(peer: &Peer, defaults: &'a [MemoryTier]) -> Vec { + if !peer.trusted { + // Untrusted: Semantic only, regardless of configuration + return vec![MemoryTier::Semantic]; + } + if peer.sync_tiers.is_empty() { + defaults.to_vec() + } else { + // Intersection: only tiers that both us and the peer agree on + defaults + .iter() + .filter(|t| peer.sync_tiers.contains(t)) + .cloned() + .collect() + } +} diff --git a/engram/crates/engram-sync/src/lib.rs b/engram/crates/engram-sync/src/lib.rs new file mode 100644 index 0000000..b68be7d --- /dev/null +++ b/engram/crates/engram-sync/src/lib.rs @@ -0,0 +1,265 @@ +/// Engram Sync — swarm memory protocol for distributed Engram instances. +/// +/// This crate turns Engram from a local-first database into a distributed +/// swarm memory protocol. Multiple independent Engram instances (peers) can +/// share memory across each other using delta sync and can fan spreading +/// activation out across the swarm, merging results by strength. +/// +/// # Architecture +/// +/// ```text +/// [Neuron-A Engram] <-> sync <-> [Neuron-B Engram] +/// | +/// [Neuron-C Engram] +/// +/// Swarm activation: seed on A → propagate locally → fan-out to B and C +/// → merge all results → unified ranked response +/// ``` +/// +/// # Protocol +/// +/// - Each peer is local and authoritative. There is no central server. +/// - Peers sync via delta exchange: "give me everything since timestamp T". +/// - Only configured memory tiers flow between peers (Semantic by default; +/// Episodic and Working are private unless explicitly enabled). +/// - Trusted peers get the full configured tier set; untrusted peers get +/// Semantic only. +/// - Swarm activation fans out to all trusted peers in parallel, deduplicates +/// by UUID (keeping strongest activation), and re-ranks. + +pub mod client; +pub mod engine; +pub mod types; + +// Public surface +pub use engine::{merge_activation_results, SyncEngine}; +pub use types::{ + MergedActivatedNode, Peer, PeerActivationResult, PeerStatus, PeerSyncResult, + SerializableActivatedNode, SyncConfig, SyncDelta, SyncReport, SwarmActivateRequest, + SwarmActivateResponse, +}; + +#[cfg(test)] +mod tests { + use super::*; + use engram_core::types::{MemoryTier, Node, NodeType}; + use uuid::Uuid; + + fn make_node(tier: MemoryTier) -> Node { + Node::new( + NodeType::Concept, + vec![0.1, 0.2, 0.3], + b"test content".to_vec(), + tier, + 0.5, + ) + } + + fn make_activated(node: Node, strength: f32, hops: u8) -> SerializableActivatedNode { + SerializableActivatedNode { + node, + activation_strength: strength, + hops, + } + } + + // ── merge_activation_results tests ─────────────────────────────────────── + + #[test] + fn merge_empty() { + let merged = merge_activation_results(&[], &[], 10); + assert!(merged.is_empty()); + } + + #[test] + fn merge_local_only() { + let node = make_node(MemoryTier::Semantic); + let node_id = node.id; + let local = vec![make_activated(node, 0.8, 1)]; + let merged = merge_activation_results(&local, &[], 10); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].node.id, node_id); + assert!(merged[0].source_peer.is_none(), "local nodes have no source_peer"); + assert!((merged[0].activation_strength - 0.8).abs() < f32::EPSILON); + } + + #[test] + fn merge_peer_result_included() { + let local: Vec = vec![]; + let node = make_node(MemoryTier::Semantic); + let peer_id = Uuid::new_v4(); + let peer_results = vec![PeerActivationResult { + peer_id, + peer_name: "peer-a".into(), + results: vec![make_activated(node, 0.6, 2)], + error: None, + }]; + let merged = merge_activation_results(&local, &peer_results, 10); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].source_peer, Some(peer_id)); + } + + #[test] + fn merge_deduplicates_by_uuid_keeps_strongest() { + let node = make_node(MemoryTier::Semantic); + let id = node.id; + + // Same node appears locally at 0.4 and from a peer at 0.9 + let local = vec![make_activated(node.clone(), 0.4, 1)]; + let peer_id = Uuid::new_v4(); + let peer_results = vec![PeerActivationResult { + peer_id, + peer_name: "peer-a".into(), + results: vec![make_activated(node, 0.9, 1)], + error: None, + }]; + + let merged = merge_activation_results(&local, &peer_results, 10); + // Should be deduplicated to 1 result + assert_eq!(merged.len(), 1); + // The stronger version (0.9, from peer) should win + assert!((merged[0].activation_strength - 0.9).abs() < f32::EPSILON); + assert_eq!(merged[0].source_peer, Some(peer_id)); + } + + #[test] + fn merge_respects_limit() { + let local: Vec = (0..20) + .map(|i| make_activated(make_node(MemoryTier::Semantic), i as f32 / 20.0, 1)) + .collect(); + let merged = merge_activation_results(&local, &[], 5); + assert_eq!(merged.len(), 5, "limit must be respected"); + } + + #[test] + fn merge_sorted_by_strength_descending() { + let strengths = vec![0.3f32, 0.9, 0.1, 0.7, 0.5]; + let local: Vec = strengths + .iter() + .map(|&s| make_activated(make_node(MemoryTier::Semantic), s, 1)) + .collect(); + let merged = merge_activation_results(&local, &[], 10); + let result_strengths: Vec = merged.iter().map(|m| m.activation_strength).collect(); + // Should be in descending order + for window in result_strengths.windows(2) { + assert!(window[0] >= window[1], "results must be sorted descending"); + } + } + + #[test] + fn merge_skips_errored_peers() { + let peer_results = vec![PeerActivationResult { + peer_id: Uuid::new_v4(), + peer_name: "failed-peer".into(), + results: vec![make_activated(make_node(MemoryTier::Semantic), 0.9, 1)], + error: Some("connection refused".into()), + }]; + let merged = merge_activation_results(&[], &peer_results, 10); + // Errored peers should be excluded + assert!(merged.is_empty()); + } + + // ── SyncDelta serialization ─────────────────────────────────────────────── + + #[test] + fn sync_delta_roundtrips_json() { + let delta = SyncDelta { + peer_id: Uuid::new_v4(), + since: 1000, + nodes: vec![make_node(MemoryTier::Semantic)], + edges: vec![], + tombstones: vec![Uuid::new_v4()], + generated_at: 2000, + }; + let json = serde_json::to_string(&delta).expect("serialize delta"); + let decoded: SyncDelta = serde_json::from_str(&json).expect("deserialize delta"); + assert_eq!(delta.peer_id, decoded.peer_id); + assert_eq!(delta.since, decoded.since); + assert_eq!(delta.nodes.len(), decoded.nodes.len()); + assert_eq!(delta.tombstones.len(), decoded.tombstones.len()); + } + + // ── SyncEngine peer management ──────────────────────────────────────────── + + #[test] + fn engine_add_remove_peer() { + use engram_core::EngramDb; + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new(Mutex::new(EngramDb::open(dir.path()).unwrap())); + let config = SyncConfig::default(); + let mut engine = SyncEngine::new(db, config); + + assert_eq!(engine.list_peers().len(), 0); + + let peer = Peer { + id: Uuid::new_v4(), + name: "test-peer".into(), + address: "http://localhost:9999".into(), + api_key: "secret".into(), + sync_tiers: vec![MemoryTier::Semantic], + last_sync_at: 0, + trusted: true, + }; + let peer_id = peer.id; + engine.add_peer(peer); + assert_eq!(engine.list_peers().len(), 1); + + engine.remove_peer(peer_id); + assert_eq!(engine.list_peers().len(), 0); + } + + #[test] + fn engine_add_peer_replaces_existing() { + use engram_core::EngramDb; + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new(Mutex::new(EngramDb::open(dir.path()).unwrap())); + let mut engine = SyncEngine::new(db, SyncConfig::default()); + + let id = Uuid::new_v4(); + for i in 0..3 { + engine.add_peer(Peer { + id, + name: format!("peer-v{}", i), + address: "http://localhost:1234".into(), + api_key: "k".into(), + sync_tiers: vec![], + last_sync_at: 0, + trusted: false, + }); + } + // Should still be just one peer (latest version) + assert_eq!(engine.list_peers().len(), 1); + assert_eq!(engine.list_peers()[0].name, "peer-v2"); + } + + // ── generate_delta ──────────────────────────────────────────────────────── + + #[test] + fn generate_delta_filters_by_tier() { + use engram_core::types::MemoryTier; + use engram_core::EngramDb; + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new(Mutex::new(EngramDb::open(dir.path()).unwrap())); + + // Insert nodes in different tiers + { + let db_locked = db.lock().unwrap(); + db_locked.put_node(make_node(MemoryTier::Semantic)).unwrap(); + db_locked.put_node(make_node(MemoryTier::Episodic)).unwrap(); + db_locked.put_node(make_node(MemoryTier::Working)).unwrap(); + } + + let engine = SyncEngine::new(db, SyncConfig::default()); + + // Only request Semantic tier + let delta = engine.generate_delta(0, &[MemoryTier::Semantic]).unwrap(); + assert_eq!(delta.nodes.len(), 1, "only Semantic node should be in delta"); + assert!(delta.nodes[0].tier == MemoryTier::Semantic); + } +} diff --git a/engram/crates/engram-sync/src/types.rs b/engram/crates/engram-sync/src/types.rs new file mode 100644 index 0000000..b0bbb4e --- /dev/null +++ b/engram/crates/engram-sync/src/types.rs @@ -0,0 +1,166 @@ +use engram_core::types::{ActivatedNode, Edge, MemoryTier, Node, NodeType}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// A remote peer that this Engram instance syncs with. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Peer { + /// Stable unique identity for this peer + pub id: Uuid, + /// Human-readable name (e.g. "neuron-will", "neuron-sarah") + pub name: String, + /// Base URL of the peer's Engram server (e.g. "https://engram.neurontechnologies.ai") + pub address: String, + /// Shared secret used in the Authorization: Bearer header + pub api_key: String, + /// Which memory tiers are allowed to flow to/from this peer. + /// Semantic by default — Episodic is private unless explicitly opted in. + pub sync_tiers: Vec, + /// Unix milliseconds of the last successful sync. 0 if never synced. + pub last_sync_at: i64, + /// Trusted peers get all configured tiers; untrusted peers get Semantic only. + pub trusted: bool, +} + +/// An incremental change set — everything that changed since a given timestamp. +/// Exchanged between peers during sync. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SyncDelta { + /// UUID of the peer that generated this delta + pub peer_id: Uuid, + /// All nodes modified or added after this Unix ms timestamp + pub since: i64, + /// Nodes added or modified since `since` + pub nodes: Vec, + /// Edges added or modified since `since` + pub edges: Vec, + /// Node IDs that were deleted (tombstones) — receivers should remove them + pub tombstones: Vec, + /// When this delta was generated + pub generated_at: i64, +} + +/// Request to fan spreading activation out across the swarm. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SwarmActivateRequest { + /// Seed node IDs to start activation from + pub seeds: Vec, + /// Query embedding for semantic scoring + pub query_embedding: Vec, + /// Maximum graph hops per peer + pub max_depth: u8, + /// Maximum results to return per peer (before merge) + pub limit: usize, + /// If true, fan out to all trusted peers and merge results + pub include_peers: bool, +} + +/// Response from a swarm activation — local results, per-peer results, and +/// the unified merged ranking across all sources. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SwarmActivateResponse { + pub local_results: Vec, + pub peer_results: Vec, + /// Deduplicated, re-ranked unified results from all sources + pub merged: Vec, +} + +/// ActivatedNode serializable form (ActivatedNode in engram-core is not Serialize). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SerializableActivatedNode { + pub node: Node, + pub activation_strength: f32, + pub hops: u8, +} + +impl From for SerializableActivatedNode { + fn from(a: ActivatedNode) -> Self { + Self { + node: a.node, + activation_strength: a.activation_strength, + hops: a.hops, + } + } +} + +/// Activation results from a single remote peer. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PeerActivationResult { + pub peer_id: Uuid, + pub peer_name: String, + /// Successfully retrieved results (empty on error) + pub results: Vec, + /// Set if the peer request failed + pub error: Option, +} + +/// A node in the merged swarm result set — annotated with its origin peer. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MergedActivatedNode { + pub content: String, + pub node_type: NodeType, + pub tier: MemoryTier, + /// Composite activation strength after merging + pub activation_strength: f32, + /// None = local, Some(uuid) = from that peer + pub source_peer: Option, + pub hops: u8, + /// Full node for callers who need it + pub node: Node, +} + +/// Aggregate report from a full sync cycle. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SyncReport { + pub peers_synced: usize, + pub nodes_received: usize, + pub nodes_sent: usize, + pub errors: Vec, +} + +/// Result from syncing a single peer. +#[derive(Debug, Clone)] +pub struct PeerSyncResult { + pub peer_id: Uuid, + pub nodes_received: usize, + pub nodes_sent: usize, +} + +/// Configuration for the sync engine. +#[derive(Debug, Clone)] +pub struct SyncConfig { + /// This instance's stable UUID + pub our_id: Uuid, + /// Human name for this instance + pub our_name: String, + /// API key peers use to authenticate with us + pub api_key: String, + /// Which tiers to sync by default (peers may also restrict this) + pub default_sync_tiers: Vec, + /// How often to run background sync, in seconds. Default: 300 (5 min) + pub sync_interval_secs: u64, +} + +impl Default for SyncConfig { + fn default() -> Self { + Self { + our_id: Uuid::new_v4(), + our_name: "engram-local".to_string(), + api_key: Uuid::new_v4().to_string(), + default_sync_tiers: vec![MemoryTier::Semantic], + sync_interval_secs: 300, + } + } +} + +/// Swarm peer health info for the /swarm/status endpoint. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PeerStatus { + pub peer_id: Uuid, + pub peer_name: String, + pub address: String, + pub last_sync_at: i64, + pub reachable: bool, + pub sync_tiers: Vec, + pub trusted: bool, +} diff --git a/engram/studio/index.html b/engram/studio/index.html new file mode 100644 index 0000000..6854609 --- /dev/null +++ b/engram/studio/index.html @@ -0,0 +1,3258 @@ + + + + + +Engram Studio + + + + + + + + + + +
+
+
Graph
+
Nodes
+
Timeline
+
Console
+
Swarm
+
+ +
+ + +
+
+ +
+
+
Tiers
+
Working
+
Episodic
+
Semantic
+
Procedural
+
+
+
+
+ + × +
+
+
+
+
+
+
+ + +
+
+
+
All
+
Working
+
Episodic
+
Semantic
+
Procedural
+
+ +
+
+ + + + + + + + + + + + + +
ID ↕Type ↕Tier ↕ContentSalience ↕Activations ↕Last Active ↕
+
+
+
+ + × +
+
+
+
+
+
Connections
+
+
+
+
+ + +
+ +
+ + +
+
+
+ engram> + +
+
+ + + + +
+
+ + + + +
+ + + +