feat: Engram sync layer — swarm memory protocol, peer delta sync, distributed activation
This commit is contained in:
Generated
+1632
-11
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,8 @@ members = [
|
||||
"crates/engram-ffi",
|
||||
"crates/engram-jni",
|
||||
"crates/engram-migrate",
|
||||
"crates/engram-sync",
|
||||
"crates/engram-server",
|
||||
# engram-wasm is in bindings/ and compiled separately via wasm-pack
|
||||
# (wasm targets can't be in the same workspace build as native targets)
|
||||
]
|
||||
|
||||
@@ -187,6 +187,34 @@ mod sled_impl {
|
||||
pub fn edge_count(&self) -> EngramResult<usize> {
|
||||
graph::edge_count(&self.db)
|
||||
}
|
||||
|
||||
// ── Bulk scan (for sync) ───────────────────────────────────────────────
|
||||
|
||||
/// Scan all nodes in the store.
|
||||
///
|
||||
/// Used by the sync engine to generate delta snapshots.
|
||||
pub fn scan_nodes(&self) -> EngramResult<Vec<crate::types::Node>> {
|
||||
storage::scan_nodes(&self.db)
|
||||
}
|
||||
|
||||
/// Scan all edges in the store (forward index only).
|
||||
pub fn scan_edges(&self) -> EngramResult<Vec<Edge>> {
|
||||
let prefix = b"edges:from:";
|
||||
let mut edges = Vec::new();
|
||||
for result in self.db.scan_prefix(prefix) {
|
||||
let (_k, v) = result?;
|
||||
let edge: Edge = bincode::deserialize(&v)?;
|
||||
edges.push(edge);
|
||||
}
|
||||
Ok(edges)
|
||||
}
|
||||
|
||||
/// Delete a node by UUID (tombstone support for sync).
|
||||
pub fn delete_node(&self, id: Uuid) -> EngramResult<()> {
|
||||
let key = storage::node_key(id);
|
||||
self.db.remove(key)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "engram-server"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "HTTP server for Engram — REST API + sync endpoints + swarm activation"
|
||||
license = "MIT"
|
||||
|
||||
[[bin]]
|
||||
name = "engram-server"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
engram-core = { path = "../engram-core" }
|
||||
engram-sync = { path = "../engram-sync" }
|
||||
axum = { version = "0.7", features = ["json"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tower = "0.4"
|
||||
tower-http = { version = "0.5", features = ["cors", "fs"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
anyhow = "1"
|
||||
thiserror = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
[dev-dependencies]
|
||||
axum-test = "14"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,31 @@
|
||||
/// Auth middleware for sync endpoints.
|
||||
///
|
||||
/// Sync and swarm endpoints require `Authorization: Bearer {api_key}`.
|
||||
/// The API key is configured at server startup and stored in AppState.
|
||||
use axum::{
|
||||
extract::{Request, State},
|
||||
http::StatusCode,
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Axum middleware that checks the Authorization header on sync/swarm routes.
|
||||
pub async fn require_auth(
|
||||
State(state): State<Arc<AppState>>,
|
||||
req: Request,
|
||||
next: Next,
|
||||
) -> Result<Response, StatusCode> {
|
||||
let api_key = req
|
||||
.headers()
|
||||
.get("Authorization")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "));
|
||||
|
||||
match api_key {
|
||||
Some(key) if key == state.api_key => Ok(next.run(req).await),
|
||||
_ => Err(StatusCode::UNAUTHORIZED),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
/// Engram Server — HTTP API for Engram with sync and swarm activation.
|
||||
///
|
||||
/// # Endpoints
|
||||
///
|
||||
/// ## Core
|
||||
/// GET /stats — node/edge counts
|
||||
/// POST /nodes — create a node
|
||||
/// GET /nodes/{id} — get a node
|
||||
/// POST /edges — create an edge
|
||||
/// GET /nodes/{id}/edges — list edges from a node
|
||||
/// POST /activate — spreading activation
|
||||
/// POST /search — embedding search
|
||||
/// POST /decay — apply salience decay
|
||||
/// POST /consolidate — promote Episodic → Semantic
|
||||
///
|
||||
/// ## Sync (auth required)
|
||||
/// GET /sync/delta?since={ms}&peer_id={uuid} — generate delta
|
||||
/// POST /sync/push — receive incoming delta
|
||||
/// POST /sync/peers — register peer
|
||||
/// GET /sync/peers — list peers
|
||||
/// DELETE /sync/peers/{id} — remove peer
|
||||
///
|
||||
/// ## Swarm
|
||||
/// POST /swarm/activate — distributed activation (auth required)
|
||||
/// GET /swarm/status — peer health (auth required)
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{
|
||||
middleware,
|
||||
routing::{delete, get, post},
|
||||
Router,
|
||||
};
|
||||
use engram_core::EngramDb;
|
||||
use engram_sync::{SyncConfig, SyncEngine};
|
||||
use tokio::time::interval;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
mod auth;
|
||||
mod routes;
|
||||
mod state;
|
||||
|
||||
use auth::require_auth;
|
||||
use state::AppState;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Logging
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
// Configuration from environment (with sensible defaults)
|
||||
let db_path = std::env::var("ENGRAM_DB_PATH").unwrap_or_else(|_| "./engram-data".to_string());
|
||||
let bind_addr = std::env::var("ENGRAM_BIND").unwrap_or_else(|_| "0.0.0.0:8742".to_string());
|
||||
let api_key = std::env::var("ENGRAM_API_KEY").unwrap_or_else(|_| {
|
||||
let key = uuid::Uuid::new_v4().to_string();
|
||||
eprintln!("No ENGRAM_API_KEY set — generated key: {}", key);
|
||||
key
|
||||
});
|
||||
|
||||
// Open database
|
||||
let db = EngramDb::open(&PathBuf::from(&db_path))?;
|
||||
let db = Arc::new(Mutex::new(db));
|
||||
|
||||
info!("Database opened at {}", db_path);
|
||||
|
||||
// Sync engine — wrapped in tokio::sync::Mutex so it can be held across .await
|
||||
let sync_config = SyncConfig {
|
||||
our_id: uuid::Uuid::new_v4(),
|
||||
our_name: std::env::var("ENGRAM_PEER_NAME").unwrap_or_else(|_| "engram-local".to_string()),
|
||||
api_key: api_key.clone(),
|
||||
default_sync_tiers: vec![engram_core::types::MemoryTier::Semantic],
|
||||
sync_interval_secs: std::env::var("ENGRAM_SYNC_INTERVAL_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(300),
|
||||
};
|
||||
let sync_interval_secs = sync_config.sync_interval_secs;
|
||||
let sync_engine = Arc::new(tokio::sync::Mutex::new(SyncEngine::new(db.clone(), sync_config)));
|
||||
|
||||
{
|
||||
let e = sync_engine.lock().await;
|
||||
info!(
|
||||
peer_name = e.our_name(),
|
||||
peer_id = %e.our_id(),
|
||||
sync_interval_secs,
|
||||
"Sync engine ready"
|
||||
);
|
||||
}
|
||||
|
||||
// Background sync task — tokio::sync::Mutex guard is Send-safe
|
||||
{
|
||||
let engine_arc = sync_engine.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = interval(Duration::from_secs(sync_interval_secs));
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
let report = {
|
||||
let mut e = engine_arc.lock().await;
|
||||
e.sync_all().await
|
||||
};
|
||||
if report.peers_synced > 0 || !report.errors.is_empty() {
|
||||
info!(
|
||||
peers_synced = report.peers_synced,
|
||||
nodes_received = report.nodes_received,
|
||||
nodes_sent = report.nodes_sent,
|
||||
errors = report.errors.len(),
|
||||
"Sync cycle complete"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Shared state
|
||||
let state = Arc::new(AppState {
|
||||
db: db.clone(),
|
||||
sync_engine: sync_engine.clone(),
|
||||
api_key: api_key.clone(),
|
||||
});
|
||||
|
||||
// Protected sync/swarm routes (auth middleware applied)
|
||||
let sync_routes = Router::new()
|
||||
.route("/sync/delta", get(routes::sync::get_delta))
|
||||
.route("/sync/push", post(routes::sync::push_delta))
|
||||
.route("/sync/peers", get(routes::sync::list_peers))
|
||||
.route("/sync/peers", post(routes::sync::register_peer))
|
||||
.route("/sync/peers/{id}", delete(routes::sync::delete_peer))
|
||||
.route("/swarm/activate", post(routes::swarm::swarm_activate))
|
||||
.route("/swarm/status", get(routes::swarm::swarm_status))
|
||||
.layer(middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
require_auth,
|
||||
));
|
||||
|
||||
// Open core routes (no auth)
|
||||
let core_routes = Router::new()
|
||||
.route("/stats", get(routes::core::get_stats))
|
||||
.route("/nodes", post(routes::core::create_node))
|
||||
.route("/nodes/{id}", get(routes::core::get_node))
|
||||
.route("/edges", post(routes::core::create_edge))
|
||||
.route("/nodes/{id}/edges", get(routes::core::get_edges_from))
|
||||
.route("/activate", post(routes::core::activate))
|
||||
.route("/search", post(routes::core::search_embedding))
|
||||
.route("/decay", post(routes::core::decay))
|
||||
.route("/consolidate", post(routes::core::consolidate));
|
||||
|
||||
let app = Router::new()
|
||||
.merge(core_routes)
|
||||
.merge(sync_routes)
|
||||
.layer(CorsLayer::permissive())
|
||||
.with_state(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&bind_addr).await?;
|
||||
info!("Engram server listening on {}", bind_addr);
|
||||
axum::serve(listener, app).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/// Core Engram API routes — nodes, edges, activation, search.
|
||||
use axum::{
|
||||
extract::{Path, State},
|
||||
http::StatusCode,
|
||||
Json,
|
||||
};
|
||||
use engram_core::types::{Edge, MemoryTier, Node, NodeType, RelationType};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
// ── Stats ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct StatsResponse {
|
||||
pub nodes: usize,
|
||||
pub edges: usize,
|
||||
}
|
||||
|
||||
pub async fn get_stats(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<StatsResponse>, StatusCode> {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let nodes = db.node_count().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let edges = db.edge_count().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(StatsResponse { nodes, edges }))
|
||||
}
|
||||
|
||||
// ── Nodes ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct CreateNodeRequest {
|
||||
pub node_type: NodeType,
|
||||
pub embedding: Vec<f32>,
|
||||
pub content: Vec<u8>,
|
||||
pub tier: MemoryTier,
|
||||
pub importance: f32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct CreateNodeResponse {
|
||||
pub id: Uuid,
|
||||
}
|
||||
|
||||
pub async fn create_node(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<CreateNodeRequest>,
|
||||
) -> Result<Json<CreateNodeResponse>, StatusCode> {
|
||||
let node = Node::new(req.node_type, req.embedding, req.content, req.tier, req.importance);
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let id = db.put_node(node).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(CreateNodeResponse { id }))
|
||||
}
|
||||
|
||||
pub async fn get_node(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> Result<Json<Node>, StatusCode> {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
match db.get_node(id).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? {
|
||||
Some(node) => Ok(Json(node)),
|
||||
None => Err(StatusCode::NOT_FOUND),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Edges ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct CreateEdgeRequest {
|
||||
pub from_id: Uuid,
|
||||
pub to_id: Uuid,
|
||||
pub relation: RelationType,
|
||||
pub weight: f32,
|
||||
}
|
||||
|
||||
pub async fn create_edge(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<CreateEdgeRequest>,
|
||||
) -> Result<StatusCode, StatusCode> {
|
||||
let edge = Edge::new(req.from_id, req.to_id, req.relation, req.weight);
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
db.put_edge(edge).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(StatusCode::CREATED)
|
||||
}
|
||||
|
||||
pub async fn get_edges_from(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> Result<Json<Vec<Edge>>, StatusCode> {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let edges = db.get_edges_from(id).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(edges))
|
||||
}
|
||||
|
||||
// ── Activation ────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ActivateRequest {
|
||||
pub seeds: Vec<Uuid>,
|
||||
pub query_embedding: Vec<f32>,
|
||||
#[serde(default = "default_depth")]
|
||||
pub max_depth: u8,
|
||||
#[serde(default = "default_limit")]
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
fn default_depth() -> u8 { 3 }
|
||||
fn default_limit() -> usize { 10 }
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ActivateResponse {
|
||||
pub results: Vec<ActivatedNodeJson>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ActivatedNodeJson {
|
||||
pub node: Node,
|
||||
pub activation_strength: f32,
|
||||
pub hops: u8,
|
||||
}
|
||||
|
||||
pub async fn activate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<ActivateRequest>,
|
||||
) -> Result<Json<ActivateResponse>, StatusCode> {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let results = db
|
||||
.activate(&req.seeds, &req.query_embedding, req.max_depth, req.limit)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
Ok(Json(ActivateResponse {
|
||||
results: results
|
||||
.into_iter()
|
||||
.map(|a| ActivatedNodeJson {
|
||||
node: a.node,
|
||||
activation_strength: a.activation_strength,
|
||||
hops: a.hops,
|
||||
})
|
||||
.collect(),
|
||||
}))
|
||||
}
|
||||
|
||||
// ── Search ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SearchRequest {
|
||||
pub embedding: Vec<f32>,
|
||||
#[serde(default = "default_limit")]
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct SearchResponse {
|
||||
pub results: Vec<ScoredNodeJson>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ScoredNodeJson {
|
||||
pub node: Node,
|
||||
pub score: f32,
|
||||
}
|
||||
|
||||
pub async fn search_embedding(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<SearchRequest>,
|
||||
) -> Result<Json<SearchResponse>, StatusCode> {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let results = db
|
||||
.search_embedding(&req.embedding, req.limit)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
Ok(Json(SearchResponse {
|
||||
results: results
|
||||
.into_iter()
|
||||
.map(|s| ScoredNodeJson { node: s.node, score: s.score })
|
||||
.collect(),
|
||||
}))
|
||||
}
|
||||
|
||||
// ── Decay / Consolidate ───────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DecayRequest {
|
||||
pub factor: f32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct DecayResponse {
|
||||
pub nodes_updated: usize,
|
||||
}
|
||||
|
||||
pub async fn decay(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<DecayRequest>,
|
||||
) -> Result<Json<DecayResponse>, StatusCode> {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let nodes_updated = db.decay(req.factor).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(DecayResponse { nodes_updated }))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ConsolidateResponse {
|
||||
pub promoted: usize,
|
||||
}
|
||||
|
||||
pub async fn consolidate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<ConsolidateResponse>, StatusCode> {
|
||||
use engram_core::ConsolidationConfig;
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let report = db
|
||||
.consolidate(&ConsolidationConfig::default())
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(ConsolidateResponse { promoted: report.promoted }))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod core;
|
||||
pub mod sync;
|
||||
pub mod swarm;
|
||||
@@ -0,0 +1,124 @@
|
||||
/// Swarm routes — distributed activation across the peer network.
|
||||
///
|
||||
/// POST /swarm/activate — SwarmActivateRequest → SwarmActivateResponse
|
||||
/// GET /swarm/status — peer health check and last sync times
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::StatusCode,
|
||||
Json,
|
||||
};
|
||||
use engram_sync::{
|
||||
client::SyncClient, merge_activation_results, Peer, PeerActivationResult, PeerStatus,
|
||||
SerializableActivatedNode, SwarmActivateRequest, SwarmActivateResponse,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
/// POST /swarm/activate
|
||||
///
|
||||
/// Runs spreading activation locally, then (if include_peers=true) fans out
|
||||
/// to all trusted peers in parallel and returns merged results.
|
||||
pub async fn swarm_activate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<SwarmActivateRequest>,
|
||||
) -> Result<Json<SwarmActivateResponse>, StatusCode> {
|
||||
// Step 1: Run local activation. Lock db, compute, drop immediately.
|
||||
let local_results: Vec<SerializableActivatedNode> = {
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let activated = db
|
||||
.activate(&req.seeds, &req.query_embedding, req.max_depth, req.limit)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
activated.into_iter().map(Into::into).collect()
|
||||
// db lock released here
|
||||
};
|
||||
|
||||
// Step 2: Snapshot peer list (lock, read, drop).
|
||||
let (our_id, trusted_peers): (Uuid, Vec<Peer>) = {
|
||||
let engine = state.sync_engine.lock().await;
|
||||
let id = engine.our_id();
|
||||
let peers = engine.list_peers().iter().filter(|p| p.trusted).cloned().collect();
|
||||
(id, peers)
|
||||
// engine lock released here
|
||||
};
|
||||
|
||||
// Step 3: Fan out to peers — no locks held across these awaits.
|
||||
let mut peer_results: Vec<PeerActivationResult> = Vec::new();
|
||||
|
||||
if req.include_peers {
|
||||
let mut handles = Vec::new();
|
||||
for peer in trusted_peers {
|
||||
let seeds = req.seeds.clone();
|
||||
let embedding = req.query_embedding.clone();
|
||||
let max_depth = req.max_depth;
|
||||
let limit = req.limit;
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let client = SyncClient::new(peer.clone(), our_id);
|
||||
match client.remote_activate(&seeds, &embedding, max_depth, limit).await {
|
||||
Ok(results) => PeerActivationResult {
|
||||
peer_id: peer.id,
|
||||
peer_name: peer.name,
|
||||
results,
|
||||
error: None,
|
||||
},
|
||||
Err(e) => PeerActivationResult {
|
||||
peer_id: peer.id,
|
||||
peer_name: peer.name,
|
||||
results: Vec::new(),
|
||||
error: Some(e.to_string()),
|
||||
},
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
if let Ok(result) = handle.await {
|
||||
peer_results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Merge and return.
|
||||
let merged = merge_activation_results(&local_results, &peer_results, req.limit);
|
||||
|
||||
Ok(Json(SwarmActivateResponse {
|
||||
local_results,
|
||||
peer_results,
|
||||
merged,
|
||||
}))
|
||||
}
|
||||
|
||||
/// GET /swarm/status
|
||||
///
|
||||
/// Returns peer list with reachability status and last sync times.
|
||||
pub async fn swarm_status(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<Vec<PeerStatus>>, StatusCode> {
|
||||
// Snapshot peer list and our_id without holding the lock across awaits.
|
||||
let (our_id, peers): (Uuid, Vec<Peer>) = {
|
||||
let engine = state.sync_engine.lock().await;
|
||||
let id = engine.our_id();
|
||||
let peers = engine.list_peers().to_vec();
|
||||
(id, peers)
|
||||
};
|
||||
|
||||
let mut statuses: Vec<PeerStatus> = Vec::new();
|
||||
for peer in peers {
|
||||
use engram_core::types::now_ms;
|
||||
let client = SyncClient::new(peer.clone(), our_id);
|
||||
let reachable = client.pull_delta(now_ms()).await.is_ok();
|
||||
statuses.push(PeerStatus {
|
||||
peer_id: peer.id,
|
||||
peer_name: peer.name,
|
||||
address: peer.address,
|
||||
last_sync_at: peer.last_sync_at,
|
||||
reachable,
|
||||
sync_tiers: peer.sync_tiers,
|
||||
trusted: peer.trusted,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Json(statuses))
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/// Sync routes — peer delta exchange and peer registry.
|
||||
///
|
||||
/// All routes under /sync require Authorization: Bearer {api_key}.
|
||||
///
|
||||
/// GET /sync/delta?since={ms}&peer_id={uuid} — generate delta for caller
|
||||
/// POST /sync/push — receive delta from peer
|
||||
/// POST /sync/peers — register a new peer
|
||||
/// GET /sync/peers — list peers
|
||||
/// DELETE /sync/peers/{id} — remove peer
|
||||
use axum::{
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
Json,
|
||||
};
|
||||
use engram_core::types::MemoryTier;
|
||||
use engram_sync::{Peer, SyncDelta};
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
// ── GET /sync/delta ───────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DeltaParams {
|
||||
pub since: Option<i64>,
|
||||
pub peer_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
pub async fn get_delta(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<DeltaParams>,
|
||||
) -> Result<Json<SyncDelta>, StatusCode> {
|
||||
let since = params.since.unwrap_or(0);
|
||||
|
||||
// Determine which tiers to expose based on caller's peer_id
|
||||
let tiers: Vec<MemoryTier> = {
|
||||
let engine = state.sync_engine.lock().await;
|
||||
if let Some(peer_id) = params.peer_id {
|
||||
if let Some(peer) = engine.get_peer(peer_id) {
|
||||
if peer.trusted {
|
||||
peer.sync_tiers.clone()
|
||||
} else {
|
||||
vec![MemoryTier::Semantic]
|
||||
}
|
||||
} else {
|
||||
vec![MemoryTier::Semantic]
|
||||
}
|
||||
} else {
|
||||
vec![MemoryTier::Semantic]
|
||||
}
|
||||
// engine lock released here
|
||||
};
|
||||
|
||||
let engine = state.sync_engine.lock().await;
|
||||
let delta = engine
|
||||
.generate_delta(since, &tiers)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
Ok(Json(delta))
|
||||
}
|
||||
|
||||
// ── POST /sync/push ───────────────────────────────────────────────────────────
|
||||
|
||||
pub async fn push_delta(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(delta): Json<SyncDelta>,
|
||||
) -> Result<StatusCode, StatusCode> {
|
||||
// Accepted tiers for incoming pushes (Semantic and Procedural by default)
|
||||
let accepted_tiers = vec![MemoryTier::Semantic, MemoryTier::Procedural];
|
||||
|
||||
// Apply the delta using the DB handle directly (no async needed)
|
||||
let db = state.db.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
// Tombstones
|
||||
for id in &delta.tombstones {
|
||||
let _ = db.delete_node(*id);
|
||||
}
|
||||
|
||||
// Nodes
|
||||
for node in delta.nodes {
|
||||
if !accepted_tiers.contains(&node.tier) {
|
||||
continue;
|
||||
}
|
||||
if db.get_node(node.id).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?.is_some() {
|
||||
continue;
|
||||
}
|
||||
db.put_node(node).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
}
|
||||
|
||||
// Edges
|
||||
for edge in delta.edges {
|
||||
let from_ok = db.get_node(edge.from_id)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.is_some();
|
||||
let to_ok = db.get_node(edge.to_id)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.is_some();
|
||||
if from_ok && to_ok {
|
||||
let _ = db.put_edge(edge);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
// ── Peer registry ─────────────────────────────────────────────────────────────
|
||||
|
||||
pub async fn list_peers(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<Vec<Peer>>, StatusCode> {
|
||||
let engine = state.sync_engine.lock().await;
|
||||
Ok(Json(engine.list_peers().to_vec()))
|
||||
}
|
||||
|
||||
pub async fn register_peer(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(peer): Json<Peer>,
|
||||
) -> Result<StatusCode, StatusCode> {
|
||||
let mut engine = state.sync_engine.lock().await;
|
||||
engine.add_peer(peer);
|
||||
Ok(StatusCode::CREATED)
|
||||
}
|
||||
|
||||
pub async fn delete_peer(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> Result<StatusCode, StatusCode> {
|
||||
let mut engine = state.sync_engine.lock().await;
|
||||
engine.remove_peer(id);
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
/// Shared application state for all request handlers.
|
||||
use engram_core::EngramDb;
|
||||
use engram_sync::SyncEngine;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
pub struct AppState {
|
||||
/// Local database — uses std::sync::Mutex (sync ops only, fast)
|
||||
pub db: Arc<Mutex<EngramDb>>,
|
||||
/// Sync engine — uses tokio::sync::Mutex so it can be held across .await
|
||||
pub sync_engine: Arc<tokio::sync::Mutex<SyncEngine>>,
|
||||
/// API key used to authenticate incoming sync requests
|
||||
pub api_key: String,
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "engram-sync"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Swarm memory sync layer for Engram — peer delta sync and distributed activation"
|
||||
license = "MIT"
|
||||
|
||||
[lib]
|
||||
name = "engram_sync"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
engram-core = { path = "../engram-core" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
reqwest = { version = "0.12", features = ["json"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
anyhow = "1"
|
||||
thiserror = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
tokio = { version = "1", features = ["full", "test-util"] }
|
||||
@@ -0,0 +1,122 @@
|
||||
/// HTTP client for talking to a remote Engram peer.
|
||||
///
|
||||
/// All peer-to-peer communication goes through this client.
|
||||
/// Authentication is via `Authorization: Bearer {api_key}` on every request.
|
||||
use crate::types::{Peer, SerializableActivatedNode, SyncDelta};
|
||||
use anyhow::Context;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub struct SyncClient {
|
||||
peer: Peer,
|
||||
http: reqwest::Client,
|
||||
our_id: Uuid,
|
||||
}
|
||||
|
||||
impl SyncClient {
|
||||
pub fn new(peer: Peer, our_id: Uuid) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("failed to build reqwest client");
|
||||
Self { peer, http, our_id }
|
||||
}
|
||||
|
||||
/// Pull a delta from the remote peer containing everything since `since` (Unix ms).
|
||||
///
|
||||
/// GET {address}/sync/delta?since={since}&peer_id={our_id}
|
||||
pub async fn pull_delta(&self, since: i64) -> anyhow::Result<SyncDelta> {
|
||||
let url = format!(
|
||||
"{}/sync/delta?since={}&peer_id={}",
|
||||
self.peer.address, since, self.our_id
|
||||
);
|
||||
let resp = self
|
||||
.http
|
||||
.get(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.peer.api_key))
|
||||
.send()
|
||||
.await
|
||||
.context("pull_delta: request failed")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("pull_delta: peer returned {}: {}", status, body);
|
||||
}
|
||||
|
||||
resp.json::<SyncDelta>()
|
||||
.await
|
||||
.context("pull_delta: failed to decode response")
|
||||
}
|
||||
|
||||
/// Push our delta to the remote peer.
|
||||
///
|
||||
/// POST {address}/sync/push
|
||||
pub async fn push_delta(&self, delta: &SyncDelta) -> anyhow::Result<()> {
|
||||
let url = format!("{}/sync/push", self.peer.address);
|
||||
let resp = self
|
||||
.http
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.peer.api_key))
|
||||
.json(delta)
|
||||
.send()
|
||||
.await
|
||||
.context("push_delta: request failed")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("push_delta: peer returned {}: {}", status, body);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fan spreading activation out to a remote peer.
|
||||
///
|
||||
/// POST {address}/swarm/activate — the remote peer runs activation locally
|
||||
/// (include_peers=false so it does not fan out further, preventing cycles).
|
||||
pub async fn remote_activate(
|
||||
&self,
|
||||
seeds: &[Uuid],
|
||||
query_embedding: &[f32],
|
||||
max_depth: u8,
|
||||
limit: usize,
|
||||
) -> anyhow::Result<Vec<SerializableActivatedNode>> {
|
||||
use crate::types::SwarmActivateRequest;
|
||||
|
||||
let url = format!("{}/swarm/activate", self.peer.address);
|
||||
let req = SwarmActivateRequest {
|
||||
seeds: seeds.to_vec(),
|
||||
query_embedding: query_embedding.to_vec(),
|
||||
max_depth,
|
||||
limit,
|
||||
include_peers: false, // no further fan-out — prevents cycles
|
||||
};
|
||||
|
||||
let resp = self
|
||||
.http
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.peer.api_key))
|
||||
.json(&req)
|
||||
.send()
|
||||
.await
|
||||
.context("remote_activate: request failed")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("remote_activate: peer returned {}: {}", status, body);
|
||||
}
|
||||
|
||||
// The remote returns a SwarmActivateResponse; we only want local_results
|
||||
let response: crate::types::SwarmActivateResponse = resp
|
||||
.json()
|
||||
.await
|
||||
.context("remote_activate: failed to decode response")?;
|
||||
Ok(response.local_results)
|
||||
}
|
||||
|
||||
pub fn peer(&self) -> &Peer {
|
||||
&self.peer
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
/// SyncEngine — orchestrates peer sync and swarm activation.
|
||||
///
|
||||
/// The engine holds a list of known peers, a handle to the local database,
|
||||
/// and owns our identity (UUID + API key). It drives two workflows:
|
||||
///
|
||||
/// 1. **Delta sync**: periodic pull-then-push with each peer, filtering by
|
||||
/// tier allowlist and skipping nodes we already have.
|
||||
///
|
||||
/// 2. **Swarm activation**: run spreading activation locally, then fan out
|
||||
/// to all trusted peers in parallel, and merge results by strength.
|
||||
use crate::client::SyncClient;
|
||||
use crate::types::{
|
||||
MergedActivatedNode, Peer, PeerActivationResult, PeerStatus, PeerSyncResult,
|
||||
SerializableActivatedNode, SyncConfig, SyncDelta, SyncReport, SwarmActivateRequest,
|
||||
SwarmActivateResponse,
|
||||
};
|
||||
use engram_core::types::{now_ms, MemoryTier};
|
||||
use engram_core::EngramDb;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub struct SyncEngine {
|
||||
db: Arc<Mutex<EngramDb>>,
|
||||
peers: Vec<Peer>,
|
||||
our_id: Uuid,
|
||||
our_name: String,
|
||||
api_key: String,
|
||||
default_sync_tiers: Vec<MemoryTier>,
|
||||
}
|
||||
|
||||
impl SyncEngine {
|
||||
pub fn new(db: Arc<Mutex<EngramDb>>, config: SyncConfig) -> Self {
|
||||
Self {
|
||||
db,
|
||||
peers: Vec::new(),
|
||||
our_id: config.our_id,
|
||||
our_name: config.our_name,
|
||||
api_key: config.api_key,
|
||||
default_sync_tiers: config.default_sync_tiers,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Peer registry ─────────────────────────────────────────────────────────
|
||||
|
||||
pub fn add_peer(&mut self, peer: Peer) {
|
||||
// Replace if already registered
|
||||
self.peers.retain(|p| p.id != peer.id);
|
||||
self.peers.push(peer);
|
||||
}
|
||||
|
||||
pub fn remove_peer(&mut self, peer_id: Uuid) {
|
||||
self.peers.retain(|p| p.id != peer_id);
|
||||
}
|
||||
|
||||
pub fn list_peers(&self) -> &[Peer] {
|
||||
&self.peers
|
||||
}
|
||||
|
||||
pub fn get_peer(&self, peer_id: Uuid) -> Option<&Peer> {
|
||||
self.peers.iter().find(|p| p.id == peer_id)
|
||||
}
|
||||
|
||||
pub fn our_id(&self) -> Uuid {
|
||||
self.our_id
|
||||
}
|
||||
|
||||
pub fn our_name(&self) -> &str {
|
||||
&self.our_name
|
||||
}
|
||||
|
||||
pub fn api_key(&self) -> &str {
|
||||
&self.api_key
|
||||
}
|
||||
|
||||
// ── Full sync cycle ───────────────────────────────────────────────────────
|
||||
|
||||
/// Sync with every registered peer. Returns a report summarising what moved.
|
||||
pub async fn sync_all(&mut self) -> SyncReport {
|
||||
let mut report = SyncReport {
|
||||
peers_synced: 0,
|
||||
nodes_received: 0,
|
||||
nodes_sent: 0,
|
||||
errors: Vec::new(),
|
||||
};
|
||||
|
||||
// Clone the peer list so we can mutate self afterwards
|
||||
let peers: Vec<Peer> = self.peers.clone();
|
||||
for peer in peers {
|
||||
match self.sync_peer(&peer).await {
|
||||
Ok(result) => {
|
||||
report.peers_synced += 1;
|
||||
report.nodes_received += result.nodes_received;
|
||||
report.nodes_sent += result.nodes_sent;
|
||||
// Update last_sync_at
|
||||
if let Some(p) = self.peers.iter_mut().find(|p| p.id == peer.id) {
|
||||
p.last_sync_at = now_ms();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push(format!("peer {}: {}", peer.name, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report
|
||||
}
|
||||
|
||||
/// Sync a single peer: pull delta, apply it, then push our delta.
|
||||
pub async fn sync_peer(&self, peer: &Peer) -> anyhow::Result<PeerSyncResult> {
|
||||
let client = SyncClient::new(peer.clone(), self.our_id);
|
||||
let tiers = effective_tiers(peer, &self.default_sync_tiers);
|
||||
|
||||
// Pull: get everything from the peer since their last known sync time
|
||||
let remote_delta = client.pull_delta(peer.last_sync_at).await?;
|
||||
let nodes_received = self.apply_delta(remote_delta, &tiers).await?;
|
||||
|
||||
// Push: send everything we have that the peer hasn't seen
|
||||
let our_delta = self.generate_delta(peer.last_sync_at, &tiers)?;
|
||||
let nodes_sent = our_delta.nodes.len();
|
||||
client.push_delta(&our_delta).await?;
|
||||
|
||||
Ok(PeerSyncResult {
|
||||
peer_id: peer.id,
|
||||
nodes_received,
|
||||
nodes_sent,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Delta generation ──────────────────────────────────────────────────────
|
||||
|
||||
/// Build a delta containing all nodes/edges modified since `since`
|
||||
/// that belong to one of the allowed `tiers`.
|
||||
pub fn generate_delta(
|
||||
&self,
|
||||
since: i64,
|
||||
tiers: &[MemoryTier],
|
||||
) -> anyhow::Result<SyncDelta> {
|
||||
let db = self
|
||||
.db
|
||||
.lock()
|
||||
.map_err(|_| anyhow::anyhow!("db lock poisoned"))?;
|
||||
|
||||
// Scan all nodes; filter by tier and modification time
|
||||
let all_nodes = db.scan_nodes()?;
|
||||
let nodes: Vec<_> = all_nodes
|
||||
.into_iter()
|
||||
.filter(|n| tiers.contains(&n.tier) && n.last_activated >= since)
|
||||
.collect();
|
||||
|
||||
// Edges: include all edges between nodes in our set
|
||||
let node_ids: std::collections::HashSet<Uuid> = nodes.iter().map(|n| n.id).collect();
|
||||
let all_edges = db.scan_edges()?;
|
||||
let edges: Vec<_> = all_edges
|
||||
.into_iter()
|
||||
.filter(|e| node_ids.contains(&e.from_id) && node_ids.contains(&e.to_id))
|
||||
.collect();
|
||||
|
||||
Ok(SyncDelta {
|
||||
peer_id: self.our_id,
|
||||
since,
|
||||
nodes,
|
||||
edges,
|
||||
tombstones: Vec::new(), // tombstone tracking requires a separate log; not yet implemented
|
||||
generated_at: now_ms(),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Delta application ─────────────────────────────────────────────────────
|
||||
|
||||
/// Merge an incoming delta into the local database.
|
||||
///
|
||||
/// - Nodes/edges we already have (same UUID) are skipped — local wins.
|
||||
/// - Tombstones cause deletion.
|
||||
/// - Only nodes in the allowed tiers are accepted.
|
||||
///
|
||||
/// Returns the number of nodes actually written.
|
||||
pub async fn apply_delta(
|
||||
&self,
|
||||
delta: SyncDelta,
|
||||
allowed_tiers: &[MemoryTier],
|
||||
) -> anyhow::Result<usize> {
|
||||
let db = self
|
||||
.db
|
||||
.lock()
|
||||
.map_err(|_| anyhow::anyhow!("db lock poisoned"))?;
|
||||
|
||||
let mut written = 0usize;
|
||||
|
||||
// Apply tombstones first
|
||||
for id in &delta.tombstones {
|
||||
let _ = db.delete_node(*id);
|
||||
}
|
||||
|
||||
// Merge nodes
|
||||
for node in delta.nodes {
|
||||
if !allowed_tiers.contains(&node.tier) {
|
||||
continue;
|
||||
}
|
||||
// Skip if we already have this UUID
|
||||
if db.get_node(node.id)?.is_some() {
|
||||
continue;
|
||||
}
|
||||
db.put_node(node)?;
|
||||
written += 1;
|
||||
}
|
||||
|
||||
// Merge edges (if both endpoints exist)
|
||||
for edge in delta.edges {
|
||||
let from_exists = db.get_node(edge.from_id)?.is_some();
|
||||
let to_exists = db.get_node(edge.to_id)?.is_some();
|
||||
if from_exists && to_exists {
|
||||
db.put_edge(edge)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
// ── Swarm activation ──────────────────────────────────────────────────────
|
||||
|
||||
/// Run spreading activation locally, then fan out to all trusted peers,
|
||||
/// and merge all results into a unified ranked list.
|
||||
pub async fn swarm_activate(
|
||||
&self,
|
||||
req: SwarmActivateRequest,
|
||||
) -> anyhow::Result<SwarmActivateResponse> {
|
||||
// Local activation
|
||||
let local_results: Vec<SerializableActivatedNode> = {
|
||||
let db = self
|
||||
.db
|
||||
.lock()
|
||||
.map_err(|_| anyhow::anyhow!("db lock poisoned"))?;
|
||||
let activated = db.activate(
|
||||
&req.seeds,
|
||||
&req.query_embedding,
|
||||
req.max_depth,
|
||||
req.limit,
|
||||
)?;
|
||||
activated.into_iter().map(Into::into).collect()
|
||||
};
|
||||
|
||||
let mut peer_results: Vec<PeerActivationResult> = Vec::new();
|
||||
|
||||
if req.include_peers {
|
||||
// Fan out to all trusted peers in parallel
|
||||
let trusted_peers: Vec<Peer> = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|p| p.trusted)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for peer in trusted_peers {
|
||||
let seeds = req.seeds.clone();
|
||||
let embedding = req.query_embedding.clone();
|
||||
let max_depth = req.max_depth;
|
||||
let limit = req.limit;
|
||||
let our_id = self.our_id;
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let client = SyncClient::new(peer.clone(), our_id);
|
||||
match client
|
||||
.remote_activate(&seeds, &embedding, max_depth, limit)
|
||||
.await
|
||||
{
|
||||
Ok(results) => PeerActivationResult {
|
||||
peer_id: peer.id,
|
||||
peer_name: peer.name,
|
||||
results,
|
||||
error: None,
|
||||
},
|
||||
Err(e) => PeerActivationResult {
|
||||
peer_id: peer.id,
|
||||
peer_name: peer.name,
|
||||
results: Vec::new(),
|
||||
error: Some(e.to_string()),
|
||||
},
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
match handle.await {
|
||||
Ok(result) => peer_results.push(result),
|
||||
Err(e) => peer_results.push(PeerActivationResult {
|
||||
peer_id: Uuid::nil(),
|
||||
peer_name: "unknown".into(),
|
||||
results: Vec::new(),
|
||||
error: Some(format!("task error: {}", e)),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let merged = merge_activation_results(&local_results, &peer_results, req.limit);
|
||||
|
||||
Ok(SwarmActivateResponse {
|
||||
local_results,
|
||||
peer_results,
|
||||
merged,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Peer health check ─────────────────────────────────────────────────────
|
||||
|
||||
/// Check which peers are reachable and return their status.
|
||||
pub async fn peer_statuses(&self) -> Vec<PeerStatus> {
|
||||
let mut statuses = Vec::new();
|
||||
for peer in &self.peers {
|
||||
let client = SyncClient::new(peer.clone(), self.our_id);
|
||||
// We attempt a health check by pulling an empty delta (since=now)
|
||||
let reachable = client.pull_delta(now_ms()).await.is_ok();
|
||||
statuses.push(PeerStatus {
|
||||
peer_id: peer.id,
|
||||
peer_name: peer.name.clone(),
|
||||
address: peer.address.clone(),
|
||||
last_sync_at: peer.last_sync_at,
|
||||
reachable,
|
||||
sync_tiers: peer.sync_tiers.clone(),
|
||||
trusted: peer.trusted,
|
||||
});
|
||||
}
|
||||
statuses
|
||||
}
|
||||
}
|
||||
|
||||
// ── Merge logic ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// Merge local and peer activation results into a unified ranked list.
|
||||
///
|
||||
/// Deduplication: two nodes are considered duplicates if they have the same UUID.
|
||||
/// When duplicates occur, the one with the highest activation strength is kept.
|
||||
/// The merged list is sorted by activation_strength descending and truncated to `limit`.
|
||||
pub fn merge_activation_results(
|
||||
local: &[SerializableActivatedNode],
|
||||
peer_results: &[PeerActivationResult],
|
||||
limit: usize,
|
||||
) -> Vec<MergedActivatedNode> {
|
||||
use std::collections::HashMap;
|
||||
|
||||
// (uuid -> MergedActivatedNode) — we keep the strongest instance of each node
|
||||
let mut map: HashMap<Uuid, MergedActivatedNode> = HashMap::new();
|
||||
|
||||
// Process local results first (source_peer = None)
|
||||
for item in local {
|
||||
let id = item.node.id;
|
||||
let candidate = MergedActivatedNode {
|
||||
content: String::from_utf8_lossy(&item.node.content).to_string(),
|
||||
node_type: item.node.node_type.clone(),
|
||||
tier: item.node.tier.clone(),
|
||||
activation_strength: item.activation_strength,
|
||||
source_peer: None,
|
||||
hops: item.hops,
|
||||
node: item.node.clone(),
|
||||
};
|
||||
map.entry(id)
|
||||
.and_modify(|existing| {
|
||||
if item.activation_strength > existing.activation_strength {
|
||||
*existing = candidate.clone();
|
||||
}
|
||||
})
|
||||
.or_insert(candidate);
|
||||
}
|
||||
|
||||
// Process peer results
|
||||
for peer_result in peer_results {
|
||||
if peer_result.error.is_some() {
|
||||
continue;
|
||||
}
|
||||
for item in &peer_result.results {
|
||||
let id = item.node.id;
|
||||
let candidate = MergedActivatedNode {
|
||||
content: String::from_utf8_lossy(&item.node.content).to_string(),
|
||||
node_type: item.node.node_type.clone(),
|
||||
tier: item.node.tier.clone(),
|
||||
activation_strength: item.activation_strength,
|
||||
source_peer: Some(peer_result.peer_id),
|
||||
hops: item.hops,
|
||||
node: item.node.clone(),
|
||||
};
|
||||
map.entry(id)
|
||||
.and_modify(|existing| {
|
||||
if item.activation_strength > existing.activation_strength {
|
||||
*existing = candidate.clone();
|
||||
}
|
||||
})
|
||||
.or_insert(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by strength descending, take top N
|
||||
let mut merged: Vec<MergedActivatedNode> = map.into_values().collect();
|
||||
merged.sort_by(|a, b| {
|
||||
b.activation_strength
|
||||
.partial_cmp(&a.activation_strength)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
merged.truncate(limit);
|
||||
merged
|
||||
}
|
||||
|
||||
// ── Tier helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// Compute the effective sync tiers for a peer — intersection of the peer's
|
||||
/// own allowlist and our defaults. Untrusted peers are further restricted
|
||||
/// to Semantic only.
|
||||
fn effective_tiers<'a>(peer: &Peer, defaults: &'a [MemoryTier]) -> Vec<MemoryTier> {
|
||||
if !peer.trusted {
|
||||
// Untrusted: Semantic only, regardless of configuration
|
||||
return vec![MemoryTier::Semantic];
|
||||
}
|
||||
if peer.sync_tiers.is_empty() {
|
||||
defaults.to_vec()
|
||||
} else {
|
||||
// Intersection: only tiers that both us and the peer agree on
|
||||
defaults
|
||||
.iter()
|
||||
.filter(|t| peer.sync_tiers.contains(t))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
/// Engram Sync — swarm memory protocol for distributed Engram instances.
|
||||
///
|
||||
/// This crate turns Engram from a local-first database into a distributed
|
||||
/// swarm memory protocol. Multiple independent Engram instances (peers) can
|
||||
/// share memory across each other using delta sync and can fan spreading
|
||||
/// activation out across the swarm, merging results by strength.
|
||||
///
|
||||
/// # Architecture
|
||||
///
|
||||
/// ```text
|
||||
/// [Neuron-A Engram] <-> sync <-> [Neuron-B Engram]
|
||||
/// |
|
||||
/// [Neuron-C Engram]
|
||||
///
|
||||
/// Swarm activation: seed on A → propagate locally → fan-out to B and C
|
||||
/// → merge all results → unified ranked response
|
||||
/// ```
|
||||
///
|
||||
/// # Protocol
|
||||
///
|
||||
/// - Each peer is local and authoritative. There is no central server.
|
||||
/// - Peers sync via delta exchange: "give me everything since timestamp T".
|
||||
/// - Only configured memory tiers flow between peers (Semantic by default;
|
||||
/// Episodic and Working are private unless explicitly enabled).
|
||||
/// - Trusted peers get the full configured tier set; untrusted peers get
|
||||
/// Semantic only.
|
||||
/// - Swarm activation fans out to all trusted peers in parallel, deduplicates
|
||||
/// by UUID (keeping strongest activation), and re-ranks.
|
||||
|
||||
pub mod client;
|
||||
pub mod engine;
|
||||
pub mod types;
|
||||
|
||||
// Public surface
|
||||
pub use engine::{merge_activation_results, SyncEngine};
|
||||
pub use types::{
|
||||
MergedActivatedNode, Peer, PeerActivationResult, PeerStatus, PeerSyncResult,
|
||||
SerializableActivatedNode, SyncConfig, SyncDelta, SyncReport, SwarmActivateRequest,
|
||||
SwarmActivateResponse,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use engram_core::types::{MemoryTier, Node, NodeType};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn make_node(tier: MemoryTier) -> Node {
|
||||
Node::new(
|
||||
NodeType::Concept,
|
||||
vec![0.1, 0.2, 0.3],
|
||||
b"test content".to_vec(),
|
||||
tier,
|
||||
0.5,
|
||||
)
|
||||
}
|
||||
|
||||
fn make_activated(node: Node, strength: f32, hops: u8) -> SerializableActivatedNode {
|
||||
SerializableActivatedNode {
|
||||
node,
|
||||
activation_strength: strength,
|
||||
hops,
|
||||
}
|
||||
}
|
||||
|
||||
// ── merge_activation_results tests ───────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn merge_empty() {
|
||||
let merged = merge_activation_results(&[], &[], 10);
|
||||
assert!(merged.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_local_only() {
|
||||
let node = make_node(MemoryTier::Semantic);
|
||||
let node_id = node.id;
|
||||
let local = vec![make_activated(node, 0.8, 1)];
|
||||
let merged = merge_activation_results(&local, &[], 10);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].node.id, node_id);
|
||||
assert!(merged[0].source_peer.is_none(), "local nodes have no source_peer");
|
||||
assert!((merged[0].activation_strength - 0.8).abs() < f32::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_peer_result_included() {
|
||||
let local: Vec<SerializableActivatedNode> = vec![];
|
||||
let node = make_node(MemoryTier::Semantic);
|
||||
let peer_id = Uuid::new_v4();
|
||||
let peer_results = vec![PeerActivationResult {
|
||||
peer_id,
|
||||
peer_name: "peer-a".into(),
|
||||
results: vec![make_activated(node, 0.6, 2)],
|
||||
error: None,
|
||||
}];
|
||||
let merged = merge_activation_results(&local, &peer_results, 10);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].source_peer, Some(peer_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_deduplicates_by_uuid_keeps_strongest() {
|
||||
let node = make_node(MemoryTier::Semantic);
|
||||
let id = node.id;
|
||||
|
||||
// Same node appears locally at 0.4 and from a peer at 0.9
|
||||
let local = vec![make_activated(node.clone(), 0.4, 1)];
|
||||
let peer_id = Uuid::new_v4();
|
||||
let peer_results = vec![PeerActivationResult {
|
||||
peer_id,
|
||||
peer_name: "peer-a".into(),
|
||||
results: vec![make_activated(node, 0.9, 1)],
|
||||
error: None,
|
||||
}];
|
||||
|
||||
let merged = merge_activation_results(&local, &peer_results, 10);
|
||||
// Should be deduplicated to 1 result
|
||||
assert_eq!(merged.len(), 1);
|
||||
// The stronger version (0.9, from peer) should win
|
||||
assert!((merged[0].activation_strength - 0.9).abs() < f32::EPSILON);
|
||||
assert_eq!(merged[0].source_peer, Some(peer_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_respects_limit() {
|
||||
let local: Vec<SerializableActivatedNode> = (0..20)
|
||||
.map(|i| make_activated(make_node(MemoryTier::Semantic), i as f32 / 20.0, 1))
|
||||
.collect();
|
||||
let merged = merge_activation_results(&local, &[], 5);
|
||||
assert_eq!(merged.len(), 5, "limit must be respected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_sorted_by_strength_descending() {
|
||||
let strengths = vec![0.3f32, 0.9, 0.1, 0.7, 0.5];
|
||||
let local: Vec<SerializableActivatedNode> = strengths
|
||||
.iter()
|
||||
.map(|&s| make_activated(make_node(MemoryTier::Semantic), s, 1))
|
||||
.collect();
|
||||
let merged = merge_activation_results(&local, &[], 10);
|
||||
let result_strengths: Vec<f32> = merged.iter().map(|m| m.activation_strength).collect();
|
||||
// Should be in descending order
|
||||
for window in result_strengths.windows(2) {
|
||||
assert!(window[0] >= window[1], "results must be sorted descending");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_skips_errored_peers() {
|
||||
let peer_results = vec![PeerActivationResult {
|
||||
peer_id: Uuid::new_v4(),
|
||||
peer_name: "failed-peer".into(),
|
||||
results: vec![make_activated(make_node(MemoryTier::Semantic), 0.9, 1)],
|
||||
error: Some("connection refused".into()),
|
||||
}];
|
||||
let merged = merge_activation_results(&[], &peer_results, 10);
|
||||
// Errored peers should be excluded
|
||||
assert!(merged.is_empty());
|
||||
}
|
||||
|
||||
// ── SyncDelta serialization ───────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn sync_delta_roundtrips_json() {
|
||||
let delta = SyncDelta {
|
||||
peer_id: Uuid::new_v4(),
|
||||
since: 1000,
|
||||
nodes: vec![make_node(MemoryTier::Semantic)],
|
||||
edges: vec![],
|
||||
tombstones: vec![Uuid::new_v4()],
|
||||
generated_at: 2000,
|
||||
};
|
||||
let json = serde_json::to_string(&delta).expect("serialize delta");
|
||||
let decoded: SyncDelta = serde_json::from_str(&json).expect("deserialize delta");
|
||||
assert_eq!(delta.peer_id, decoded.peer_id);
|
||||
assert_eq!(delta.since, decoded.since);
|
||||
assert_eq!(delta.nodes.len(), decoded.nodes.len());
|
||||
assert_eq!(delta.tombstones.len(), decoded.tombstones.len());
|
||||
}
|
||||
|
||||
// ── SyncEngine peer management ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn engine_add_remove_peer() {
|
||||
use engram_core::EngramDb;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Arc::new(Mutex::new(EngramDb::open(dir.path()).unwrap()));
|
||||
let config = SyncConfig::default();
|
||||
let mut engine = SyncEngine::new(db, config);
|
||||
|
||||
assert_eq!(engine.list_peers().len(), 0);
|
||||
|
||||
let peer = Peer {
|
||||
id: Uuid::new_v4(),
|
||||
name: "test-peer".into(),
|
||||
address: "http://localhost:9999".into(),
|
||||
api_key: "secret".into(),
|
||||
sync_tiers: vec![MemoryTier::Semantic],
|
||||
last_sync_at: 0,
|
||||
trusted: true,
|
||||
};
|
||||
let peer_id = peer.id;
|
||||
engine.add_peer(peer);
|
||||
assert_eq!(engine.list_peers().len(), 1);
|
||||
|
||||
engine.remove_peer(peer_id);
|
||||
assert_eq!(engine.list_peers().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_add_peer_replaces_existing() {
|
||||
use engram_core::EngramDb;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Arc::new(Mutex::new(EngramDb::open(dir.path()).unwrap()));
|
||||
let mut engine = SyncEngine::new(db, SyncConfig::default());
|
||||
|
||||
let id = Uuid::new_v4();
|
||||
for i in 0..3 {
|
||||
engine.add_peer(Peer {
|
||||
id,
|
||||
name: format!("peer-v{}", i),
|
||||
address: "http://localhost:1234".into(),
|
||||
api_key: "k".into(),
|
||||
sync_tiers: vec![],
|
||||
last_sync_at: 0,
|
||||
trusted: false,
|
||||
});
|
||||
}
|
||||
// Should still be just one peer (latest version)
|
||||
assert_eq!(engine.list_peers().len(), 1);
|
||||
assert_eq!(engine.list_peers()[0].name, "peer-v2");
|
||||
}
|
||||
|
||||
// ── generate_delta ────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn generate_delta_filters_by_tier() {
|
||||
use engram_core::types::MemoryTier;
|
||||
use engram_core::EngramDb;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Arc::new(Mutex::new(EngramDb::open(dir.path()).unwrap()));
|
||||
|
||||
// Insert nodes in different tiers
|
||||
{
|
||||
let db_locked = db.lock().unwrap();
|
||||
db_locked.put_node(make_node(MemoryTier::Semantic)).unwrap();
|
||||
db_locked.put_node(make_node(MemoryTier::Episodic)).unwrap();
|
||||
db_locked.put_node(make_node(MemoryTier::Working)).unwrap();
|
||||
}
|
||||
|
||||
let engine = SyncEngine::new(db, SyncConfig::default());
|
||||
|
||||
// Only request Semantic tier
|
||||
let delta = engine.generate_delta(0, &[MemoryTier::Semantic]).unwrap();
|
||||
assert_eq!(delta.nodes.len(), 1, "only Semantic node should be in delta");
|
||||
assert!(delta.nodes[0].tier == MemoryTier::Semantic);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
use engram_core::types::{ActivatedNode, Edge, MemoryTier, Node, NodeType};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A remote peer that this Engram instance syncs with.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Peer {
|
||||
/// Stable unique identity for this peer
|
||||
pub id: Uuid,
|
||||
/// Human-readable name (e.g. "neuron-will", "neuron-sarah")
|
||||
pub name: String,
|
||||
/// Base URL of the peer's Engram server (e.g. "https://engram.neurontechnologies.ai")
|
||||
pub address: String,
|
||||
/// Shared secret used in the Authorization: Bearer header
|
||||
pub api_key: String,
|
||||
/// Which memory tiers are allowed to flow to/from this peer.
|
||||
/// Semantic by default — Episodic is private unless explicitly opted in.
|
||||
pub sync_tiers: Vec<MemoryTier>,
|
||||
/// Unix milliseconds of the last successful sync. 0 if never synced.
|
||||
pub last_sync_at: i64,
|
||||
/// Trusted peers get all configured tiers; untrusted peers get Semantic only.
|
||||
pub trusted: bool,
|
||||
}
|
||||
|
||||
/// An incremental change set — everything that changed since a given timestamp.
|
||||
/// Exchanged between peers during sync.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SyncDelta {
|
||||
/// UUID of the peer that generated this delta
|
||||
pub peer_id: Uuid,
|
||||
/// All nodes modified or added after this Unix ms timestamp
|
||||
pub since: i64,
|
||||
/// Nodes added or modified since `since`
|
||||
pub nodes: Vec<Node>,
|
||||
/// Edges added or modified since `since`
|
||||
pub edges: Vec<Edge>,
|
||||
/// Node IDs that were deleted (tombstones) — receivers should remove them
|
||||
pub tombstones: Vec<Uuid>,
|
||||
/// When this delta was generated
|
||||
pub generated_at: i64,
|
||||
}
|
||||
|
||||
/// Request to fan spreading activation out across the swarm.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SwarmActivateRequest {
|
||||
/// Seed node IDs to start activation from
|
||||
pub seeds: Vec<Uuid>,
|
||||
/// Query embedding for semantic scoring
|
||||
pub query_embedding: Vec<f32>,
|
||||
/// Maximum graph hops per peer
|
||||
pub max_depth: u8,
|
||||
/// Maximum results to return per peer (before merge)
|
||||
pub limit: usize,
|
||||
/// If true, fan out to all trusted peers and merge results
|
||||
pub include_peers: bool,
|
||||
}
|
||||
|
||||
/// Response from a swarm activation — local results, per-peer results, and
|
||||
/// the unified merged ranking across all sources.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SwarmActivateResponse {
|
||||
pub local_results: Vec<SerializableActivatedNode>,
|
||||
pub peer_results: Vec<PeerActivationResult>,
|
||||
/// Deduplicated, re-ranked unified results from all sources
|
||||
pub merged: Vec<MergedActivatedNode>,
|
||||
}
|
||||
|
||||
/// ActivatedNode serializable form (ActivatedNode in engram-core is not Serialize).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SerializableActivatedNode {
|
||||
pub node: Node,
|
||||
pub activation_strength: f32,
|
||||
pub hops: u8,
|
||||
}
|
||||
|
||||
impl From<ActivatedNode> for SerializableActivatedNode {
|
||||
fn from(a: ActivatedNode) -> Self {
|
||||
Self {
|
||||
node: a.node,
|
||||
activation_strength: a.activation_strength,
|
||||
hops: a.hops,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Activation results from a single remote peer.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PeerActivationResult {
|
||||
pub peer_id: Uuid,
|
||||
pub peer_name: String,
|
||||
/// Successfully retrieved results (empty on error)
|
||||
pub results: Vec<SerializableActivatedNode>,
|
||||
/// Set if the peer request failed
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// A node in the merged swarm result set — annotated with its origin peer.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MergedActivatedNode {
|
||||
pub content: String,
|
||||
pub node_type: NodeType,
|
||||
pub tier: MemoryTier,
|
||||
/// Composite activation strength after merging
|
||||
pub activation_strength: f32,
|
||||
/// None = local, Some(uuid) = from that peer
|
||||
pub source_peer: Option<Uuid>,
|
||||
pub hops: u8,
|
||||
/// Full node for callers who need it
|
||||
pub node: Node,
|
||||
}
|
||||
|
||||
/// Aggregate report from a full sync cycle.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SyncReport {
|
||||
pub peers_synced: usize,
|
||||
pub nodes_received: usize,
|
||||
pub nodes_sent: usize,
|
||||
pub errors: Vec<String>,
|
||||
}
|
||||
|
||||
/// Result from syncing a single peer.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PeerSyncResult {
|
||||
pub peer_id: Uuid,
|
||||
pub nodes_received: usize,
|
||||
pub nodes_sent: usize,
|
||||
}
|
||||
|
||||
/// Configuration for the sync engine.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SyncConfig {
|
||||
/// This instance's stable UUID
|
||||
pub our_id: Uuid,
|
||||
/// Human name for this instance
|
||||
pub our_name: String,
|
||||
/// API key peers use to authenticate with us
|
||||
pub api_key: String,
|
||||
/// Which tiers to sync by default (peers may also restrict this)
|
||||
pub default_sync_tiers: Vec<MemoryTier>,
|
||||
/// How often to run background sync, in seconds. Default: 300 (5 min)
|
||||
pub sync_interval_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for SyncConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
our_id: Uuid::new_v4(),
|
||||
our_name: "engram-local".to_string(),
|
||||
api_key: Uuid::new_v4().to_string(),
|
||||
default_sync_tiers: vec![MemoryTier::Semantic],
|
||||
sync_interval_secs: 300,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Swarm peer health info for the /swarm/status endpoint.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PeerStatus {
|
||||
pub peer_id: Uuid,
|
||||
pub peer_name: String,
|
||||
pub address: String,
|
||||
pub last_sync_at: i64,
|
||||
pub reachable: bool,
|
||||
pub sync_tiers: Vec<MemoryTier>,
|
||||
pub trusted: bool,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user