Archived
feat: Engram sync layer — swarm memory protocol, peer delta sync, distributed activation
This commit is contained in:
Will Anderson
@@ -0,0 +1,165 @@
|
||||
/// Engram Server — HTTP API for Engram with sync and swarm activation.
|
||||
///
|
||||
/// # Endpoints
|
||||
///
|
||||
/// ## Core
|
||||
/// GET /stats — node/edge counts
|
||||
/// POST /nodes — create a node
|
||||
/// GET /nodes/{id} — get a node
|
||||
/// POST /edges — create an edge
|
||||
/// GET /nodes/{id}/edges — list edges from a node
|
||||
/// POST /activate — spreading activation
|
||||
/// POST /search — embedding search
|
||||
/// POST /decay — apply salience decay
|
||||
/// POST /consolidate — promote Episodic → Semantic
|
||||
///
|
||||
/// ## Sync (auth required)
|
||||
/// GET /sync/delta?since={ms}&peer_id={uuid} — generate delta
|
||||
/// POST /sync/push — receive incoming delta
|
||||
/// POST /sync/peers — register peer
|
||||
/// GET /sync/peers — list peers
|
||||
/// DELETE /sync/peers/{id} — remove peer
|
||||
///
|
||||
/// ## Swarm
|
||||
/// POST /swarm/activate — distributed activation (auth required)
|
||||
/// GET /swarm/status — peer health (auth required)
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{
|
||||
middleware,
|
||||
routing::{delete, get, post},
|
||||
Router,
|
||||
};
|
||||
use engram_core::EngramDb;
|
||||
use engram_sync::{SyncConfig, SyncEngine};
|
||||
use tokio::time::interval;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
mod auth;
|
||||
mod routes;
|
||||
mod state;
|
||||
|
||||
use auth::require_auth;
|
||||
use state::AppState;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Logging
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
// Configuration from environment (with sensible defaults)
|
||||
let db_path = std::env::var("ENGRAM_DB_PATH").unwrap_or_else(|_| "./engram-data".to_string());
|
||||
let bind_addr = std::env::var("ENGRAM_BIND").unwrap_or_else(|_| "0.0.0.0:8742".to_string());
|
||||
let api_key = std::env::var("ENGRAM_API_KEY").unwrap_or_else(|_| {
|
||||
let key = uuid::Uuid::new_v4().to_string();
|
||||
eprintln!("No ENGRAM_API_KEY set — generated key: {}", key);
|
||||
key
|
||||
});
|
||||
|
||||
// Open database
|
||||
let db = EngramDb::open(&PathBuf::from(&db_path))?;
|
||||
let db = Arc::new(Mutex::new(db));
|
||||
|
||||
info!("Database opened at {}", db_path);
|
||||
|
||||
// Sync engine — wrapped in tokio::sync::Mutex so it can be held across .await
|
||||
let sync_config = SyncConfig {
|
||||
our_id: uuid::Uuid::new_v4(),
|
||||
our_name: std::env::var("ENGRAM_PEER_NAME").unwrap_or_else(|_| "engram-local".to_string()),
|
||||
api_key: api_key.clone(),
|
||||
default_sync_tiers: vec![engram_core::types::MemoryTier::Semantic],
|
||||
sync_interval_secs: std::env::var("ENGRAM_SYNC_INTERVAL_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(300),
|
||||
};
|
||||
let sync_interval_secs = sync_config.sync_interval_secs;
|
||||
let sync_engine = Arc::new(tokio::sync::Mutex::new(SyncEngine::new(db.clone(), sync_config)));
|
||||
|
||||
{
|
||||
let e = sync_engine.lock().await;
|
||||
info!(
|
||||
peer_name = e.our_name(),
|
||||
peer_id = %e.our_id(),
|
||||
sync_interval_secs,
|
||||
"Sync engine ready"
|
||||
);
|
||||
}
|
||||
|
||||
// Background sync task — tokio::sync::Mutex guard is Send-safe
|
||||
{
|
||||
let engine_arc = sync_engine.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = interval(Duration::from_secs(sync_interval_secs));
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
let report = {
|
||||
let mut e = engine_arc.lock().await;
|
||||
e.sync_all().await
|
||||
};
|
||||
if report.peers_synced > 0 || !report.errors.is_empty() {
|
||||
info!(
|
||||
peers_synced = report.peers_synced,
|
||||
nodes_received = report.nodes_received,
|
||||
nodes_sent = report.nodes_sent,
|
||||
errors = report.errors.len(),
|
||||
"Sync cycle complete"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Shared state
|
||||
let state = Arc::new(AppState {
|
||||
db: db.clone(),
|
||||
sync_engine: sync_engine.clone(),
|
||||
api_key: api_key.clone(),
|
||||
});
|
||||
|
||||
// Protected sync/swarm routes (auth middleware applied)
|
||||
let sync_routes = Router::new()
|
||||
.route("/sync/delta", get(routes::sync::get_delta))
|
||||
.route("/sync/push", post(routes::sync::push_delta))
|
||||
.route("/sync/peers", get(routes::sync::list_peers))
|
||||
.route("/sync/peers", post(routes::sync::register_peer))
|
||||
.route("/sync/peers/{id}", delete(routes::sync::delete_peer))
|
||||
.route("/swarm/activate", post(routes::swarm::swarm_activate))
|
||||
.route("/swarm/status", get(routes::swarm::swarm_status))
|
||||
.layer(middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
require_auth,
|
||||
));
|
||||
|
||||
// Open core routes (no auth)
|
||||
let core_routes = Router::new()
|
||||
.route("/stats", get(routes::core::get_stats))
|
||||
.route("/nodes", post(routes::core::create_node))
|
||||
.route("/nodes/{id}", get(routes::core::get_node))
|
||||
.route("/edges", post(routes::core::create_edge))
|
||||
.route("/nodes/{id}/edges", get(routes::core::get_edges_from))
|
||||
.route("/activate", post(routes::core::activate))
|
||||
.route("/search", post(routes::core::search_embedding))
|
||||
.route("/decay", post(routes::core::decay))
|
||||
.route("/consolidate", post(routes::core::consolidate));
|
||||
|
||||
let app = Router::new()
|
||||
.merge(core_routes)
|
||||
.merge(sync_routes)
|
||||
.layer(CorsLayer::permissive())
|
||||
.with_state(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&bind_addr).await?;
|
||||
info!("Engram server listening on {}", bind_addr);
|
||||
axum::serve(listener, app).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user