fix cross-repo path deps; remove el compiler from neuron-lang/
- neuron-runtime, neuron-store, neuron-migrate: fix path deps (products/engram/ → foundation/engram/engrams/, products/el/ → foundation/el/engrams/) - neuron-rs workspace: fix axon-events/axon-protocol paths (../../../platform/ → ../../platform/, paths were off by one level) - neuron-lang/compiler/ removed (el self-hosting compiler now lives in foundation/el/el-compiler/)
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "neuron-migrate"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "neuron-migrate"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
neuron-domain = { path = "../neuron-domain" }
|
||||
neuron-store = { path = "../neuron-store" }
|
||||
engram-core = { path = "../../../../foundation/engram/engrams/engram-core" }
|
||||
rusqlite = { version = "0.31", features = ["bundled"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = "0.3"
|
||||
@@ -0,0 +1,794 @@
|
||||
//! neuron-migrate: Migrate the Neuron production SQLite database into Engram DB format.
|
||||
//!
|
||||
//! Source: ~/.neuron/neuron.db (old SQLite schema)
|
||||
//! Target: ~/.neuron/engram/ (new Engram format via neuron-store)
|
||||
//!
|
||||
//! Tables migrated:
|
||||
//! memory_nodes → Memory domain type → MemoryStore (NodeType::Memory)
|
||||
//! knowledge → KnowledgeEntry → KnowledgeStore (NodeType::Entity)
|
||||
//! backlog_items → BacklogItem → BacklogStore (NodeType::Process)
|
||||
//! execution_contexts → ExecutionContext → ContextStore (NodeType::Event)
|
||||
//! artifacts → Memory (tagged) → MemoryStore (no Artifact domain type yet)
|
||||
//! config_entries → Memory (tagged) → MemoryStore (preserved as structured JSON)
|
||||
//!
|
||||
//! IMPORTANT: sled's EngramDb uses an exclusive file lock per path, so we open
|
||||
//! the DB *once* and perform all node writes through that single handle, bypassing
|
||||
//! the individual store wrappers (which each take ownership of a separate EngramDb).
|
||||
//! The encoding (serde_json) and NodeType assignments mirror neuron-store exactly.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use engram_core::{EngramDb, MemoryTier, Node, NodeType};
|
||||
use neuron_domain::{
|
||||
BacklogItem, BacklogStatus, ContextStatus, ContextStep, ExecutionContext, Importance,
|
||||
KnowledgeEntry, KnowledgeTier, ItemType, Memory, Priority,
|
||||
};
|
||||
use rusqlite::Connection;
|
||||
use serde_json::Value;
|
||||
use std::path::PathBuf;
|
||||
use uuid::Uuid;
|
||||
|
||||
// ── Codec (mirrors neuron-store/src/codec.rs) ─────────────────────────────────
|
||||
|
||||
fn encode<T: serde::Serialize>(value: &T) -> Vec<u8> {
|
||||
serde_json::to_vec(value).expect("serialization should not fail for well-formed domain types")
|
||||
}
|
||||
|
||||
const EMBEDDING_DIM: usize = 64;
|
||||
|
||||
fn hash_embedding(text: &str) -> Vec<f32> {
|
||||
let bytes = text.as_bytes();
|
||||
let mut vals = vec![0.0f32; EMBEDDING_DIM];
|
||||
for (i, b) in bytes.iter().enumerate() {
|
||||
vals[i % EMBEDDING_DIM] += *b as f32;
|
||||
}
|
||||
let norm: f32 = vals.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||
if norm > 0.0 {
|
||||
for v in &mut vals {
|
||||
*v /= norm;
|
||||
}
|
||||
} else {
|
||||
vals[0] = 1.0;
|
||||
}
|
||||
vals
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Parse a pipe/comma-separated or JSON-array tag string into a Vec<String>.
|
||||
fn parse_tags(raw: &str) -> Vec<String> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return vec![];
|
||||
}
|
||||
if let Ok(Value::Array(arr)) = serde_json::from_str(trimmed) {
|
||||
return arr
|
||||
.into_iter()
|
||||
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
||||
.collect();
|
||||
}
|
||||
trimmed
|
||||
.split([',', '|'])
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Parse a JSON array of UUID strings into Vec<Uuid>.
|
||||
fn parse_uuid_list(raw: &str) -> Vec<Uuid> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() || trimmed == "[]" {
|
||||
return vec![];
|
||||
}
|
||||
if let Ok(Value::Array(arr)) = serde_json::from_str(trimmed) {
|
||||
return arr
|
||||
.into_iter()
|
||||
.filter_map(|v| v.as_str().and_then(|s| Uuid::parse_str(s).ok()))
|
||||
.collect();
|
||||
}
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// Parse a JSON array of strings into Vec<String>.
|
||||
fn parse_string_list(raw: &str) -> Vec<String> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() || trimmed == "[]" {
|
||||
return vec![];
|
||||
}
|
||||
if let Ok(Value::Array(arr)) = serde_json::from_str(trimmed) {
|
||||
return arr
|
||||
.into_iter()
|
||||
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
||||
.collect();
|
||||
}
|
||||
vec![trimmed.to_string()]
|
||||
}
|
||||
|
||||
/// Parse a nullable string column to Option<String>, treating empty strings as None.
|
||||
fn opt_str(s: Option<String>) -> Option<String> {
|
||||
s.and_then(|v| if v.is_empty() { None } else { Some(v) })
|
||||
}
|
||||
|
||||
/// Parse an ISO8601 timestamp string or integer ms into i64 ms.
|
||||
fn parse_iso_or_ms(s: &str) -> i64 {
|
||||
let trimmed = s.trim();
|
||||
if let Ok(n) = trimmed.parse::<i64>() {
|
||||
return n;
|
||||
}
|
||||
use chrono::DateTime;
|
||||
if let Ok(dt) = DateTime::parse_from_rfc3339(trimmed) {
|
||||
return dt.timestamp_millis();
|
||||
}
|
||||
// Try "YYYY-MM-DD HH:MM:SS+TZ" format
|
||||
if let Ok(dt) = chrono::DateTime::parse_from_str(trimmed, "%Y-%m-%d %H:%M:%S%.f%z") {
|
||||
return dt.timestamp_millis();
|
||||
}
|
||||
neuron_domain::now_ms()
|
||||
}
|
||||
|
||||
/// Derive a deterministic UUID from a string identifier by simple mixing.
|
||||
fn deterministic_uuid(s: &str) -> Uuid {
|
||||
let bytes = s.as_bytes();
|
||||
let mut out = [0u8; 16];
|
||||
for (i, b) in bytes.iter().enumerate() {
|
||||
out[i % 16] ^= b.wrapping_add((i as u8).wrapping_mul(31));
|
||||
}
|
||||
// Set UUID version 5 bits (variant 2, version 5)
|
||||
out[6] = (out[6] & 0x0f) | 0x50;
|
||||
out[8] = (out[8] & 0x3f) | 0x80;
|
||||
Uuid::from_bytes(out)
|
||||
}
|
||||
|
||||
/// Parse an old-style prefixed ID (e.g. "mem-xxxxxxxx") into a Uuid.
|
||||
fn parse_id(id_str: &str, prefix: &str) -> Uuid {
|
||||
// Try plain UUID first
|
||||
if let Ok(u) = Uuid::parse_str(id_str) {
|
||||
return u;
|
||||
}
|
||||
// Strip known prefix and try again
|
||||
let stripped = id_str.trim_start_matches(prefix).trim_start_matches('-');
|
||||
if let Ok(u) = Uuid::parse_str(stripped) {
|
||||
return u;
|
||||
}
|
||||
// Fall back to deterministic derivation
|
||||
deterministic_uuid(id_str)
|
||||
}
|
||||
|
||||
// ── Stats ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
struct Stats {
|
||||
migrated: usize,
|
||||
errors: usize,
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
fn new() -> Self { Self { migrated: 0, errors: 0 } }
|
||||
}
|
||||
|
||||
// ── Migration functions ───────────────────────────────────────────────────────
|
||||
|
||||
fn migrate_memory_nodes(src: &Connection, db: &EngramDb) -> Result<Stats> {
|
||||
let mut stmt = src.prepare(
|
||||
"SELECT id, content, tags, importance, superseded_by, \
|
||||
created_at, updated_at, project, metadata, pinned \
|
||||
FROM memory_nodes",
|
||||
)?;
|
||||
|
||||
let mut stats = Stats::new();
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, Option<String>>(4)?,
|
||||
row.get::<_, i64>(5)?,
|
||||
row.get::<_, i64>(6)?,
|
||||
row.get::<_, Option<String>>(7)?,
|
||||
row.get::<_, Option<String>>(8)?,
|
||||
row.get::<_, Option<i64>>(9)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
for row_result in rows {
|
||||
let (id_str, content, tags_raw, importance_str, superseded_by,
|
||||
created_at, updated_at, project, metadata_raw, pinned) = match row_result {
|
||||
Ok(r) => r,
|
||||
Err(e) => { eprintln!(" [memory] row error: {e}"); stats.errors += 1; continue; }
|
||||
};
|
||||
|
||||
let id = parse_id(&id_str, "mem");
|
||||
let supersedes_id = superseded_by.as_deref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| parse_id(s, "mem"));
|
||||
|
||||
let mut tags = parse_tags(&tags_raw);
|
||||
if pinned == Some(1) { tags.push("pinned".to_string()); }
|
||||
|
||||
let extra = match &metadata_raw {
|
||||
Some(m) if m.trim() != "{}" && !m.trim().is_empty() => {
|
||||
format!("\n\n[metadata: {}]", m.trim())
|
||||
}
|
||||
_ => String::new(),
|
||||
};
|
||||
let full_content = if extra.is_empty() { content } else { format!("{}{}", content, extra) };
|
||||
|
||||
let memory = Memory {
|
||||
id,
|
||||
content: full_content,
|
||||
tags,
|
||||
project: opt_str(project),
|
||||
importance: Importance::from_str_lossy(&importance_str),
|
||||
supersedes_id,
|
||||
created_at,
|
||||
updated_at,
|
||||
};
|
||||
|
||||
let content_bytes = encode(&memory);
|
||||
let embedding = hash_embedding(&memory.content);
|
||||
let importance = memory.importance.to_f32();
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Memory,
|
||||
embedding,
|
||||
content_bytes,
|
||||
MemoryTier::Episodic,
|
||||
importance,
|
||||
).with_id(memory.id);
|
||||
|
||||
match db.put_node(node) {
|
||||
Ok(_) => stats.migrated += 1,
|
||||
Err(e) => { eprintln!(" [memory] put error for {id_str}: {e}"); stats.errors += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
fn migrate_knowledge(src: &Connection, db: &EngramDb) -> Result<Stats> {
|
||||
let mut stmt = src.prepare(
|
||||
"SELECT id, key, description, tags, raw_content, compiled_content, \
|
||||
tier, disposition, created_at \
|
||||
FROM knowledge",
|
||||
)?;
|
||||
|
||||
let mut stats = Stats::new();
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, String>(4)?,
|
||||
row.get::<_, String>(5)?,
|
||||
row.get::<_, String>(6)?,
|
||||
row.get::<_, String>(7)?,
|
||||
row.get::<_, String>(8)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
for row_result in rows {
|
||||
let (id_str, key, description, tags_raw, raw_content, compiled_content,
|
||||
tier_str, disposition, created_at_str) = match row_result {
|
||||
Ok(r) => r,
|
||||
Err(e) => { eprintln!(" [knowledge] row error: {e}"); stats.errors += 1; continue; }
|
||||
};
|
||||
|
||||
let id = parse_id(&id_str, "kn");
|
||||
|
||||
// Use description as title when it differs from the path key
|
||||
let title = if description.is_empty() || description == key {
|
||||
key.clone()
|
||||
} else {
|
||||
description.clone()
|
||||
};
|
||||
|
||||
// Prefer compiled_content; fall back to raw_content
|
||||
let content = if !compiled_content.trim().is_empty() {
|
||||
compiled_content
|
||||
} else {
|
||||
raw_content
|
||||
};
|
||||
|
||||
let mut tags = parse_tags(&tags_raw);
|
||||
if !disposition.is_empty() && disposition != "experimental" {
|
||||
tags.push(format!("disposition:{}", disposition));
|
||||
}
|
||||
|
||||
// Infer category from path prefix
|
||||
let category = key.splitn(2, '/').next().unwrap_or("general").to_string();
|
||||
let created_at = parse_iso_or_ms(&created_at_str);
|
||||
|
||||
let entry = KnowledgeEntry {
|
||||
id,
|
||||
title,
|
||||
content: content.clone(),
|
||||
category,
|
||||
tier: KnowledgeTier::from_str_lossy(&tier_str),
|
||||
tags,
|
||||
project: None,
|
||||
created_at,
|
||||
};
|
||||
|
||||
let content_bytes = encode(&entry);
|
||||
let text = format!("{} {} {}", entry.title, entry.category, content);
|
||||
let embedding = hash_embedding(&text);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Entity,
|
||||
embedding,
|
||||
content_bytes,
|
||||
MemoryTier::Semantic,
|
||||
0.7,
|
||||
).with_id(entry.id);
|
||||
|
||||
match db.put_node(node) {
|
||||
Ok(_) => stats.migrated += 1,
|
||||
Err(e) => { eprintln!(" [knowledge] put error for {id_str}: {e}"); stats.errors += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
fn migrate_backlog(src: &Connection, db: &EngramDb) -> Result<Stats> {
|
||||
let mut stmt = src.prepare(
|
||||
"SELECT id, title, description, priority, status, project, tags, \
|
||||
item_type, depends_on, created_at, updated_at, \
|
||||
complexity, effort_estimate, resolution_notes, assigned_to, process, metadata \
|
||||
FROM backlog_items",
|
||||
)?;
|
||||
|
||||
let mut stats = Stats::new();
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, String>(4)?,
|
||||
row.get::<_, Option<String>>(5)?,
|
||||
row.get::<_, String>(6)?,
|
||||
row.get::<_, String>(7)?,
|
||||
row.get::<_, String>(8)?,
|
||||
row.get::<_, i64>(9)?,
|
||||
row.get::<_, i64>(10)?,
|
||||
row.get::<_, Option<String>>(11)?,
|
||||
row.get::<_, Option<String>>(12)?,
|
||||
row.get::<_, Option<String>>(13)?,
|
||||
row.get::<_, Option<String>>(14)?,
|
||||
row.get::<_, Option<String>>(15)?,
|
||||
row.get::<_, Option<String>>(16)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
for row_result in rows {
|
||||
let (id_str, title, description, priority_str, status_str, project, tags_raw,
|
||||
item_type_str, depends_on_raw, created_at, updated_at,
|
||||
complexity, effort_estimate, resolution_notes, assigned_to, process, metadata_raw)
|
||||
= match row_result {
|
||||
Ok(r) => r,
|
||||
Err(e) => { eprintln!(" [backlog] row error: {e}"); stats.errors += 1; continue; }
|
||||
};
|
||||
|
||||
let id = parse_id(&id_str, "bl");
|
||||
|
||||
let mut tags = parse_tags(&tags_raw);
|
||||
if let Some(c) = complexity.as_deref() {
|
||||
if !c.is_empty() && c != "moderate" { tags.push(format!("complexity:{}", c)); }
|
||||
}
|
||||
if let Some(e) = effort_estimate.as_deref() {
|
||||
if !e.is_empty() { tags.push(format!("effort:{}", e)); }
|
||||
}
|
||||
if let Some(a) = assigned_to.as_deref() {
|
||||
if !a.is_empty() { tags.push(format!("assigned:{}", a)); }
|
||||
}
|
||||
if let Some(p) = process.as_deref() {
|
||||
if !p.is_empty() { tags.push(format!("process:{}", p)); }
|
||||
}
|
||||
|
||||
let mut full_description = description;
|
||||
if let Some(notes) = resolution_notes.as_deref() {
|
||||
if !notes.is_empty() { full_description.push_str(&format!("\n\n[Resolution: {}]", notes)); }
|
||||
}
|
||||
if let Some(m) = metadata_raw.as_deref() {
|
||||
if m.trim() != "{}" && !m.trim().is_empty() {
|
||||
full_description.push_str(&format!("\n\n[metadata: {}]", m.trim()));
|
||||
}
|
||||
}
|
||||
|
||||
let depends_on = parse_uuid_list(&depends_on_raw);
|
||||
|
||||
let item = BacklogItem {
|
||||
id,
|
||||
title: title.clone(),
|
||||
description: full_description.clone(),
|
||||
item_type: ItemType::from_str_lossy(&item_type_str),
|
||||
priority: Priority::from_str_lossy(&priority_str),
|
||||
status: BacklogStatus::from_str_lossy(&status_str),
|
||||
project: opt_str(project),
|
||||
tags,
|
||||
depends_on,
|
||||
created_at,
|
||||
updated_at,
|
||||
};
|
||||
|
||||
let content_bytes = encode(&item);
|
||||
let text = format!("{} {}", title, full_description);
|
||||
let embedding = hash_embedding(&text);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Process,
|
||||
embedding,
|
||||
content_bytes,
|
||||
MemoryTier::Working,
|
||||
0.6,
|
||||
).with_id(item.id);
|
||||
|
||||
match db.put_node(node) {
|
||||
Ok(_) => stats.migrated += 1,
|
||||
Err(e) => { eprintln!(" [backlog] put error for {id_str}: {e}"); stats.errors += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
fn migrate_contexts(src: &Connection, db: &EngramDb) -> Result<Stats> {
|
||||
let mut stmt = src.prepare(
|
||||
"SELECT id, process_name, description, objective, status, project, \
|
||||
COALESCE(steps_json, '[]'), COALESCE(file_refs, '[]'), \
|
||||
COALESCE(key_decisions, '[]'), COALESCE(lessons_learned, '[]'), \
|
||||
created_at, updated_at, summary, results_summary, error_message \
|
||||
FROM execution_contexts",
|
||||
)?;
|
||||
|
||||
let mut stats = Stats::new();
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, String>(4)?,
|
||||
row.get::<_, Option<String>>(5)?,
|
||||
row.get::<_, String>(6)?,
|
||||
row.get::<_, String>(7)?,
|
||||
row.get::<_, String>(8)?,
|
||||
row.get::<_, String>(9)?,
|
||||
row.get::<_, i64>(10)?,
|
||||
row.get::<_, i64>(11)?,
|
||||
row.get::<_, Option<String>>(12)?,
|
||||
row.get::<_, Option<String>>(13)?,
|
||||
row.get::<_, Option<String>>(14)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
for row_result in rows {
|
||||
let (id_str, process_name, mut description, objective, status_str, project,
|
||||
steps_json, file_refs_raw, key_decisions_raw, lessons_learned_raw,
|
||||
created_at, updated_at, summary, results_summary, error_message)
|
||||
= match row_result {
|
||||
Ok(r) => r,
|
||||
Err(e) => { eprintln!(" [contexts] row error: {e}"); stats.errors += 1; continue; }
|
||||
};
|
||||
|
||||
let id = parse_id(&id_str, "ctx");
|
||||
|
||||
if let Some(s) = summary.as_deref() {
|
||||
if !s.is_empty() { description.push_str(&format!("\n\n[Summary: {}]", s)); }
|
||||
}
|
||||
if let Some(r) = results_summary.as_deref() {
|
||||
if !r.is_empty() { description.push_str(&format!("\n\n[Results: {}]", r)); }
|
||||
}
|
||||
if let Some(e) = error_message.as_deref() {
|
||||
if !e.is_empty() { description.push_str(&format!("\n\n[Error: {}]", e)); }
|
||||
}
|
||||
|
||||
let steps = parse_steps_json(&steps_json);
|
||||
let file_refs = parse_string_list(&file_refs_raw);
|
||||
let key_decisions = parse_string_list(&key_decisions_raw);
|
||||
let lessons_learned = parse_string_list(&lessons_learned_raw);
|
||||
|
||||
let ctx = ExecutionContext {
|
||||
id,
|
||||
process_name: process_name.clone(),
|
||||
description,
|
||||
objective: objective.clone(),
|
||||
status: ContextStatus::from_str_lossy(&status_str),
|
||||
project: opt_str(project),
|
||||
steps,
|
||||
file_refs,
|
||||
key_decisions,
|
||||
lessons_learned,
|
||||
created_at,
|
||||
updated_at,
|
||||
};
|
||||
|
||||
let content_bytes = encode(&ctx);
|
||||
let text = format!("{} {}", process_name, objective);
|
||||
let embedding = hash_embedding(&text);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Event,
|
||||
embedding,
|
||||
content_bytes,
|
||||
MemoryTier::Working,
|
||||
0.7,
|
||||
).with_id(ctx.id);
|
||||
|
||||
match db.put_node(node) {
|
||||
Ok(_) => stats.migrated += 1,
|
||||
Err(e) => { eprintln!(" [contexts] put error for {id_str}: {e}"); stats.errors += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Migrate artifacts as Memory nodes tagged with "artifact".
|
||||
fn migrate_artifacts(src: &Connection, db: &EngramDb) -> Result<Stats> {
|
||||
let mut stmt = src.prepare(
|
||||
"SELECT id, title, content, artifact_types, status, project, \
|
||||
tags, version, created_at, updated_at \
|
||||
FROM artifacts",
|
||||
)?;
|
||||
|
||||
let mut stats = Stats::new();
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, String>(4)?,
|
||||
row.get::<_, Option<String>>(5)?,
|
||||
row.get::<_, String>(6)?,
|
||||
row.get::<_, i64>(7)?,
|
||||
row.get::<_, i64>(8)?,
|
||||
row.get::<_, i64>(9)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
for row_result in rows {
|
||||
let (id_str, title, content, artifact_types, status, project,
|
||||
tags_raw, version, created_at, updated_at) = match row_result {
|
||||
Ok(r) => r,
|
||||
Err(e) => { eprintln!(" [artifacts] row error: {e}"); stats.errors += 1; continue; }
|
||||
};
|
||||
|
||||
let id = parse_id(&id_str, "art");
|
||||
|
||||
let mut tags = parse_tags(&tags_raw);
|
||||
tags.push("artifact".to_string());
|
||||
tags.push(format!("artifact_status:{}", status));
|
||||
for atype in parse_tags(&artifact_types) {
|
||||
tags.push(format!("artifact_type:{}", atype));
|
||||
}
|
||||
if version > 1 { tags.push(format!("version:{}", version)); }
|
||||
|
||||
// Embed full artifact as structured JSON so it's fully recoverable
|
||||
let artifact_doc = serde_json::json!({
|
||||
"type": "artifact",
|
||||
"id": id_str,
|
||||
"title": title,
|
||||
"status": status,
|
||||
"artifact_types": parse_tags(&artifact_types),
|
||||
"version": version,
|
||||
"content": content,
|
||||
});
|
||||
let memory_content = format!(
|
||||
"ARTIFACT: {}\n\n{}\n\n---\n{}",
|
||||
title, content, artifact_doc,
|
||||
);
|
||||
|
||||
let memory = Memory {
|
||||
id,
|
||||
content: memory_content,
|
||||
tags,
|
||||
project: opt_str(project),
|
||||
importance: Importance::High,
|
||||
supersedes_id: None,
|
||||
created_at,
|
||||
updated_at,
|
||||
};
|
||||
|
||||
let content_bytes = encode(&memory);
|
||||
let embedding = hash_embedding(&memory.content);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Memory,
|
||||
embedding,
|
||||
content_bytes,
|
||||
MemoryTier::Episodic,
|
||||
Importance::High.to_f32(),
|
||||
).with_id(memory.id);
|
||||
|
||||
match db.put_node(node) {
|
||||
Ok(_) => stats.migrated += 1,
|
||||
Err(e) => { eprintln!(" [artifacts] put error for {id_str}: {e}"); stats.errors += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Migrate config_entries as Memory nodes tagged with "config".
|
||||
fn migrate_config(src: &Connection, db: &EngramDb) -> Result<Stats> {
|
||||
let mut stmt = src.prepare(
|
||||
"SELECT key, value, updated_at FROM config_entries",
|
||||
)?;
|
||||
|
||||
let mut stats = Stats::new();
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
for row_result in rows {
|
||||
let (key, value, updated_at) = match row_result {
|
||||
Ok(r) => r,
|
||||
Err(e) => { eprintln!(" [config] row error: {e}"); stats.errors += 1; continue; }
|
||||
};
|
||||
|
||||
let id = deterministic_uuid(&format!("config:{}", key));
|
||||
let content_str = format!("CONFIG: {}\n\nValue: {}", key, value);
|
||||
|
||||
let memory = Memory {
|
||||
id,
|
||||
content: content_str,
|
||||
tags: vec!["config".to_string(), format!("config_key:{}", key)],
|
||||
project: None,
|
||||
importance: Importance::High,
|
||||
supersedes_id: None,
|
||||
created_at: updated_at,
|
||||
updated_at,
|
||||
};
|
||||
|
||||
let content_bytes = encode(&memory);
|
||||
let embedding = hash_embedding(&memory.content);
|
||||
|
||||
let node = Node::new(
|
||||
NodeType::Memory,
|
||||
embedding,
|
||||
content_bytes,
|
||||
MemoryTier::Episodic,
|
||||
Importance::High.to_f32(),
|
||||
).with_id(memory.id);
|
||||
|
||||
match db.put_node(node) {
|
||||
Ok(_) => {
|
||||
println!(" config: {}", key);
|
||||
stats.migrated += 1;
|
||||
}
|
||||
Err(e) => { eprintln!(" [config] put error for {key}: {e}"); stats.errors += 1; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
// ── Step parsers ──────────────────────────────────────────────────────────────
|
||||
|
||||
fn parse_steps_json(raw: &str) -> Vec<ContextStep> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() || trimmed == "[]" { return vec![]; }
|
||||
let arr: Vec<Value> = match serde_json::from_str(trimmed) {
|
||||
Ok(Value::Array(a)) => a,
|
||||
_ => return vec![],
|
||||
};
|
||||
|
||||
arr.into_iter().filter_map(|v| {
|
||||
let name = v.get("name").and_then(|n| n.as_str())
|
||||
.or_else(|| v.get("action").and_then(|a| a.as_str()))
|
||||
.unwrap_or("unknown").to_string();
|
||||
let status = v.get("status").and_then(|s| s.as_str())
|
||||
.unwrap_or("completed").to_string();
|
||||
let notes = v.get("notes").and_then(|n| n.as_str())
|
||||
.filter(|s| !s.is_empty()).map(|s| s.to_string());
|
||||
let started_at = v.get("started_at").and_then(|t| t.as_i64())
|
||||
.unwrap_or_else(neuron_domain::now_ms);
|
||||
let completed_at = v.get("completed_at").and_then(|t| t.as_i64());
|
||||
Some(ContextStep { name, status, notes, started_at, completed_at })
|
||||
}).collect()
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter("warn")
|
||||
.init();
|
||||
|
||||
let home = std::env::var("HOME").unwrap_or_else(|_| "/root".to_string());
|
||||
let src_path = PathBuf::from(&home).join(".neuron/neuron.db");
|
||||
let dst_path_str = std::env::var("NEURON_DB_PATH")
|
||||
.unwrap_or_else(|_| format!("{}/.neuron/engram", home));
|
||||
let dst_path = PathBuf::from(&dst_path_str);
|
||||
|
||||
println!("neuron-migrate");
|
||||
println!(" source: {}", src_path.display());
|
||||
println!(" target: {}", dst_path.display());
|
||||
println!();
|
||||
|
||||
if !src_path.exists() {
|
||||
anyhow::bail!("source database not found: {}", src_path.display());
|
||||
}
|
||||
|
||||
std::fs::create_dir_all(&dst_path)
|
||||
.with_context(|| format!("failed to create target dir: {}", dst_path.display()))?;
|
||||
|
||||
// Open source DB (read-only)
|
||||
let src = Connection::open_with_flags(
|
||||
&src_path,
|
||||
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
|
||||
).with_context(|| format!("failed to open source DB: {}", src_path.display()))?;
|
||||
|
||||
// Open target Engram DB — SINGLE handle for the entire migration.
|
||||
// sled uses an exclusive file lock; opening multiple handles to the same path
|
||||
// in the same process will fail with WouldBlock on the second open.
|
||||
let db = EngramDb::open(&dst_path)
|
||||
.map_err(|e| anyhow::anyhow!("failed to open Engram DB: {e}"))?;
|
||||
|
||||
println!("Migrating memory_nodes...");
|
||||
let mem_stats = migrate_memory_nodes(&src, &db)?;
|
||||
println!(" done: {} migrated, {} errors", mem_stats.migrated, mem_stats.errors);
|
||||
|
||||
println!("Migrating knowledge...");
|
||||
let kn_stats = migrate_knowledge(&src, &db)?;
|
||||
println!(" done: {} migrated, {} errors", kn_stats.migrated, kn_stats.errors);
|
||||
|
||||
println!("Migrating backlog_items...");
|
||||
let bl_stats = migrate_backlog(&src, &db)?;
|
||||
println!(" done: {} migrated, {} errors", bl_stats.migrated, bl_stats.errors);
|
||||
|
||||
println!("Migrating execution_contexts...");
|
||||
let ctx_stats = migrate_contexts(&src, &db)?;
|
||||
println!(" done: {} migrated, {} errors", ctx_stats.migrated, ctx_stats.errors);
|
||||
|
||||
println!("Migrating artifacts (as tagged memory nodes)...");
|
||||
let art_stats = migrate_artifacts(&src, &db)?;
|
||||
println!(" done: {} migrated, {} errors", art_stats.migrated, art_stats.errors);
|
||||
|
||||
println!("Migrating config_entries (as tagged memory nodes)...");
|
||||
let cfg_stats = migrate_config(&src, &db)?;
|
||||
println!(" done: {} migrated, {} errors", cfg_stats.migrated, cfg_stats.errors);
|
||||
|
||||
let total_migrated = mem_stats.migrated + kn_stats.migrated + bl_stats.migrated
|
||||
+ ctx_stats.migrated + art_stats.migrated + cfg_stats.migrated;
|
||||
let total_errors = mem_stats.errors + kn_stats.errors + bl_stats.errors
|
||||
+ ctx_stats.errors + art_stats.errors + cfg_stats.errors;
|
||||
|
||||
println!();
|
||||
println!("=== Migration Report ===");
|
||||
println!(" memory_nodes : {:>4} records", mem_stats.migrated);
|
||||
println!(" knowledge : {:>4} records", kn_stats.migrated);
|
||||
println!(" backlog_items : {:>4} records", bl_stats.migrated);
|
||||
println!(" execution_contexts : {:>4} records", ctx_stats.migrated);
|
||||
println!(" artifacts : {:>4} records", art_stats.migrated);
|
||||
println!(" config_entries : {:>4} records", cfg_stats.migrated);
|
||||
println!(" ---");
|
||||
println!(" TOTAL MIGRATED : {:>4} records", total_migrated);
|
||||
if total_errors > 0 {
|
||||
println!(" TOTAL ERRORS : {:>4} records", total_errors);
|
||||
}
|
||||
println!();
|
||||
println!("Engram DB: {}", dst_path.display());
|
||||
println!("Source DB preserved: {}", src_path.display());
|
||||
|
||||
if total_errors > 0 {
|
||||
eprintln!("WARNING: {} record(s) could not be migrated (see errors above).", total_errors);
|
||||
} else {
|
||||
println!("Migration completed successfully with zero errors.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user