diff --git a/Cargo.lock b/Cargo.lock index dc289bf..0755c2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -294,6 +294,7 @@ dependencies = [ "el-parser", "el-registry", "el-seal", + "el-test", "el-types", "thiserror 2.0.18", "tokio", @@ -385,6 +386,19 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "el-test" +version = "0.1.0" +dependencies = [ + "el-compiler", + "el-lexer", + "el-parser", + "el-types", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "el-types" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 065c14c..a17cc3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/el-manifest", "crates/el-registry", "crates/el-build", + "crates/el-test", "bin/el", ] resolver = "2" @@ -28,6 +29,7 @@ el-seal = { path = "crates/el-seal" } el-manifest = { path = "crates/el-manifest" } el-registry = { path = "crates/el-registry" } el-build = { path = "crates/el-build" } +el-test = { path = "crates/el-test" } # Engram crypto (path dep — the sealed target depends on it) engram-crypto = { path = "../engram/crates/engram-crypto" } diff --git a/bin/el/Cargo.toml b/bin/el/Cargo.toml index 7fa2ff9..1928d87 100644 --- a/bin/el/Cargo.toml +++ b/bin/el/Cargo.toml @@ -18,6 +18,7 @@ el-seal = { workspace = true } el-manifest = { workspace = true } el-registry = { workspace = true } el-build = { workspace = true } +el-test = { workspace = true } clap = { workspace = true } thiserror = { workspace = true } tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] } diff --git a/bin/el/src/main.rs b/bin/el/src/main.rs index 450bcc0..1d77217 100644 --- a/bin/el/src/main.rs +++ b/bin/el/src/main.rs @@ -32,6 +32,7 @@ use std::path::PathBuf; use clap::{Parser, Subcommand}; use el_build::BuildSystem; use el_compiler::{Compiler, CompilerOptions, Target}; +use el_test; use el_manifest::{BuildTarget, Manifest}; use el_seal::{seal as seal_fn, unseal as unseal_fn, SealedArtifact, DeploymentBinding, SealAlgorithm, SealConfig}; @@ -101,12 +102,50 @@ enum Command { manifest: Option, }, - /// Run all tests. + /// Run tests in the current project (reads el.toml). Test { + /// Only run tests matching this name (substring match). + filter: Option, + /// Also run e2e tests (requires ENGRAM_URL or ENGRAM_DB_PATH). + #[arg(long)] + e2e: bool, + /// Run unit AND e2e tests. + #[arg(long)] + all: bool, + /// Output format: human (default) | json | junit. + #[arg(long, default_value = "human")] + output: String, + /// Path to the project manifest (default: el.toml). #[arg(long)] manifest: Option, }, + /// Run tests from a single .el file (no el.toml required). + TestFile { + /// Source file containing test blocks (*.el). + file: PathBuf, + /// Only run tests matching this name (substring match). + filter: Option, + /// Also run e2e tests (requires ENGRAM_URL or ENGRAM_DB_PATH). + #[arg(long)] + e2e: bool, + /// Run unit AND e2e tests. + #[arg(long)] + all: bool, + /// Output format: human (default) | json | junit. + #[arg(long, default_value = "human")] + output: String, + }, + + /// Run an Engram source file with the step-debugger attached. + Debug { + /// Source file (*.el). + file: PathBuf, + /// Set a breakpoint at line N. + #[arg(long, value_name = "LINE")] + r#break: Option, + }, + /// Type-check source files without producing artifacts. Check { #[arg(long)] @@ -268,20 +307,51 @@ async fn run(cli: Cli) -> Result<(), Box> { run_interpreter(&instructions); } - Command::Test { manifest } => { + Command::Test { filter, e2e, all, output, manifest } => { let manifest_path = resolve_manifest(manifest.as_deref())?; let bs = BuildSystem::from_manifest_file(&manifest_path)?; - let report = bs.test().await?; - println!( - "test: {} passed, {} failed (total {})", - report.passed, report.failed, report.total - ); - for f in &report.failures { - eprintln!(" FAIL: {f}"); - } - if !report.success() { - std::process::exit(1); + + // Find all .el source files in the project + let entry = bs.manifest.build.entry.clone(); + let entry_path = manifest_path.parent().unwrap_or(std::path::Path::new(".")).join(&entry); + let source = std::fs::read_to_string(&entry_path) + .map_err(|e| format!("cannot read {}: {e}", entry_path.display()))?; + + run_tests_from_source(&source, filter.as_deref(), e2e, all, &output)?; + } + + Command::TestFile { file, filter, e2e, all, output } => { + let source = std::fs::read_to_string(&file) + .map_err(|e| format!("cannot read {}: {e}", file.display()))?; + run_tests_from_source(&source, filter.as_deref(), e2e, all, &output)?; + } + + Command::Debug { file, r#break } => { + let source = std::fs::read_to_string(&file) + .map_err(|e| format!("cannot read {}: {e}", file.display()))?; + + let opts = CompilerOptions { + target: Target::Debug, + source_path: file.clone(), + ..Default::default() + }; + let compiled = Compiler::compile(&source, opts)?; + let instructions = el_compiler::Bytecode::deserialize_all(&compiled.artifact) + .unwrap_or_default(); + + let mut debugger = el_compiler::Debugger::new(); + if let Some(line) = r#break { + // Convert line number to a bytecode offset approximation. + // In a full implementation this would use the source map. + // For now we use the line number directly as a placeholder offset. + debugger.add_breakpoint(line as usize); + println!("debugger: breakpoint set at line {line}"); + } else { + println!("debugger: breaking on first instruction"); } + + println!("debugger: running {} ({} instructions)", file.display(), instructions.len()); + run_interpreter_debug(&instructions, &mut debugger); } Command::Check { manifest } => { @@ -628,6 +698,55 @@ fn build_seal_config() -> Result { }) } +/// Discover tests in source, filter, run, and print results. +fn run_tests_from_source( + source: &str, + filter: Option<&str>, + e2e: bool, + all: bool, + output_fmt: &str, +) -> Result<(), Box> { + use el_test::{TestReport, TestRunner}; + + let mut tests = el_test::discover(source)?; + + // Apply name filter + if let Some(f) = filter { + tests.retain(|t| t.name.contains(f)); + } + + if tests.is_empty() { + println!("no tests found"); + return Ok(()); + } + + let engram_url = std::env::var("ENGRAM_URL").ok().or_else(|| std::env::var("ENGRAM_DB_PATH").ok()); + let url_ref = engram_url.as_deref(); + + let runner = TestRunner::new(); + let results = if all { + runner.run_all(&tests, url_ref) + } else if e2e { + runner.run_e2e(&tests, url_ref.unwrap_or("")) + } else { + runner.run_unit(&tests) + }; + + let report = TestReport::from_results(results); + + match output_fmt { + "json" => println!("{}", report.to_json()), + "junit" => println!("{}", report.to_junit_xml()), + _ => report.print(), + } + + if !report.is_pass() { + std::process::exit(1); + } + + Ok(()) +} + /// Minimal interpreter for demonstration. fn run_interpreter(instructions: &[el_compiler::Bytecode]) { use el_compiler::{Bytecode, Value}; @@ -732,3 +851,88 @@ fn run_interpreter(instructions: &[el_compiler::Bytecode]) { ip += 1; } } + +/// Interpreter with debugger support — emits DebugEvents as it runs. +fn run_interpreter_debug(instructions: &[el_compiler::Bytecode], debugger: &mut el_compiler::Debugger) { + use el_compiler::{Bytecode, Value}; + let mut stack: Vec = Vec::new(); + let mut locals: std::collections::HashMap = std::collections::HashMap::new(); + let mut ip = 0usize; + + while ip < instructions.len() { + // Check if we should pause here + if debugger.should_pause(ip) { + debugger.on_pause(ip, locals.clone()); + for event in debugger.drain_events() { + match event { + el_compiler::DebugEvent::Breakpoint { offset, frame } => { + println!("[break] offset={offset} fn={} {}:{}:{}", frame.function_name, frame.source_file, frame.line, frame.col); + } + el_compiler::DebugEvent::Step { frame, locals: step_locals } => { + let var_list: Vec = step_locals.iter() + .map(|(k, v)| format!("{k}={v}")) + .collect(); + println!("[step] offset={ip} {}:{} vars=[{}]", frame.line, frame.col, var_list.join(", ")); + } + _ => {} + } + } + } + + match &instructions[ip] { + Bytecode::Push(v) => stack.push(v.clone()), + Bytecode::Pop => { stack.pop(); } + Bytecode::Add => { + let (b, a) = (stack.pop().unwrap_or(Value::Nil), stack.pop().unwrap_or(Value::Nil)); + stack.push(match (a, b) { + (Value::Int(x), Value::Int(y)) => Value::Int(x + y), + (Value::Float(x), Value::Float(y)) => Value::Float(x + y), + (Value::Str(x), Value::Str(y)) => Value::Str(x + &y), + _ => Value::Nil, + }); + } + Bytecode::StoreLocal(name) => { + let v = stack.pop().unwrap_or(Value::Nil); + locals.insert(name.clone(), v); + } + Bytecode::LoadLocal(name) => { + let v = locals.get(name).cloned().unwrap_or(Value::Nil); + stack.push(v); + } + Bytecode::Call { name, .. } => { + if name == "print" || name == "println" { + let v = stack.pop().unwrap_or(Value::Nil); + println!("{v}"); + stack.push(Value::Nil); + } + } + Bytecode::Jump(offset) => { + let new_ip = (ip as i32 + 1 + offset) as usize; + ip = new_ip; + continue; + } + Bytecode::JumpIf(offset) => { + let cond = stack.pop().unwrap_or(Value::Nil); + if matches!(cond, Value::Bool(true)) { + let new_ip = (ip as i32 + 1 + offset) as usize; + ip = new_ip; + continue; + } + } + Bytecode::JumpIfNot(offset) => { + let cond = stack.pop().unwrap_or(Value::Nil); + if !matches!(cond, Value::Bool(true)) { + let new_ip = (ip as i32 + 1 + offset) as usize; + ip = new_ip; + continue; + } + } + Bytecode::Return | Bytecode::Halt => { + debugger.pop_frame(stack.last().cloned().unwrap_or(Value::Nil)); + break; + } + _ => {} + } + ip += 1; + } +} diff --git a/crates/el-parser/src/parser.rs b/crates/el-parser/src/parser.rs index 5e6d87d..7448c63 100644 --- a/crates/el-parser/src/parser.rs +++ b/crates/el-parser/src/parser.rs @@ -84,6 +84,38 @@ impl Parser { } } + /// Like `expect_ident` but also accepts keywords as bare names. + /// Used in contexts like seed field names where `type:` must work. + fn expect_ident_or_keyword(&mut self) -> Result<(String, Span), ParseError> { + let span = self.peek_span(); + let name = match self.peek().clone() { + Token::Ident(name) => name, + // Accept any keyword as an identifier in seed field position + Token::Type => "type".to_string(), + Token::Fn => "fn".to_string(), + Token::Let => "let".to_string(), + Token::Enum => "enum".to_string(), + Token::Match => "match".to_string(), + Token::Return => "return".to_string(), + Token::Activate => "activate".to_string(), + Token::Where => "where".to_string(), + Token::Sealed => "sealed".to_string(), + Token::If => "if".to_string(), + Token::Else => "else".to_string(), + Token::For => "for".to_string(), + Token::In => "in".to_string(), + Token::Seed => "seed".to_string(), + Token::Assert => "assert".to_string(), + Token::Target => "target".to_string(), + tok => return Err(ParseError::new( + ParseErrorKind::ExpectedIdent(tok.to_string()), + span, + )), + }; + self.advance(); + Ok((name, span)) + } + fn eat(&mut self, tok: &Token) -> bool { if self.peek() == tok { self.advance(); @@ -178,7 +210,7 @@ impl Parser { let mut tier: Option = None; while !matches!(self.peek(), Token::RBrace | Token::Eof) { - let (field_name, _) = self.expect_ident()?; + let (field_name, _) = self.expect_ident_or_keyword()?; self.expect(&Token::Colon)?; match field_name.as_str() { "type" => { @@ -221,7 +253,7 @@ impl Parser { let mut weight: f32 = 1.0; while !matches!(self.peek(), Token::RBrace | Token::Eof) { - let (field_name, _) = self.expect_ident()?; + let (field_name, _) = self.expect_ident_or_keyword()?; self.expect(&Token::Colon)?; match field_name.as_str() { "from" => { diff --git a/crates/el-test/Cargo.toml b/crates/el-test/Cargo.toml new file mode 100644 index 0000000..d5705c8 --- /dev/null +++ b/crates/el-test/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "el-test" +description = "Engram language unified testing framework — unit and e2e same syntax, seed-based graph testing" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +el-lexer = { workspace = true } +el-parser = { workspace = true } +el-types = { workspace = true } +el-compiler = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/el-test/src/discovery.rs b/crates/el-test/src/discovery.rs new file mode 100644 index 0000000..de89961 --- /dev/null +++ b/crates/el-test/src/discovery.rs @@ -0,0 +1,138 @@ +//! Test discovery — finds all `test` blocks in an `.el` source file. + +use el_lexer::tokenize; +use el_parser::{parse, Stmt}; + +use crate::types::{TestCase, TestTarget}; + +/// Parse a source string and extract all `test` block definitions. +/// +/// Returns an error if the source cannot be lexed or parsed. +pub fn discover(source: &str) -> Result, String> { + let tokens = tokenize(source).map_err(|e| format!("lex error: {e}"))?; + let program = parse(tokens, source.to_string()).map_err(|e| format!("parse error: {e}"))?; + let mut cases = Vec::new(); + collect_tests(&program.stmts, &mut cases); + Ok(cases) +} + +fn collect_tests(stmts: &[Stmt], out: &mut Vec) { + for stmt in stmts { + if let Stmt::TestDef { name, target, body, .. } = stmt { + out.push(TestCase { + name: name.clone(), + target: TestTarget::from(target.clone()), + body: body.clone(), + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_discover_empty_source() { + let cases = discover("").unwrap(); + assert!(cases.is_empty()); + } + + #[test] + fn test_discover_no_tests() { + let src = r#"let x: Int = 42"#; + let cases = discover(src).unwrap(); + assert!(cases.is_empty()); + } + + #[test] + fn test_discover_single_test() { + let src = r#" +test "basic arithmetic" { + let x: Int = 6 + let y: Int = 7 +} +"#; + let cases = discover(src).unwrap(); + assert_eq!(cases.len(), 1); + assert_eq!(cases[0].name, "basic arithmetic"); + assert_eq!(cases[0].target, TestTarget::Unit); + } + + #[test] + fn test_discover_multiple_tests() { + let src = r#" +test "test one" { + let x: Int = 1 +} +test "test two" { + let y: Int = 2 +} +"#; + let cases = discover(src).unwrap(); + assert_eq!(cases.len(), 2); + assert_eq!(cases[0].name, "test one"); + assert_eq!(cases[1].name, "test two"); + } + + #[test] + fn test_discover_e2e_target() { + let src = r#" +test "production lookup" target: e2e { + let x: Int = 1 +} +"#; + let cases = discover(src).unwrap(); + assert_eq!(cases.len(), 1); + assert_eq!(cases[0].target, TestTarget::E2e); + } + + #[test] + fn test_discover_both_target() { + let src = r#" +test "dual target" target: both { + let x: Int = 1 +} +"#; + let cases = discover(src).unwrap(); + assert_eq!(cases[0].target, TestTarget::Both); + } + + #[test] + fn test_discover_default_target_is_unit() { + let src = r#"test "no target" { let x: Int = 1 }"#; + let cases = discover(src).unwrap(); + assert_eq!(cases[0].target, TestTarget::Unit); + } + + #[test] + fn test_discover_mixed_stmts() { + let src = r#" +let x: Int = 42 +test "my test" { + let y: Int = x +} +fn helper() -> Int { return 1 } +"#; + let cases = discover(src).unwrap(); + assert_eq!(cases.len(), 1); + } + + #[test] + fn test_discover_lex_error() { + let result = discover(r#""unterminated"#); + assert!(result.is_err()); + } + + #[test] + fn test_discover_body_preserved() { + let src = r#" +test "body check" { + let x: Int = 1 + let y: Int = 2 +} +"#; + let cases = discover(src).unwrap(); + assert_eq!(cases[0].body.len(), 2); + } +} diff --git a/crates/el-test/src/eval.rs b/crates/el-test/src/eval.rs new file mode 100644 index 0000000..6539124 --- /dev/null +++ b/crates/el-test/src/eval.rs @@ -0,0 +1,547 @@ +//! Test expression evaluator. +//! +//! Walks AST expressions inside a test block and produces runtime `EvalValue`s. +//! This is a lightweight direct evaluator (not the full bytecode VM) specifically +//! designed for the simple expression patterns that appear in test assertions. + +use std::collections::HashMap; + +use el_parser::{BinOp, Expr, Literal, Stmt}; + +use crate::graph::{ActivatedNode, ReasoningResult, TestGraph}; +use crate::types::AssertionResult; + +/// A runtime value in the test evaluator. +#[derive(Debug, Clone, PartialEq)] +pub enum EvalValue { + Int(i64), + Float(f64), + Str(String), + Bool(bool), + Nil, + /// A list of activated graph nodes (result of `activate`) + NodeList(Vec), + /// Result of a `reason()` call + Reasoning(ReasoningResultValue), +} + +/// Simplified reasoning result value (owns the verdict string for comparison). +#[derive(Debug, Clone, PartialEq)] +pub struct ReasoningResultValue { + pub verdict: String, + pub confidence: f32, +} + +impl EvalValue { + /// Try to get the length of a list/string value. + pub fn len(&self) -> Option { + match self { + EvalValue::NodeList(v) => Some(v.len() as i64), + EvalValue::Str(s) => Some(s.len() as i64), + _ => None, + } + } + + /// Try to index into a list. + pub fn index(&self, i: i64) -> Option { + match self { + EvalValue::NodeList(v) => { + let idx = if i < 0 { + let uidx = (-i) as usize; + v.len().checked_sub(uidx)? + } else { + i as usize + }; + let node = v.get(idx)?; + // Return the node as a struct-like value — we special-case field access + Some(EvalValue::NodeList(vec![node.clone()])) + } + _ => None, + } + } + + /// Check if this value contains a substring (for string `contains` keyword). + pub fn contains_str(&self, needle: &str) -> bool { + match self { + EvalValue::Str(s) => s.contains(needle), + EvalValue::NodeList(v) => v.iter().any(|n| n.content.contains(needle)), + _ => false, + } + } + + pub fn as_bool(&self) -> bool { + match self { + EvalValue::Bool(b) => *b, + EvalValue::Nil => false, + _ => true, + } + } +} + +impl std::fmt::Display for EvalValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EvalValue::Int(n) => write!(f, "{n}"), + EvalValue::Float(n) => write!(f, "{n}"), + EvalValue::Str(s) => write!(f, "{s}"), + EvalValue::Bool(b) => write!(f, "{b}"), + EvalValue::Nil => write!(f, "nil"), + EvalValue::NodeList(v) => write!(f, "[{} node(s)]", v.len()), + EvalValue::Reasoning(r) => write!(f, "ReasoningResult {{ verdict: {} }}", r.verdict), + } + } +} + +/// Evaluator context — holds local bindings and the graph. +pub struct Evaluator<'g> { + locals: HashMap, + graph: &'g TestGraph, +} + +impl<'g> Evaluator<'g> { + pub fn new(graph: &'g TestGraph) -> Self { + Self { + locals: HashMap::new(), + graph, + } + } + + /// Execute a statement. Returns `Ok(())` or an error string. + pub fn exec_stmt(&mut self, stmt: &Stmt) -> Result<(), String> { + match stmt { + Stmt::Let { name, value, .. } => { + let v = self.eval_expr(value)?; + self.locals.insert(name.clone(), v); + } + Stmt::Expr(expr, _) => { + self.eval_expr(expr)?; + } + Stmt::Return(..) => { + // Return from test body — treat as no-op continuation + } + // Seed statements are handled before eval in the runner + Stmt::Seed(..) | Stmt::TestDef { .. } | Stmt::Assert(..) => {} + Stmt::FnDef { .. } | Stmt::TypeDef { .. } | Stmt::EnumDef { .. } => {} + } + Ok(()) + } + + /// Evaluate an expression, returning its value. + pub fn eval_expr(&mut self, expr: &Expr) -> Result { + match expr { + Expr::Literal(lit) => Ok(match lit { + Literal::Int(n) => EvalValue::Int(*n), + Literal::Float(f) => EvalValue::Float(*f), + Literal::Str(s) => EvalValue::Str(s.clone()), + Literal::Bool(b) => EvalValue::Bool(*b), + }), + + Expr::Ident(name) => { + self.locals.get(name).cloned().ok_or_else(|| format!("undefined variable '{name}'")) + } + + Expr::BinOp { op, left, right } => { + // Special: handle ` contains ` — parsed as BinOp in the `contains` handler + // Actually `contains` is a keyword identifier parsed as Ident in the postfix context. + // We handle it as a Call pattern below. + let lv = self.eval_expr(left)?; + let rv = self.eval_expr(right)?; + self.eval_binop(op, lv, rv) + } + + Expr::Call { func, args } => { + match func.as_ref() { + // reason("hypothesis") — built-in + Expr::Ident(name) if name == "reason" => { + let arg = args.first().ok_or("reason() requires one argument")?; + let hypothesis = match self.eval_expr(arg)? { + EvalValue::Str(s) => s, + other => return Err(format!("reason() expects a string, got {other}")), + }; + let result: ReasoningResult = self.graph.reason(&hypothesis); + Ok(EvalValue::Reasoning(ReasoningResultValue { + verdict: result.verdict.to_string(), + confidence: result.confidence, + })) + } + + // results.len() — method call on a local + Expr::Field { object, field } if field == "len" => { + let obj = self.eval_expr(object)?; + match obj.len() { + Some(n) => Ok(EvalValue::Int(n)), + None => Err(format!("cannot call .len() on {obj}")), + } + } + + // results[0].content contains "needle" + // `contains` is parsed as an Ident called as a method + Expr::Field { object, field } if field == "contains" => { + let obj = self.eval_expr(object)?; + let needle_arg = args.first().ok_or("contains() requires one argument")?; + let needle = match self.eval_expr(needle_arg)? { + EvalValue::Str(s) => s, + other => return Err(format!("contains() expects a string, got {other}")), + }; + Ok(EvalValue::Bool(obj.contains_str(&needle))) + } + + _ => { + // Unknown call — return nil + Ok(EvalValue::Nil) + } + } + } + + Expr::Field { object, field } => { + let obj = self.eval_expr(object)?; + match &obj { + EvalValue::NodeList(nodes) if nodes.len() == 1 => { + let node = &nodes[0]; + match field.as_str() { + "content" => Ok(EvalValue::Str(node.content.clone())), + "node_type" => Ok(EvalValue::Str(node.node_type.clone())), + "importance" => Ok(EvalValue::Float(node.importance as f64)), + "id" => Ok(EvalValue::Str(node.id.to_string())), + other => Err(format!("no field '{other}' on ActivatedNode")), + } + } + EvalValue::Reasoning(r) => match field.as_str() { + "verdict" => Ok(EvalValue::Str(r.verdict.clone())), + "confidence" => Ok(EvalValue::Float(r.confidence as f64)), + other => Err(format!("no field '{other}' on ReasoningResult")), + }, + _ => Err(format!("cannot access field '{field}' on {obj}")), + } + } + + Expr::Index { object, index } => { + let obj = self.eval_expr(object)?; + let idx = self.eval_expr(index)?; + let i = match idx { + EvalValue::Int(n) => n, + other => return Err(format!("index must be Int, got {other}")), + }; + obj.index(i).ok_or_else(|| format!("index {i} out of bounds")) + } + + Expr::Activate { type_name, query } => { + let nodes = self.graph.activate(query, Some(type_name)); + Ok(EvalValue::NodeList(nodes)) + } + + Expr::UnaryNot(inner) => { + let v = self.eval_expr(inner)?; + Ok(EvalValue::Bool(!v.as_bool())) + } + + Expr::Block(stmts) => { + for s in stmts { + self.exec_stmt(s)?; + } + Ok(EvalValue::Nil) + } + + Expr::If { cond, then, else_ } => { + let cv = self.eval_expr(cond)?; + if cv.as_bool() { + self.eval_expr(then) + } else if let Some(e) = else_ { + self.eval_expr(e) + } else { + Ok(EvalValue::Nil) + } + } + + Expr::Array(elems) => { + // For simplicity — arrays of primitives in tests + let mut vals = Vec::new(); + for e in elems { + vals.push(self.eval_expr(e)?); + } + // Return first if all same, else nil + Ok(EvalValue::Nil) + } + + Expr::Path { segments } => { + // Enum variant reference — return as string (e.g. "Insufficient") + Ok(EvalValue::Str(segments.last().cloned().unwrap_or_default())) + } + + Expr::Match { subject, arms } => { + let sv = self.eval_expr(subject)?; + for arm in arms { + // Simplified: compare subject to pattern + let matches = match &arm.pattern { + el_parser::Pattern::Wildcard => true, + el_parser::Pattern::Literal(lit) => { + let lv = match lit { + Literal::Int(n) => EvalValue::Int(*n), + Literal::Float(f) => EvalValue::Float(*f), + Literal::Str(s) => EvalValue::Str(s.clone()), + Literal::Bool(b) => EvalValue::Bool(*b), + }; + sv == lv + } + el_parser::Pattern::Binding(name) => { + self.locals.insert(name.clone(), sv.clone()); + true + } + el_parser::Pattern::EnumVariant { variant, .. } => { + sv == EvalValue::Str(variant.clone()) + } + }; + if matches { + return self.eval_expr(&arm.body); + } + } + Ok(EvalValue::Nil) + } + + Expr::Sealed(stmts) => { + for s in stmts { + self.exec_stmt(s)?; + } + Ok(EvalValue::Nil) + } + } + } + + fn eval_binop(&self, op: &BinOp, lv: EvalValue, rv: EvalValue) -> Result { + match op { + BinOp::Add => match (lv, rv) { + (EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a + b)), + (EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a + b)), + (EvalValue::Str(a), EvalValue::Str(b)) => Ok(EvalValue::Str(a + &b)), + (a, b) => Err(format!("cannot add {a} and {b}")), + }, + BinOp::Sub => match (lv, rv) { + (EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a - b)), + (EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a - b)), + (a, b) => Err(format!("cannot subtract {a} and {b}")), + }, + BinOp::Mul => match (lv, rv) { + (EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a * b)), + (EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a * b)), + (a, b) => Err(format!("cannot multiply {a} and {b}")), + }, + BinOp::Div => match (lv, rv) { + (EvalValue::Int(a), EvalValue::Int(b)) if b != 0 => Ok(EvalValue::Int(a / b)), + (EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a / b)), + (_, EvalValue::Int(0)) => Err("division by zero".into()), + (a, b) => Err(format!("cannot divide {a} and {b}")), + }, + BinOp::Eq => Ok(EvalValue::Bool(lv == rv)), + BinOp::NotEq => Ok(EvalValue::Bool(lv != rv)), + BinOp::Lt => self.compare_ord(lv, rv, |o| o == std::cmp::Ordering::Less), + BinOp::Gt => self.compare_ord(lv, rv, |o| o == std::cmp::Ordering::Greater), + BinOp::LtEq => self.compare_ord(lv, rv, |o| o != std::cmp::Ordering::Greater), + BinOp::GtEq => self.compare_ord(lv, rv, |o| o != std::cmp::Ordering::Less), + BinOp::And => Ok(EvalValue::Bool(lv.as_bool() && rv.as_bool())), + BinOp::Or => Ok(EvalValue::Bool(lv.as_bool() || rv.as_bool())), + } + } + + fn compare_ord(&self, lv: EvalValue, rv: EvalValue, pred: F) -> Result + where + F: Fn(std::cmp::Ordering) -> bool, + { + let ord = match (&lv, &rv) { + (EvalValue::Int(a), EvalValue::Int(b)) => a.cmp(b), + (EvalValue::Float(a), EvalValue::Float(b)) => { + a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal) + } + _ => return Err(format!("cannot compare {lv} and {rv}")), + }; + Ok(EvalValue::Bool(pred(ord))) + } +} + +/// Evaluate a single `assert` expression and return an `AssertionResult`. +/// +/// The `expr_text` is the source text representation of the assertion (for +/// diagnostic output). +pub fn evaluate_assert( + eval: &mut Evaluator, + expr: &Expr, + expr_text: &str, +) -> AssertionResult { + match eval.eval_expr(expr) { + Ok(val) => { + let passed = val.as_bool(); + AssertionResult { + expression: expr_text.to_string(), + passed, + actual: Some(val.to_string()), + expected: None, + } + } + Err(_e) => AssertionResult { + expression: expr_text.to_string(), + passed: false, + actual: None, + expected: None, + } + } +} + +/// Render an expression as a human-readable string (best-effort, for display). +pub fn expr_to_text(expr: &Expr) -> String { + match expr { + Expr::Literal(Literal::Int(n)) => n.to_string(), + Expr::Literal(Literal::Float(f)) => f.to_string(), + Expr::Literal(Literal::Str(s)) => format!("\"{s}\""), + Expr::Literal(Literal::Bool(b)) => b.to_string(), + Expr::Ident(name) => name.clone(), + Expr::BinOp { op, left, right } => { + let op_str = match op { + BinOp::Add => "+", BinOp::Sub => "-", BinOp::Mul => "*", BinOp::Div => "/", + BinOp::Eq => "==", BinOp::NotEq => "!=", + BinOp::Lt => "<", BinOp::Gt => ">", BinOp::LtEq => "<=", BinOp::GtEq => ">=", + BinOp::And => "&&", BinOp::Or => "||", + }; + format!("{} {op_str} {}", expr_to_text(left), expr_to_text(right)) + } + Expr::Field { object, field } => format!("{}.{field}", expr_to_text(object)), + Expr::Call { func, args } => { + let args_str: Vec<_> = args.iter().map(expr_to_text).collect(); + format!("{}({})", expr_to_text(func), args_str.join(", ")) + } + Expr::Index { object, index } => format!("{}[{}]", expr_to_text(object), expr_to_text(index)), + Expr::Activate { type_name, query } => format!("activate {type_name} where \"{query}\""), + Expr::UnaryNot(inner) => format!("!{}", expr_to_text(inner)), + Expr::Path { segments } => segments.join("::"), + _ => "".to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::graph::TestGraph; + + fn fresh_eval(g: &TestGraph) -> Evaluator { + Evaluator::new(g) + } + + #[test] + fn test_eval_int_literal() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::Literal(Literal::Int(42)); + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(42)); + } + + #[test] + fn test_eval_bool_literal() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::Literal(Literal::Bool(true)); + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Bool(true)); + } + + #[test] + fn test_eval_string_literal() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::Literal(Literal::Str("hello".into())); + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Str("hello".into())); + } + + #[test] + fn test_eval_addition() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::BinOp { + op: BinOp::Add, + left: Box::new(Expr::Literal(Literal::Int(3))), + right: Box::new(Expr::Literal(Literal::Int(4))), + }; + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(7)); + } + + #[test] + fn test_eval_multiplication() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::BinOp { + op: BinOp::Mul, + left: Box::new(Expr::Literal(Literal::Int(6))), + right: Box::new(Expr::Literal(Literal::Int(7))), + }; + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(42)); + } + + #[test] + fn test_eval_equality_true() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::BinOp { + op: BinOp::Eq, + left: Box::new(Expr::Literal(Literal::Int(42))), + right: Box::new(Expr::Literal(Literal::Int(42))), + }; + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Bool(true)); + } + + #[test] + fn test_eval_greater_than() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::BinOp { + op: BinOp::Gt, + left: Box::new(Expr::Literal(Literal::Int(5))), + right: Box::new(Expr::Literal(Literal::Int(3))), + }; + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Bool(true)); + } + + #[test] + fn test_eval_activate_empty_graph() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::Activate { + type_name: "Customer".into(), + query: "anything".into(), + }; + let result = e.eval_expr(&expr).unwrap(); + assert!(matches!(result, EvalValue::NodeList(ref v) if v.is_empty())); + } + + #[test] + fn test_eval_activate_with_seeds() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Will Anderson, founding member", 0.9, None); + let mut e = fresh_eval(&g); + let expr = Expr::Activate { + type_name: "Customer".into(), + query: "founding".into(), + }; + let result = e.eval_expr(&expr).unwrap(); + assert!(matches!(result, EvalValue::NodeList(ref v) if v.len() == 1)); + } + + #[test] + fn test_eval_let_and_ident() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + e.locals.insert("x".into(), EvalValue::Int(10)); + let expr = Expr::Ident("x".into()); + assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(10)); + } + + #[test] + fn test_eval_reason_empty_graph() { + let g = TestGraph::new(); + let mut e = fresh_eval(&g); + let expr = Expr::Call { + func: Box::new(Expr::Ident("reason".into())), + args: vec![Expr::Literal(Literal::Str("Is there a customer?".into()))], + }; + let result = e.eval_expr(&expr).unwrap(); + match result { + EvalValue::Reasoning(r) => assert_eq!(r.verdict, "Insufficient"), + _ => panic!("expected Reasoning"), + } + } +} diff --git a/crates/el-test/src/graph.rs b/crates/el-test/src/graph.rs new file mode 100644 index 0000000..6243f0c --- /dev/null +++ b/crates/el-test/src/graph.rs @@ -0,0 +1,335 @@ +//! In-memory graph for test seeding and activation. +//! +//! `TestGraph` provides the runtime behind `seed Node { ... }`, `seed Edge { ... }`, +//! `activate T where "query"`, and `reason("hypothesis")` in test blocks. +//! +//! For **unit tests** the graph is in-memory only — no disk, no external DB. +//! For **e2e tests** the graph would delegate to a real Engram database +//! (full implementation when Engram DB Rust client is available). + +use std::collections::HashMap; + +/// Unique node identifier (simplified UUID representation). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct NodeId(pub String); + +impl NodeId { + pub fn new() -> Self { + // Simple deterministic ID for tests (real impl would use uuid crate) + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(1); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + NodeId(format!("node-{n:08x}")) + } +} + +impl Default for NodeId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for NodeId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A node in the test graph. +#[derive(Debug, Clone)] +pub struct GraphNode { + pub id: NodeId, + pub node_type: String, + pub content: String, + pub importance: f32, + #[allow(dead_code)] + pub tier: Option, +} + +/// A directed edge between two graph nodes. +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub struct GraphEdge { + pub from: NodeId, + pub to: NodeId, + pub relation: String, + pub weight: f32, +} + +/// A node returned by an `activate` query. +#[derive(Debug, Clone, PartialEq)] +pub struct ActivatedNode { + pub id: NodeId, + pub node_type: String, + pub content: String, + pub importance: f32, +} + +/// The verdict of a `reason()` call. +#[derive(Debug, Clone, PartialEq)] +pub enum ReasoningVerdict { + /// Evidence found; hypothesis is supported. + Supported, + /// Evidence found; hypothesis is contradicted. + Contradicted, + /// Insufficient evidence to evaluate hypothesis. + Insufficient, +} + +impl std::fmt::Display for ReasoningVerdict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ReasoningVerdict::Supported => write!(f, "Supported"), + ReasoningVerdict::Contradicted => write!(f, "Contradicted"), + ReasoningVerdict::Insufficient => write!(f, "Insufficient"), + } + } +} + +/// Result of a `reason()` call. +#[derive(Debug, Clone)] +pub struct ReasoningResult { + pub verdict: ReasoningVerdict, + pub evidence: Vec, + pub confidence: f32, +} + +/// The in-memory graph used during unit tests. +/// +/// Seeds accumulate nodes and edges. Activation queries search over them +/// using simple substring/keyword matching (a proxy for real embedding search). +pub struct TestGraph { + nodes: HashMap, + edges: Vec, +} + +impl TestGraph { + /// Create an empty graph (unit test mode — pure in-memory). + pub fn new() -> Self { + Self { + nodes: HashMap::new(), + edges: Vec::new(), + } + } + + /// Seed a node into the graph. Returns the node's ID. + pub fn seed_node( + &mut self, + node_type: &str, + content: &str, + importance: f32, + tier: Option<&str>, + ) -> NodeId { + let id = NodeId::new(); + let node = GraphNode { + id: id.clone(), + node_type: node_type.to_string(), + content: content.to_string(), + importance, + tier: tier.map(String::from), + }; + self.nodes.insert(id.clone(), node); + id + } + + /// Seed a directed edge between two nodes. + pub fn seed_edge(&mut self, from: NodeId, to: NodeId, relation: &str, weight: f32) { + self.edges.push(GraphEdge { + from, + to, + relation: relation.to_string(), + weight, + }); + } + + /// Activate — query nodes by type and keyword relevance. + /// + /// Matches nodes whose `node_type` equals `node_type` (case-insensitive) + /// and whose content contains any keyword from the query. + /// Results are sorted by importance descending. + pub fn activate(&self, query: &str, node_type: Option<&str>) -> Vec { + let query_lower = query.to_lowercase(); + let keywords: Vec<&str> = query_lower.split_whitespace().collect(); + + let mut results: Vec = self + .nodes + .values() + .filter(|node| { + // Type filter + if let Some(ty) = node_type { + if !node.node_type.eq_ignore_ascii_case(ty) { + return false; + } + } + // If no query keywords, match all nodes of that type + if keywords.is_empty() { + return true; + } + // Keyword relevance: content must contain at least one keyword + let content_lower = node.content.to_lowercase(); + keywords.iter().any(|kw| content_lower.contains(kw)) + }) + .map(|node| ActivatedNode { + id: node.id.clone(), + node_type: node.node_type.clone(), + content: node.content.clone(), + importance: node.importance, + }) + .collect(); + + results.sort_by(|a, b| b.importance.partial_cmp(&a.importance).unwrap_or(std::cmp::Ordering::Equal)); + results + } + + /// Reason — evaluate a hypothesis against the seeded graph. + /// + /// If the graph is empty, returns `Insufficient`. + /// If matching nodes exist, returns `Supported` with those nodes. + pub fn reason(&self, hypothesis: &str) -> ReasoningResult { + if self.nodes.is_empty() { + return ReasoningResult { + verdict: ReasoningVerdict::Insufficient, + evidence: vec![], + confidence: 0.0, + }; + } + + let evidence = self.activate(hypothesis, None); + if evidence.is_empty() { + ReasoningResult { + verdict: ReasoningVerdict::Insufficient, + evidence: vec![], + confidence: 0.0, + } + } else { + let confidence = evidence.iter().map(|n| n.importance).sum::() + / evidence.len() as f32; + ReasoningResult { + verdict: ReasoningVerdict::Supported, + evidence, + confidence, + } + } + } + + /// Number of seeded nodes. + pub fn node_count(&self) -> usize { + self.nodes.len() + } + + /// Number of seeded edges. + pub fn edge_count(&self) -> usize { + self.edges.len() + } +} + +impl Default for TestGraph { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_seed_node_returns_id() { + let mut g = TestGraph::new(); + let id = g.seed_node("Customer", "Will Anderson", 0.9, Some("Semantic")); + assert!(!id.0.is_empty()); + } + + #[test] + fn test_activate_returns_matching_nodes() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Will Anderson, founding member", 0.9, None); + g.seed_node("Customer", "Jane Smith, standard client", 0.6, None); + let results = g.activate("founding", Some("Customer")); + assert_eq!(results.len(), 1); + assert!(results[0].content.contains("Will Anderson")); + } + + #[test] + fn test_activate_empty_query_matches_all_of_type() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Alice", 0.9, None); + g.seed_node("Customer", "Bob", 0.8, None); + g.seed_node("Order", "Order #1", 0.7, None); + let results = g.activate("", Some("Customer")); + assert_eq!(results.len(), 2); + } + + #[test] + fn test_activate_wrong_type_returns_empty() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Will Anderson", 0.9, None); + let results = g.activate("Will", Some("Order")); + assert!(results.is_empty()); + } + + #[test] + fn test_activate_sorts_by_importance() { + let mut g = TestGraph::new(); + g.seed_node("Item", "low importance apple", 0.3, None); + g.seed_node("Item", "high importance apple", 0.9, None); + g.seed_node("Item", "medium importance apple", 0.6, None); + let results = g.activate("apple", Some("Item")); + assert_eq!(results.len(), 3); + assert!(results[0].importance >= results[1].importance); + assert!(results[1].importance >= results[2].importance); + } + + #[test] + fn test_activate_no_type_filter() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Will Anderson", 0.9, None); + g.seed_node("Order", "Will's order", 0.7, None); + let results = g.activate("Will", None); + assert_eq!(results.len(), 2); + } + + #[test] + fn test_reason_empty_graph_returns_insufficient() { + let g = TestGraph::new(); + let result = g.reason("Is there a customer?"); + assert_eq!(result.verdict, ReasoningVerdict::Insufficient); + assert_eq!(result.confidence, 0.0); + } + + #[test] + fn test_reason_with_matching_nodes_returns_supported() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Will Anderson, founding member", 0.9, None); + let result = g.reason("founding member"); + assert_eq!(result.verdict, ReasoningVerdict::Supported); + assert!(!result.evidence.is_empty()); + } + + #[test] + fn test_reason_no_match_returns_insufficient() { + let mut g = TestGraph::new(); + g.seed_node("Customer", "Will Anderson", 0.9, None); + let result = g.reason("quantum teleportation"); + assert_eq!(result.verdict, ReasoningVerdict::Insufficient); + } + + #[test] + fn test_seed_edge() { + let mut g = TestGraph::new(); + let cust = g.seed_node("Customer", "Will", 0.9, None); + let order = g.seed_node("Order", "Order #1", 0.7, None); + g.seed_edge(cust, order, "Purchased", 0.9); + assert_eq!(g.edge_count(), 1); + } + + #[test] + fn test_node_count() { + let mut g = TestGraph::new(); + assert_eq!(g.node_count(), 0); + g.seed_node("X", "content", 0.5, None); + assert_eq!(g.node_count(), 1); + g.seed_node("Y", "more content", 0.5, None); + assert_eq!(g.node_count(), 2); + } +} diff --git a/crates/el-test/src/lib.rs b/crates/el-test/src/lib.rs new file mode 100644 index 0000000..c4724e3 --- /dev/null +++ b/crates/el-test/src/lib.rs @@ -0,0 +1,33 @@ +//! el-test — Engram language unified testing framework. +//! +//! # Core insight +//! +//! All Engram state is graph nodes. A test seeds the graph and makes +//! assertions. Unit test = seed a few nodes in-memory. E2e test = point at +//! the real Engram database. **The test code is identical. Only the graph +//! differs.** No mocking framework. No dependency injection. One syntax. +//! +//! # Usage +//! +//! ```rust,ignore +//! use el_test::{TestRunner, TestReport}; +//! +//! let source = std::fs::read_to_string("tests.el").unwrap(); +//! let tests = el_test::discover(&source).unwrap(); +//! let runner = TestRunner::new(); +//! let report = runner.run_all(&tests, None); +//! report.print(); +//! ``` + +mod discovery; +mod eval; +mod graph; +mod report; +mod runner; +mod types; + +pub use discovery::discover; +pub use graph::TestGraph; +pub use report::TestReport; +pub use runner::TestRunner; +pub use types::{AssertionResult, TestCase, TestResult, TestStatus, TestTarget}; diff --git a/crates/el-test/src/report.rs b/crates/el-test/src/report.rs new file mode 100644 index 0000000..1353482 --- /dev/null +++ b/crates/el-test/src/report.rs @@ -0,0 +1,319 @@ +//! Test report formatting — human-readable, JSON, and JUnit XML output. + +use crate::types::{TestResult, TestStatus}; + +/// Aggregated report for a set of test runs. +pub struct TestReport { + pub total: u32, + pub passed: u32, + pub failed: u32, + pub skipped: u32, + pub errors: u32, + pub duration_ms: u64, + pub results: Vec, +} + +impl TestReport { + /// Build a report from a slice of individual test results. + pub fn from_results(results: Vec) -> Self { + let total = results.len() as u32; + let passed = results.iter().filter(|r| r.status == TestStatus::Pass).count() as u32; + let failed = results.iter().filter(|r| r.status == TestStatus::Fail).count() as u32; + let skipped = results.iter().filter(|r| r.status == TestStatus::Skip).count() as u32; + let errors = results.iter().filter(|r| r.status == TestStatus::Error).count() as u32; + let duration_ms = results.iter().map(|r| r.duration_ms).sum(); + Self { total, passed, failed, skipped, errors, duration_ms, results } + } + + /// Print a human-readable summary to stdout. + pub fn print(&self) { + let target_label = format!("({}ms total)", self.duration_ms); + + println!("\nRunning {} tests...\n", self.total); + + for r in &self.results { + let icon = match r.status { + TestStatus::Pass => " ok ", + TestStatus::Fail => " FAIL ", + TestStatus::Skip => " SKIP ", + TestStatus::Error => "ERROR ", + }; + println!(" [{icon}] {} ({}ms)", r.name, r.duration_ms); + + // Show failing assertions + if r.status == TestStatus::Fail { + for (i, a) in r.assertions.iter().enumerate() { + if !a.passed { + println!(" assert {}", a.expression); + if let Some(actual) = &a.actual { + println!(" actual: {actual}"); + } + if let Some(expected) = &a.expected { + println!(" expected: {expected}"); + } + println!(" at assertion {}", i + 1); + } + } + } + if let Some(err) = &r.error { + println!(" error: {err}"); + } + } + + println!("\nResults: {} passed, {} failed, {} skipped {}", self.passed, self.failed, self.skipped, target_label); + if self.errors > 0 { + println!(" ({} error(s) — see above)", self.errors); + } + } + + /// Serialize to JSON. + pub fn to_json(&self) -> String { + let results: Vec = self + .results + .iter() + .map(|r| { + let assertions: Vec = r + .assertions + .iter() + .map(|a| { + serde_json::json!({ + "expression": a.expression, + "passed": a.passed, + "actual": a.actual, + "expected": a.expected, + }) + }) + .collect(); + serde_json::json!({ + "name": r.name, + "target": r.target.to_string(), + "status": r.status.to_string(), + "duration_ms": r.duration_ms, + "assertions": assertions, + "error": r.error, + }) + }) + .collect(); + + let report = serde_json::json!({ + "total": self.total, + "passed": self.passed, + "failed": self.failed, + "skipped": self.skipped, + "errors": self.errors, + "duration_ms": self.duration_ms, + "results": results, + }); + + serde_json::to_string_pretty(&report).unwrap_or_else(|_| "{}".to_string()) + } + + /// Serialize to JUnit XML (for CI integration). + pub fn to_junit_xml(&self) -> String { + let mut xml = String::new(); + xml.push_str("\n"); + xml.push_str(&format!( + "\n", + self.total, + self.failed, + self.errors, + self.skipped, + self.duration_ms as f64 / 1000.0, + )); + + for r in &self.results { + let classname = "el_test"; + let time = r.duration_ms as f64 / 1000.0; + let name_escaped = xml_escape(&r.name); + + match r.status { + TestStatus::Pass => { + xml.push_str(&format!( + " \n" + )); + } + TestStatus::Fail => { + xml.push_str(&format!( + " \n" + )); + for a in r.assertions.iter().filter(|a| !a.passed) { + let msg = xml_escape(&a.expression); + let details = match (&a.actual, &a.expected) { + (Some(act), Some(exp)) => format!("actual: {act}, expected: {exp}"), + (Some(act), None) => format!("actual: {act}"), + _ => "assertion failed".to_string(), + }; + let details_esc = xml_escape(&details); + xml.push_str(&format!( + " {details_esc}\n" + )); + } + xml.push_str(" \n"); + } + TestStatus::Skip => { + xml.push_str(&format!( + " \n \n \n" + )); + } + TestStatus::Error => { + let err_msg = xml_escape(r.error.as_deref().unwrap_or("unknown error")); + xml.push_str(&format!( + " \n \n \n" + )); + } + } + } + + xml.push_str("\n"); + xml + } + + /// Whether the overall test run passed (no failures or errors). + pub fn is_pass(&self) -> bool { + self.failed == 0 && self.errors == 0 + } +} + +fn xml_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{AssertionResult, TestTarget}; + + fn make_pass(name: &str) -> TestResult { + TestResult { + name: name.to_string(), + target: TestTarget::Unit, + status: TestStatus::Pass, + duration_ms: 5, + assertions: vec![AssertionResult { + expression: "x == 42".into(), + passed: true, + actual: Some("true".into()), + expected: None, + }], + error: None, + } + } + + fn make_fail(name: &str) -> TestResult { + TestResult { + name: name.to_string(), + target: TestTarget::Unit, + status: TestStatus::Fail, + duration_ms: 3, + assertions: vec![AssertionResult { + expression: "x == 99".into(), + passed: false, + actual: Some("42".into()), + expected: Some("99".into()), + }], + error: None, + } + } + + fn make_skip(name: &str) -> TestResult { + TestResult { + name: name.to_string(), + target: TestTarget::E2e, + status: TestStatus::Skip, + duration_ms: 0, + assertions: vec![], + error: Some("ENGRAM_URL not set".into()), + } + } + + #[test] + fn test_report_counts() { + let report = TestReport::from_results(vec![ + make_pass("a"), + make_fail("b"), + make_skip("c"), + ]); + assert_eq!(report.total, 3); + assert_eq!(report.passed, 1); + assert_eq!(report.failed, 1); + assert_eq!(report.skipped, 1); + } + + #[test] + fn test_report_is_pass() { + let report = TestReport::from_results(vec![make_pass("a"), make_pass("b")]); + assert!(report.is_pass()); + } + + #[test] + fn test_report_is_not_pass_on_fail() { + let report = TestReport::from_results(vec![make_pass("a"), make_fail("b")]); + assert!(!report.is_pass()); + } + + #[test] + fn test_to_json_valid() { + let report = TestReport::from_results(vec![make_pass("test")]); + let json = report.to_json(); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!(parsed["total"], 1); + assert_eq!(parsed["passed"], 1); + } + + #[test] + fn test_to_json_contains_results() { + let report = TestReport::from_results(vec![make_pass("hello")]); + let json = report.to_json(); + assert!(json.contains("hello")); + } + + #[test] + fn test_to_junit_xml_valid() { + let report = TestReport::from_results(vec![make_pass("a"), make_fail("b")]); + let xml = report.to_junit_xml(); + assert!(xml.starts_with("")); + } + + #[test] + fn test_to_junit_xml_pass_testcase() { + let report = TestReport::from_results(vec![make_pass("arithmetic")]); + let xml = report.to_junit_xml(); + assert!(xml.contains("arithmetic")); + assert!(!xml.contains(" d \"e\" 'f'"); + assert!(escaped.contains("<")); + assert!(escaped.contains("&")); + assert!(escaped.contains(">")); + assert!(escaped.contains(""")); + } +} diff --git a/crates/el-test/src/runner.rs b/crates/el-test/src/runner.rs new file mode 100644 index 0000000..8cb0a4f --- /dev/null +++ b/crates/el-test/src/runner.rs @@ -0,0 +1,382 @@ +//! Test runner — executes test cases and produces results. + +use std::time::Instant; + +use el_parser::{SeedStmt, Stmt}; + +use crate::eval::{evaluate_assert, expr_to_text, Evaluator}; +use crate::graph::TestGraph; +use crate::types::{AssertionResult, TestCase, TestResult, TestStatus, TestTarget}; + +/// Runs test cases and collects results. +pub struct TestRunner; + +impl TestRunner { + pub fn new() -> Self { + Self + } + + /// Run all tests. E2e tests are skipped if `engram_url` is `None`. + pub fn run_all(&self, tests: &[TestCase], engram_url: Option<&str>) -> Vec { + let mut results = Vec::new(); + for test in tests { + match &test.target { + TestTarget::Unit => { + results.push(self.run_unit_test(test)); + } + TestTarget::E2e => { + if let Some(url) = engram_url { + results.push(self.run_e2e_test(test, url)); + } else { + results.push(TestResult { + name: test.name.clone(), + target: TestTarget::E2e, + status: TestStatus::Skip, + duration_ms: 0, + assertions: vec![], + error: Some("ENGRAM_URL not set; skipping e2e test".into()), + }); + } + } + TestTarget::Both => { + results.push(self.run_unit_test(test)); + if let Some(url) = engram_url { + results.push(self.run_e2e_test(test, url)); + } else { + results.push(TestResult { + name: format!("{} (e2e)", test.name), + target: TestTarget::E2e, + status: TestStatus::Skip, + duration_ms: 0, + assertions: vec![], + error: Some("ENGRAM_URL not set; skipping e2e test".into()), + }); + } + } + } + } + results + } + + /// Run only unit tests. + pub fn run_unit(&self, tests: &[TestCase]) -> Vec { + tests + .iter() + .filter(|t| matches!(t.target, TestTarget::Unit | TestTarget::Both)) + .map(|t| self.run_unit_test(t)) + .collect() + } + + /// Run only e2e tests. + pub fn run_e2e<'a>(&self, tests: &'a [TestCase], engram_url: &str) -> Vec { + tests + .iter() + .filter(|t| matches!(t.target, TestTarget::E2e | TestTarget::Both)) + .map(|t| self.run_e2e_test(t, engram_url)) + .collect() + } + + /// Run a single test against an in-memory graph. + pub fn run_one(&self, test: &TestCase, engram_url: Option<&str>) -> TestResult { + match (&test.target, engram_url) { + (TestTarget::E2e, Some(url)) => self.run_e2e_test(test, url), + (TestTarget::E2e, None) => TestResult { + name: test.name.clone(), + target: TestTarget::E2e, + status: TestStatus::Skip, + duration_ms: 0, + assertions: vec![], + error: Some("ENGRAM_URL not set; skipping e2e test".into()), + }, + _ => self.run_unit_test(test), + } + } + + // ── Private ─────────────────────────────────────────────────────────────── + + fn run_unit_test(&self, test: &TestCase) -> TestResult { + let start = Instant::now(); + let mut graph = TestGraph::new(); + + // Apply seed statements first + for stmt in &test.body { + if let Stmt::Seed(seed, _) = stmt { + apply_seed(&mut graph, seed); + } + } + + self.execute_test(test, &graph, TestTarget::Unit, start) + } + + fn run_e2e_test(&self, test: &TestCase, _engram_url: &str) -> TestResult { + let start = Instant::now(); + // For e2e, we still use an in-memory graph for now (real DB client TBD). + // The distinction is that e2e tests skip the seed step (they use real data). + let graph = TestGraph::new(); + self.execute_test(test, &graph, TestTarget::E2e, start) + } + + fn execute_test( + &self, + test: &TestCase, + graph: &TestGraph, + target: TestTarget, + start: Instant, + ) -> TestResult { + let mut eval = Evaluator::new(graph); + let mut assertions: Vec = Vec::new(); + let mut error: Option = None; + + for stmt in &test.body { + match stmt { + Stmt::Seed(..) => { + // Already processed before eval + } + Stmt::Assert(expr, _) => { + let text = expr_to_text(expr); + let result = evaluate_assert(&mut eval, expr, &text); + assertions.push(result); + } + other => { + if let Err(e) = eval.exec_stmt(other) { + error = Some(e); + break; + } + } + } + } + + let duration_ms = start.elapsed().as_millis() as u64; + let all_passed = assertions.iter().all(|a| a.passed); + let status = if error.is_some() { + TestStatus::Error + } else if all_passed { + TestStatus::Pass + } else { + TestStatus::Fail + }; + + TestResult { + name: test.name.clone(), + target, + status, + duration_ms, + assertions, + error, + } + } +} + +impl Default for TestRunner { + fn default() -> Self { + Self::new() + } +} + +fn apply_seed(graph: &mut TestGraph, seed: &SeedStmt) { + match seed { + SeedStmt::Node { node_type, content, importance, tier } => { + graph.seed_node(node_type, content, *importance, tier.as_deref()); + } + SeedStmt::Edge { from, to, relation, weight } => { + // For edges we use string IDs — in a real impl these would be looked + // up from the graph's node registry. We create placeholder node IDs. + use crate::graph::NodeId; + graph.seed_edge( + NodeId(from.clone()), + NodeId(to.clone()), + relation, + *weight, + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use el_parser::{Expr, Literal, BinOp, Stmt}; + use el_lexer::Span; + + fn dummy_span() -> Span { + Span::new(0, 0, 1, 1) + } + + fn make_assert(expr: Expr) -> Stmt { + Stmt::Assert(expr, dummy_span()) + } + + fn make_let(name: &str, expr: Expr) -> Stmt { + Stmt::Let { + name: name.to_string(), + type_ann: None, + value: expr, + span: dummy_span(), + } + } + + fn int_lit(n: i64) -> Expr { + Expr::Literal(Literal::Int(n)) + } + + fn str_lit(s: &str) -> Expr { + Expr::Literal(Literal::Str(s.to_string())) + } + + fn bool_lit(b: bool) -> Expr { + Expr::Literal(Literal::Bool(b)) + } + + fn binop(op: BinOp, l: Expr, r: Expr) -> Expr { + Expr::BinOp { op, left: Box::new(l), right: Box::new(r) } + } + + fn test_case(name: &str, body: Vec) -> TestCase { + TestCase { + name: name.to_string(), + target: TestTarget::Unit, + body, + } + } + + #[test] + fn test_runner_pass() { + let runner = TestRunner::new(); + let tc = test_case("arithmetic", vec![ + make_let("x", int_lit(6)), + make_let("y", int_lit(7)), + make_let("result", binop(BinOp::Mul, Expr::Ident("x".into()), Expr::Ident("y".into()))), + make_assert(binop(BinOp::Eq, Expr::Ident("result".into()), int_lit(42))), + ]); + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Pass); + assert!(result.assertions[0].passed); + } + + #[test] + fn test_runner_fail() { + let runner = TestRunner::new(); + let tc = test_case("failing", vec![ + make_assert(binop(BinOp::Eq, int_lit(1), int_lit(2))), + ]); + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Fail); + assert!(!result.assertions[0].passed); + } + + #[test] + fn test_runner_multiple_assertions_partial_fail() { + let runner = TestRunner::new(); + let tc = test_case("partial", vec![ + make_assert(bool_lit(true)), + make_assert(binop(BinOp::Eq, int_lit(1), int_lit(2))), // fails + make_assert(bool_lit(true)), + ]); + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Fail); + assert!(result.assertions[0].passed); + assert!(!result.assertions[1].passed); + assert!(result.assertions[2].passed); + } + + #[test] + fn test_runner_e2e_skip_without_url() { + let runner = TestRunner::new(); + let tc = TestCase { + name: "e2e test".to_string(), + target: TestTarget::E2e, + body: vec![], + }; + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Skip); + } + + #[test] + fn test_runner_run_all_skips_e2e() { + let runner = TestRunner::new(); + let tests = vec![ + TestCase { name: "unit".into(), target: TestTarget::Unit, body: vec![] }, + TestCase { name: "e2e".into(), target: TestTarget::E2e, body: vec![] }, + ]; + let results = runner.run_all(&tests, None); + assert_eq!(results.len(), 2); + assert_eq!(results[0].status, TestStatus::Pass); // empty = pass + assert_eq!(results[1].status, TestStatus::Skip); + } + + #[test] + fn test_runner_run_unit_filters_unit_only() { + let runner = TestRunner::new(); + let tests = vec![ + TestCase { name: "unit".into(), target: TestTarget::Unit, body: vec![] }, + TestCase { name: "e2e".into(), target: TestTarget::E2e, body: vec![] }, + ]; + let results = runner.run_unit(&tests); + assert_eq!(results.len(), 1); + assert_eq!(results[0].name, "unit"); + } + + #[test] + fn test_runner_empty_test_passes() { + let runner = TestRunner::new(); + let tc = test_case("empty", vec![]); + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Pass); + } + + #[test] + fn test_runner_with_seed_and_activate() { + use el_parser::SeedStmt; + let runner = TestRunner::new(); + let seed = Stmt::Seed( + SeedStmt::Node { + node_type: "Customer".into(), + content: "Will Anderson, founding member".into(), + importance: 0.9, + tier: Some("Semantic".into()), + }, + dummy_span(), + ); + // activate Customer where "founding" + let activate = Expr::Activate { + type_name: "Customer".into(), + query: "founding".into(), + }; + let let_results = make_let("results", activate); + // assert results.len() > 0 => assert results.len() > 0 + // We'll call .len() via a Call to Field + let len_call = Expr::Call { + func: Box::new(Expr::Field { + object: Box::new(Expr::Ident("results".into())), + field: "len".into(), + }), + args: vec![], + }; + let assert_len = make_assert(binop(BinOp::Gt, len_call, int_lit(0))); + let tc = test_case("seed and activate", vec![seed, let_results, assert_len]); + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Pass); + } + + #[test] + fn test_runner_empty_graph_activate_returns_empty() { + let runner = TestRunner::new(); + // No seeds — activate should return empty list + let activate = Expr::Activate { + type_name: "Customer".into(), + query: "anything".into(), + }; + let let_results = make_let("results", activate); + let len_call = Expr::Call { + func: Box::new(Expr::Field { + object: Box::new(Expr::Ident("results".into())), + field: "len".into(), + }), + args: vec![], + }; + let assert_zero = make_assert(binop(BinOp::Eq, len_call, int_lit(0))); + let tc = test_case("empty graph", vec![let_results, assert_zero]); + let result = runner.run_one(&tc, None); + assert_eq!(result.status, TestStatus::Pass); + } +} diff --git a/crates/el-test/src/types.rs b/crates/el-test/src/types.rs new file mode 100644 index 0000000..73ae06f --- /dev/null +++ b/crates/el-test/src/types.rs @@ -0,0 +1,84 @@ +//! Core data types for the testing framework. + +use el_parser::Stmt; + +/// Which graph a test should execute against. +#[derive(Debug, Clone, PartialEq)] +pub enum TestTarget { + /// In-memory graph — default, zero external dependencies. + Unit, + /// Real Engram database pointed at by `ENGRAM_URL` / `ENGRAM_DB_PATH`. + E2e, + /// Run against both unit (in-memory) and e2e (real DB). + Both, +} + +impl std::fmt::Display for TestTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TestTarget::Unit => write!(f, "unit"), + TestTarget::E2e => write!(f, "e2e"), + TestTarget::Both => write!(f, "both"), + } + } +} + +impl From for TestTarget { + fn from(t: el_parser::TestTarget) -> Self { + match t { + el_parser::TestTarget::Unit => TestTarget::Unit, + el_parser::TestTarget::E2e => TestTarget::E2e, + el_parser::TestTarget::Both => TestTarget::Both, + } + } +} + +/// A single test case extracted from an `.el` source file. +pub struct TestCase { + pub name: String, + pub target: TestTarget, + /// The body statements of the test block. + pub body: Vec, +} + +/// Status of a single test run. +#[derive(Debug, Clone, PartialEq)] +pub enum TestStatus { + Pass, + Fail, + /// Skipped because the required target is not available. + Skip, + /// Unexpected runtime error (not an assertion failure). + Error, +} + +impl std::fmt::Display for TestStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TestStatus::Pass => write!(f, "pass"), + TestStatus::Fail => write!(f, "fail"), + TestStatus::Skip => write!(f, "skip"), + TestStatus::Error => write!(f, "error"), + } + } +} + +/// Result of evaluating a single `assert` statement. +#[derive(Debug, Clone)] +pub struct AssertionResult { + /// The `assert` expression as source text. + pub expression: String, + pub passed: bool, + pub actual: Option, + pub expected: Option, +} + +/// The full result of one test execution. +pub struct TestResult { + pub name: String, + pub target: TestTarget, + pub status: TestStatus, + pub duration_ms: u64, + pub assertions: Vec, + pub error: Option, +} diff --git a/examples/hello-project/src/tests.el b/examples/hello-project/src/tests.el new file mode 100644 index 0000000..7555f77 --- /dev/null +++ b/examples/hello-project/src/tests.el @@ -0,0 +1,57 @@ +// Engram language — example test file +// Run with: el test-file examples/hello-project/src/tests.el + +test "basic arithmetic" { + let x: Int = 6 + let y: Int = 7 + let result: Int = x * y + assert result == 42 +} + +test "string operations" { + let greeting: String = "Hello" + let name: String = "Engram" + let full: String = greeting + ", " + name + assert full == "Hello, Engram" +} + +test "type graph has Point type" { + seed Node { + type: "Point" + content: "A 2D coordinate" + importance: 0.8 + tier: Semantic + } + + let results = activate Point where "coordinate" + assert results.len() > 0 +} + +test "empty graph" { + let results = activate Customer where "anything" + assert results.len() == 0 +} + +test "empty graph returns insufficient" { + let result = reason("Is there a customer named Will?") + assert result.verdict == "Insufficient" +} + +test "seeded graph reasoning" { + seed Node { + type: "Customer" + content: "Will Anderson, founding member" + importance: 0.9 + tier: Semantic + } + + let result = reason("founding member") + assert result.verdict == "Supported" +} + +test "production customer lookup" target: e2e { + // No seeds — uses the real Engram database + // (skipped when ENGRAM_URL is not set) + let results = activate Customer where "founding member" + assert results.len() > 0 +}