neuron-technologies/el
2
0
Fork 0
Code Issues 1 Pull Requests 6 Actions Packages Projects Releases 5 Wiki Activity

feat: unified testing framework — unit and e2e same syntax, seed-based graph testing, debugger infrastructure

Browse Source 
- New crate el-test: test discovery, in-memory graph seeding, assertion evaluator, TestRunner, TestReport with human/JSON/JUnit XML output
- New keywords: test, seed, assert, target — fully integrated into lexer, parser, codegen, type-checker
- Parser extensions: TestDef, SeedStmt, Assert AST nodes; seed blocks handle type: as field name (keyword-as-ident in seed context)
- Debugger: DebugEvent, Debugger, StepMode, StackFrame in el-compiler — breakpoints, step-over, step-into, step-out
- CLI: el test-file <file.el> runs tests; el test integrates with project; el debug attaches debugger; --output json|junit for CI
- 52 new tests in el-test covering discovery, graph seeding, assertion evaluation, pass/fail/error/skip, report generation, JUnit XML
- Example: examples/hello-project/src/tests.el — 6 unit tests pass, 1 e2e test correctly skipped without ENGRAM_URL
This commit is contained in:
Will Anderson
2026-04-27 19:11:59 -05:00
parent 48b72843e1
commit 0a36a454f9
14 changed files with 2177 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+14
View File
@@ -294,6 +294,7 @@ dependencies = [
"el-parser",
"el-registry",
"el-seal",
"el-test",
"el-types",
"thiserror 2.0.18",
"tokio",
@@ -385,6 +386,19 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "el-test"
version = "0.1.0"
dependencies = [
"el-compiler",
"el-lexer",
"el-parser",
"el-types",
"serde",
"serde_json",
"thiserror 2.0.18",
]
[[package]]
name = "el-types"
version = "0.1.0"
Cargo.toml
+2
View File
@@ -8,6 +8,7 @@ members = [
"crates/el-manifest",
"crates/el-registry",
"crates/el-build",
"crates/el-test",
"bin/el",
]
resolver = "2"
@@ -28,6 +29,7 @@ el-seal = { path = "crates/el-seal" }
el-manifest = { path = "crates/el-manifest" }
el-registry = { path = "crates/el-registry" }
el-build = { path = "crates/el-build" }
el-test = { path = "crates/el-test" }
# Engram crypto (path dep — the sealed target depends on it)
engram-crypto = { path = "../engram/crates/engram-crypto" }
bin/el/Cargo.toml
+1
View File
@@ -18,6 +18,7 @@ el-seal = { workspace = true }
el-manifest = { workspace = true }
el-registry = { workspace = true }
el-build = { workspace = true }
el-test = { workspace = true }
clap = { workspace = true }
thiserror = { workspace = true }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
bin/el/src/main.rs
+216 -12
View File
@@ -32,6 +32,7 @@ use std::path::PathBuf;
use clap::{Parser, Subcommand};
use el_build::BuildSystem;
use el_compiler::{Compiler, CompilerOptions, Target};
use el_test;
use el_manifest::{BuildTarget, Manifest};
use el_seal::{seal as seal_fn, unseal as unseal_fn, SealedArtifact, DeploymentBinding, SealAlgorithm, SealConfig};
@@ -101,12 +102,50 @@ enum Command {
manifest: Option<PathBuf>,
},
/// Run all tests.
/// Run tests in the current project (reads el.toml).
Test {
/// Only run tests matching this name (substring match).
filter: Option<String>,
/// Also run e2e tests (requires ENGRAM_URL or ENGRAM_DB_PATH).
#[arg(long)]
e2e: bool,
/// Run unit AND e2e tests.
#[arg(long)]
all: bool,
/// Output format: human (default) | json | junit.
#[arg(long, default_value = "human")]
output: String,
/// Path to the project manifest (default: el.toml).
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Run tests from a single .el file (no el.toml required).
TestFile {
/// Source file containing test blocks (*.el).
file: PathBuf,
/// Only run tests matching this name (substring match).
filter: Option<String>,
/// Also run e2e tests (requires ENGRAM_URL or ENGRAM_DB_PATH).
#[arg(long)]
e2e: bool,
/// Run unit AND e2e tests.
#[arg(long)]
all: bool,
/// Output format: human (default) | json | junit.
#[arg(long, default_value = "human")]
output: String,
},
/// Run an Engram source file with the step-debugger attached.
Debug {
/// Source file (*.el).
file: PathBuf,
/// Set a breakpoint at line N.
#[arg(long, value_name = "LINE")]
r#break: Option<u32>,
},
/// Type-check source files without producing artifacts.
Check {
#[arg(long)]
@@ -268,20 +307,51 @@ async fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
run_interpreter(&instructions);
}
Command::Test { manifest } => {
Command::Test { filter, e2e, all, output, manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
let report = bs.test().await?;
println!(
"test: {} passed, {} failed (total {})",
report.passed, report.failed, report.total
);
for f in &report.failures {
eprintln!(" FAIL: {f}");
}
if !report.success() {
std::process::exit(1);
// Find all .el source files in the project
let entry = bs.manifest.build.entry.clone();
let entry_path = manifest_path.parent().unwrap_or(std::path::Path::new(".")).join(&entry);
let source = std::fs::read_to_string(&entry_path)
.map_err(|e| format!("cannot read {}: {e}", entry_path.display()))?;
run_tests_from_source(&source, filter.as_deref(), e2e, all, &output)?;
}
Command::TestFile { file, filter, e2e, all, output } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
run_tests_from_source(&source, filter.as_deref(), e2e, all, &output)?;
}
Command::Debug { file, r#break } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
let opts = CompilerOptions {
target: Target::Debug,
source_path: file.clone(),
..Default::default()
};
let compiled = Compiler::compile(&source, opts)?;
let instructions = el_compiler::Bytecode::deserialize_all(&compiled.artifact)
.unwrap_or_default();
let mut debugger = el_compiler::Debugger::new();
if let Some(line) = r#break {
// Convert line number to a bytecode offset approximation.
// In a full implementation this would use the source map.
// For now we use the line number directly as a placeholder offset.
debugger.add_breakpoint(line as usize);
println!("debugger: breakpoint set at line {line}");
} else {
println!("debugger: breaking on first instruction");
}
println!("debugger: running {} ({} instructions)", file.display(), instructions.len());
run_interpreter_debug(&instructions, &mut debugger);
}
Command::Check { manifest } => {
@@ -628,6 +698,55 @@ fn build_seal_config() -> Result<SealConfig, String> {
})
}
/// Discover tests in source, filter, run, and print results.
fn run_tests_from_source(
source: &str,
filter: Option<&str>,
e2e: bool,
all: bool,
output_fmt: &str,
) -> Result<(), Box<dyn std::error::Error>> {
use el_test::{TestReport, TestRunner};
let mut tests = el_test::discover(source)?;
// Apply name filter
if let Some(f) = filter {
tests.retain(|t| t.name.contains(f));
}
if tests.is_empty() {
println!("no tests found");
return Ok(());
}
let engram_url = std::env::var("ENGRAM_URL").ok().or_else(|| std::env::var("ENGRAM_DB_PATH").ok());
let url_ref = engram_url.as_deref();
let runner = TestRunner::new();
let results = if all {
runner.run_all(&tests, url_ref)
} else if e2e {
runner.run_e2e(&tests, url_ref.unwrap_or(""))
} else {
runner.run_unit(&tests)
};
let report = TestReport::from_results(results);
match output_fmt {
"json" => println!("{}", report.to_json()),
"junit" => println!("{}", report.to_junit_xml()),
_ => report.print(),
}
if !report.is_pass() {
std::process::exit(1);
}
Ok(())
}
/// Minimal interpreter for demonstration.
fn run_interpreter(instructions: &[el_compiler::Bytecode]) {
use el_compiler::{Bytecode, Value};
@@ -732,3 +851,88 @@ fn run_interpreter(instructions: &[el_compiler::Bytecode]) {
ip += 1;
}
}
/// Interpreter with debugger support — emits DebugEvents as it runs.
fn run_interpreter_debug(instructions: &[el_compiler::Bytecode], debugger: &mut el_compiler::Debugger) {
use el_compiler::{Bytecode, Value};
let mut stack: Vec<Value> = Vec::new();
let mut locals: std::collections::HashMap<String, Value> = std::collections::HashMap::new();
let mut ip = 0usize;
while ip < instructions.len() {
// Check if we should pause here
if debugger.should_pause(ip) {
debugger.on_pause(ip, locals.clone());
for event in debugger.drain_events() {
match event {
el_compiler::DebugEvent::Breakpoint { offset, frame } => {
println!("[break] offset={offset} fn={} {}:{}:{}", frame.function_name, frame.source_file, frame.line, frame.col);
}
el_compiler::DebugEvent::Step { frame, locals: step_locals } => {
let var_list: Vec<String> = step_locals.iter()
.map(|(k, v)| format!("{k}={v}"))
.collect();
println!("[step] offset={ip} {}:{} vars=[{}]", frame.line, frame.col, var_list.join(", "));
}
_ => {}
}
}
}
match &instructions[ip] {
Bytecode::Push(v) => stack.push(v.clone()),
Bytecode::Pop => { stack.pop(); }
Bytecode::Add => {
let (b, a) = (stack.pop().unwrap_or(Value::Nil), stack.pop().unwrap_or(Value::Nil));
stack.push(match (a, b) {
(Value::Int(x), Value::Int(y)) => Value::Int(x + y),
(Value::Float(x), Value::Float(y)) => Value::Float(x + y),
(Value::Str(x), Value::Str(y)) => Value::Str(x + &y),
_ => Value::Nil,
});
}
Bytecode::StoreLocal(name) => {
let v = stack.pop().unwrap_or(Value::Nil);
locals.insert(name.clone(), v);
}
Bytecode::LoadLocal(name) => {
let v = locals.get(name).cloned().unwrap_or(Value::Nil);
stack.push(v);
}
Bytecode::Call { name, .. } => {
if name == "print" || name == "println" {
let v = stack.pop().unwrap_or(Value::Nil);
println!("{v}");
stack.push(Value::Nil);
}
}
Bytecode::Jump(offset) => {
let new_ip = (ip as i32 + 1 + offset) as usize;
ip = new_ip;
continue;
}
Bytecode::JumpIf(offset) => {
let cond = stack.pop().unwrap_or(Value::Nil);
if matches!(cond, Value::Bool(true)) {
let new_ip = (ip as i32 + 1 + offset) as usize;
ip = new_ip;
continue;
}
}
Bytecode::JumpIfNot(offset) => {
let cond = stack.pop().unwrap_or(Value::Nil);
if !matches!(cond, Value::Bool(true)) {
let new_ip = (ip as i32 + 1 + offset) as usize;
ip = new_ip;
continue;
}
}
Bytecode::Return | Bytecode::Halt => {
debugger.pop_frame(stack.last().cloned().unwrap_or(Value::Nil));
break;
}
_ => {}
}
ip += 1;
}
}
crates/el-parser/src/parser.rs
+34 -2
View File
@@ -84,6 +84,38 @@ impl Parser {
}
}
/// Like `expect_ident` but also accepts keywords as bare names.
/// Used in contexts like seed field names where `type:` must work.
fn expect_ident_or_keyword(&mut self) -> Result<(String, Span), ParseError> {
let span = self.peek_span();
let name = match self.peek().clone() {
Token::Ident(name) => name,
// Accept any keyword as an identifier in seed field position
Token::Type => "type".to_string(),
Token::Fn => "fn".to_string(),
Token::Let => "let".to_string(),
Token::Enum => "enum".to_string(),
Token::Match => "match".to_string(),
Token::Return => "return".to_string(),
Token::Activate => "activate".to_string(),
Token::Where => "where".to_string(),
Token::Sealed => "sealed".to_string(),
Token::If => "if".to_string(),
Token::Else => "else".to_string(),
Token::For => "for".to_string(),
Token::In => "in".to_string(),
Token::Seed => "seed".to_string(),
Token::Assert => "assert".to_string(),
Token::Target => "target".to_string(),
tok => return Err(ParseError::new(
ParseErrorKind::ExpectedIdent(tok.to_string()),
span,
)),
};
self.advance();
Ok((name, span))
}
fn eat(&mut self, tok: &Token) -> bool {
if self.peek() == tok {
self.advance();
@@ -178,7 +210,7 @@ impl Parser {
let mut tier: Option<String> = None;
while !matches!(self.peek(), Token::RBrace | Token::Eof) {
let (field_name, _) = self.expect_ident()?;
let (field_name, _) = self.expect_ident_or_keyword()?;
self.expect(&Token::Colon)?;
match field_name.as_str() {
"type" => {
@@ -221,7 +253,7 @@ impl Parser {
let mut weight: f32 = 1.0;
while !matches!(self.peek(), Token::RBrace | Token::Eof) {
let (field_name, _) = self.expect_ident()?;
let (field_name, _) = self.expect_ident_or_keyword()?;
self.expect(&Token::Colon)?;
match field_name.as_str() {
"from" => {
crates/el-test/Cargo.toml
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "el-test"
description = "Engram language unified testing framework — unit and e2e same syntax, seed-based graph testing"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
el-lexer = { workspace = true }
el-parser = { workspace = true }
el-types = { workspace = true }
el-compiler = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
crates/el-test/src/discovery.rs
+138
View File
@@ -0,0 +1,138 @@
//! Test discovery — finds all `test` blocks in an `.el` source file.
use el_lexer::tokenize;
use el_parser::{parse, Stmt};
use crate::types::{TestCase, TestTarget};
/// Parse a source string and extract all `test` block definitions.
///
/// Returns an error if the source cannot be lexed or parsed.
pub fn discover(source: &str) -> Result<Vec<TestCase>, String> {
let tokens = tokenize(source).map_err(|e| format!("lex error: {e}"))?;
let program = parse(tokens, source.to_string()).map_err(|e| format!("parse error: {e}"))?;
let mut cases = Vec::new();
collect_tests(&program.stmts, &mut cases);
Ok(cases)
}
fn collect_tests(stmts: &[Stmt], out: &mut Vec<TestCase>) {
for stmt in stmts {
if let Stmt::TestDef { name, target, body, .. } = stmt {
out.push(TestCase {
name: name.clone(),
target: TestTarget::from(target.clone()),
body: body.clone(),
});
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_discover_empty_source() {
let cases = discover("").unwrap();
assert!(cases.is_empty());
}
#[test]
fn test_discover_no_tests() {
let src = r#"let x: Int = 42"#;
let cases = discover(src).unwrap();
assert!(cases.is_empty());
}
#[test]
fn test_discover_single_test() {
let src = r#"
test "basic arithmetic" {
let x: Int = 6
let y: Int = 7
}
"#;
let cases = discover(src).unwrap();
assert_eq!(cases.len(), 1);
assert_eq!(cases[0].name, "basic arithmetic");
assert_eq!(cases[0].target, TestTarget::Unit);
}
#[test]
fn test_discover_multiple_tests() {
let src = r#"
test "test one" {
let x: Int = 1
}
test "test two" {
let y: Int = 2
}
"#;
let cases = discover(src).unwrap();
assert_eq!(cases.len(), 2);
assert_eq!(cases[0].name, "test one");
assert_eq!(cases[1].name, "test two");
}
#[test]
fn test_discover_e2e_target() {
let src = r#"
test "production lookup" target: e2e {
let x: Int = 1
}
"#;
let cases = discover(src).unwrap();
assert_eq!(cases.len(), 1);
assert_eq!(cases[0].target, TestTarget::E2e);
}
#[test]
fn test_discover_both_target() {
let src = r#"
test "dual target" target: both {
let x: Int = 1
}
"#;
let cases = discover(src).unwrap();
assert_eq!(cases[0].target, TestTarget::Both);
}
#[test]
fn test_discover_default_target_is_unit() {
let src = r#"test "no target" { let x: Int = 1 }"#;
let cases = discover(src).unwrap();
assert_eq!(cases[0].target, TestTarget::Unit);
}
#[test]
fn test_discover_mixed_stmts() {
let src = r#"
let x: Int = 42
test "my test" {
let y: Int = x
}
fn helper() -> Int { return 1 }
"#;
let cases = discover(src).unwrap();
assert_eq!(cases.len(), 1);
}
#[test]
fn test_discover_lex_error() {
let result = discover(r#""unterminated"#);
assert!(result.is_err());
}
#[test]
fn test_discover_body_preserved() {
let src = r#"
test "body check" {
let x: Int = 1
let y: Int = 2
}
"#;
let cases = discover(src).unwrap();
assert_eq!(cases[0].body.len(), 2);
}
}
crates/el-test/src/eval.rs
+547
View File
@@ -0,0 +1,547 @@
//! Test expression evaluator.
//!
//! Walks AST expressions inside a test block and produces runtime `EvalValue`s.
//! This is a lightweight direct evaluator (not the full bytecode VM) specifically
//! designed for the simple expression patterns that appear in test assertions.
use std::collections::HashMap;
use el_parser::{BinOp, Expr, Literal, Stmt};
use crate::graph::{ActivatedNode, ReasoningResult, TestGraph};
use crate::types::AssertionResult;
/// A runtime value in the test evaluator.
#[derive(Debug, Clone, PartialEq)]
pub enum EvalValue {
Int(i64),
Float(f64),
Str(String),
Bool(bool),
Nil,
/// A list of activated graph nodes (result of `activate`)
NodeList(Vec<ActivatedNode>),
/// Result of a `reason()` call
Reasoning(ReasoningResultValue),
}
/// Simplified reasoning result value (owns the verdict string for comparison).
#[derive(Debug, Clone, PartialEq)]
pub struct ReasoningResultValue {
pub verdict: String,
pub confidence: f32,
}
impl EvalValue {
/// Try to get the length of a list/string value.
pub fn len(&self) -> Option<i64> {
match self {
EvalValue::NodeList(v) => Some(v.len() as i64),
EvalValue::Str(s) => Some(s.len() as i64),
_ => None,
}
}
/// Try to index into a list.
pub fn index(&self, i: i64) -> Option<EvalValue> {
match self {
EvalValue::NodeList(v) => {
let idx = if i < 0 {
let uidx = (-i) as usize;
v.len().checked_sub(uidx)?
} else {
i as usize
};
let node = v.get(idx)?;
// Return the node as a struct-like value — we special-case field access
Some(EvalValue::NodeList(vec![node.clone()]))
}
_ => None,
}
}
/// Check if this value contains a substring (for string `contains` keyword).
pub fn contains_str(&self, needle: &str) -> bool {
match self {
EvalValue::Str(s) => s.contains(needle),
EvalValue::NodeList(v) => v.iter().any(|n| n.content.contains(needle)),
_ => false,
}
}
pub fn as_bool(&self) -> bool {
match self {
EvalValue::Bool(b) => *b,
EvalValue::Nil => false,
_ => true,
}
}
}
impl std::fmt::Display for EvalValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
EvalValue::Int(n) => write!(f, "{n}"),
EvalValue::Float(n) => write!(f, "{n}"),
EvalValue::Str(s) => write!(f, "{s}"),
EvalValue::Bool(b) => write!(f, "{b}"),
EvalValue::Nil => write!(f, "nil"),
EvalValue::NodeList(v) => write!(f, "[{} node(s)]", v.len()),
EvalValue::Reasoning(r) => write!(f, "ReasoningResult {{ verdict: {} }}", r.verdict),
}
}
}
/// Evaluator context — holds local bindings and the graph.
pub struct Evaluator<'g> {
locals: HashMap<String, EvalValue>,
graph: &'g TestGraph,
}
impl<'g> Evaluator<'g> {
pub fn new(graph: &'g TestGraph) -> Self {
Self {
locals: HashMap::new(),
graph,
}
}
/// Execute a statement. Returns `Ok(())` or an error string.
pub fn exec_stmt(&mut self, stmt: &Stmt) -> Result<(), String> {
match stmt {
Stmt::Let { name, value, .. } => {
let v = self.eval_expr(value)?;
self.locals.insert(name.clone(), v);
}
Stmt::Expr(expr, _) => {
self.eval_expr(expr)?;
}
Stmt::Return(..) => {
// Return from test body — treat as no-op continuation
}
// Seed statements are handled before eval in the runner
Stmt::Seed(..) | Stmt::TestDef { .. } | Stmt::Assert(..) => {}
Stmt::FnDef { .. } | Stmt::TypeDef { .. } | Stmt::EnumDef { .. } => {}
}
Ok(())
}
/// Evaluate an expression, returning its value.
pub fn eval_expr(&mut self, expr: &Expr) -> Result<EvalValue, String> {
match expr {
Expr::Literal(lit) => Ok(match lit {
Literal::Int(n) => EvalValue::Int(*n),
Literal::Float(f) => EvalValue::Float(*f),
Literal::Str(s) => EvalValue::Str(s.clone()),
Literal::Bool(b) => EvalValue::Bool(*b),
}),
Expr::Ident(name) => {
self.locals.get(name).cloned().ok_or_else(|| format!("undefined variable '{name}'"))
}
Expr::BinOp { op, left, right } => {
// Special: handle `<expr> contains <str>` — parsed as BinOp in the `contains` handler
// Actually `contains` is a keyword identifier parsed as Ident in the postfix context.
// We handle it as a Call pattern below.
let lv = self.eval_expr(left)?;
let rv = self.eval_expr(right)?;
self.eval_binop(op, lv, rv)
}
Expr::Call { func, args } => {
match func.as_ref() {
// reason("hypothesis") — built-in
Expr::Ident(name) if name == "reason" => {
let arg = args.first().ok_or("reason() requires one argument")?;
let hypothesis = match self.eval_expr(arg)? {
EvalValue::Str(s) => s,
other => return Err(format!("reason() expects a string, got {other}")),
};
let result: ReasoningResult = self.graph.reason(&hypothesis);
Ok(EvalValue::Reasoning(ReasoningResultValue {
verdict: result.verdict.to_string(),
confidence: result.confidence,
}))
}
// results.len() — method call on a local
Expr::Field { object, field } if field == "len" => {
let obj = self.eval_expr(object)?;
match obj.len() {
Some(n) => Ok(EvalValue::Int(n)),
None => Err(format!("cannot call .len() on {obj}")),
}
}
// results[0].content contains "needle"
// `contains` is parsed as an Ident called as a method
Expr::Field { object, field } if field == "contains" => {
let obj = self.eval_expr(object)?;
let needle_arg = args.first().ok_or("contains() requires one argument")?;
let needle = match self.eval_expr(needle_arg)? {
EvalValue::Str(s) => s,
other => return Err(format!("contains() expects a string, got {other}")),
};
Ok(EvalValue::Bool(obj.contains_str(&needle)))
}
_ => {
// Unknown call — return nil
Ok(EvalValue::Nil)
}
}
}
Expr::Field { object, field } => {
let obj = self.eval_expr(object)?;
match &obj {
EvalValue::NodeList(nodes) if nodes.len() == 1 => {
let node = &nodes[0];
match field.as_str() {
"content" => Ok(EvalValue::Str(node.content.clone())),
"node_type" => Ok(EvalValue::Str(node.node_type.clone())),
"importance" => Ok(EvalValue::Float(node.importance as f64)),
"id" => Ok(EvalValue::Str(node.id.to_string())),
other => Err(format!("no field '{other}' on ActivatedNode")),
}
}
EvalValue::Reasoning(r) => match field.as_str() {
"verdict" => Ok(EvalValue::Str(r.verdict.clone())),
"confidence" => Ok(EvalValue::Float(r.confidence as f64)),
other => Err(format!("no field '{other}' on ReasoningResult")),
},
_ => Err(format!("cannot access field '{field}' on {obj}")),
}
}
Expr::Index { object, index } => {
let obj = self.eval_expr(object)?;
let idx = self.eval_expr(index)?;
let i = match idx {
EvalValue::Int(n) => n,
other => return Err(format!("index must be Int, got {other}")),
};
obj.index(i).ok_or_else(|| format!("index {i} out of bounds"))
}
Expr::Activate { type_name, query } => {
let nodes = self.graph.activate(query, Some(type_name));
Ok(EvalValue::NodeList(nodes))
}
Expr::UnaryNot(inner) => {
let v = self.eval_expr(inner)?;
Ok(EvalValue::Bool(!v.as_bool()))
}
Expr::Block(stmts) => {
for s in stmts {
self.exec_stmt(s)?;
}
Ok(EvalValue::Nil)
}
Expr::If { cond, then, else_ } => {
let cv = self.eval_expr(cond)?;
if cv.as_bool() {
self.eval_expr(then)
} else if let Some(e) = else_ {
self.eval_expr(e)
} else {
Ok(EvalValue::Nil)
}
}
Expr::Array(elems) => {
// For simplicity — arrays of primitives in tests
let mut vals = Vec::new();
for e in elems {
vals.push(self.eval_expr(e)?);
}
// Return first if all same, else nil
Ok(EvalValue::Nil)
}
Expr::Path { segments } => {
// Enum variant reference — return as string (e.g. "Insufficient")
Ok(EvalValue::Str(segments.last().cloned().unwrap_or_default()))
}
Expr::Match { subject, arms } => {
let sv = self.eval_expr(subject)?;
for arm in arms {
// Simplified: compare subject to pattern
let matches = match &arm.pattern {
el_parser::Pattern::Wildcard => true,
el_parser::Pattern::Literal(lit) => {
let lv = match lit {
Literal::Int(n) => EvalValue::Int(*n),
Literal::Float(f) => EvalValue::Float(*f),
Literal::Str(s) => EvalValue::Str(s.clone()),
Literal::Bool(b) => EvalValue::Bool(*b),
};
sv == lv
}
el_parser::Pattern::Binding(name) => {
self.locals.insert(name.clone(), sv.clone());
true
}
el_parser::Pattern::EnumVariant { variant, .. } => {
sv == EvalValue::Str(variant.clone())
}
};
if matches {
return self.eval_expr(&arm.body);
}
}
Ok(EvalValue::Nil)
}
Expr::Sealed(stmts) => {
for s in stmts {
self.exec_stmt(s)?;
}
Ok(EvalValue::Nil)
}
}
}
fn eval_binop(&self, op: &BinOp, lv: EvalValue, rv: EvalValue) -> Result<EvalValue, String> {
match op {
BinOp::Add => match (lv, rv) {
(EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a + b)),
(EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a + b)),
(EvalValue::Str(a), EvalValue::Str(b)) => Ok(EvalValue::Str(a + &b)),
(a, b) => Err(format!("cannot add {a} and {b}")),
},
BinOp::Sub => match (lv, rv) {
(EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a - b)),
(EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a - b)),
(a, b) => Err(format!("cannot subtract {a} and {b}")),
},
BinOp::Mul => match (lv, rv) {
(EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a * b)),
(EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a * b)),
(a, b) => Err(format!("cannot multiply {a} and {b}")),
},
BinOp::Div => match (lv, rv) {
(EvalValue::Int(a), EvalValue::Int(b)) if b != 0 => Ok(EvalValue::Int(a / b)),
(EvalValue::Float(a), EvalValue::Float(b)) => Ok(EvalValue::Float(a / b)),
(_, EvalValue::Int(0)) => Err("division by zero".into()),
(a, b) => Err(format!("cannot divide {a} and {b}")),
},
BinOp::Eq => Ok(EvalValue::Bool(lv == rv)),
BinOp::NotEq => Ok(EvalValue::Bool(lv != rv)),
BinOp::Lt => self.compare_ord(lv, rv, |o| o == std::cmp::Ordering::Less),
BinOp::Gt => self.compare_ord(lv, rv, |o| o == std::cmp::Ordering::Greater),
BinOp::LtEq => self.compare_ord(lv, rv, |o| o != std::cmp::Ordering::Greater),
BinOp::GtEq => self.compare_ord(lv, rv, |o| o != std::cmp::Ordering::Less),
BinOp::And => Ok(EvalValue::Bool(lv.as_bool() && rv.as_bool())),
BinOp::Or => Ok(EvalValue::Bool(lv.as_bool() || rv.as_bool())),
}
}
fn compare_ord<F>(&self, lv: EvalValue, rv: EvalValue, pred: F) -> Result<EvalValue, String>
where
F: Fn(std::cmp::Ordering) -> bool,
{
let ord = match (&lv, &rv) {
(EvalValue::Int(a), EvalValue::Int(b)) => a.cmp(b),
(EvalValue::Float(a), EvalValue::Float(b)) => {
a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)
}
_ => return Err(format!("cannot compare {lv} and {rv}")),
};
Ok(EvalValue::Bool(pred(ord)))
}
}
/// Evaluate a single `assert` expression and return an `AssertionResult`.
///
/// The `expr_text` is the source text representation of the assertion (for
/// diagnostic output).
pub fn evaluate_assert(
eval: &mut Evaluator,
expr: &Expr,
expr_text: &str,
) -> AssertionResult {
match eval.eval_expr(expr) {
Ok(val) => {
let passed = val.as_bool();
AssertionResult {
expression: expr_text.to_string(),
passed,
actual: Some(val.to_string()),
expected: None,
}
}
Err(_e) => AssertionResult {
expression: expr_text.to_string(),
passed: false,
actual: None,
expected: None,
}
}
}
/// Render an expression as a human-readable string (best-effort, for display).
pub fn expr_to_text(expr: &Expr) -> String {
match expr {
Expr::Literal(Literal::Int(n)) => n.to_string(),
Expr::Literal(Literal::Float(f)) => f.to_string(),
Expr::Literal(Literal::Str(s)) => format!("\"{s}\""),
Expr::Literal(Literal::Bool(b)) => b.to_string(),
Expr::Ident(name) => name.clone(),
Expr::BinOp { op, left, right } => {
let op_str = match op {
BinOp::Add => "+", BinOp::Sub => "-", BinOp::Mul => "*", BinOp::Div => "/",
BinOp::Eq => "==", BinOp::NotEq => "!=",
BinOp::Lt => "<", BinOp::Gt => ">", BinOp::LtEq => "<=", BinOp::GtEq => ">=",
BinOp::And => "&&", BinOp::Or => "||",
};
format!("{} {op_str} {}", expr_to_text(left), expr_to_text(right))
}
Expr::Field { object, field } => format!("{}.{field}", expr_to_text(object)),
Expr::Call { func, args } => {
let args_str: Vec<_> = args.iter().map(expr_to_text).collect();
format!("{}({})", expr_to_text(func), args_str.join(", "))
}
Expr::Index { object, index } => format!("{}[{}]", expr_to_text(object), expr_to_text(index)),
Expr::Activate { type_name, query } => format!("activate {type_name} where \"{query}\""),
Expr::UnaryNot(inner) => format!("!{}", expr_to_text(inner)),
Expr::Path { segments } => segments.join("::"),
_ => "<expr>".to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::TestGraph;
fn fresh_eval(g: &TestGraph) -> Evaluator {
Evaluator::new(g)
}
#[test]
fn test_eval_int_literal() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::Literal(Literal::Int(42));
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(42));
}
#[test]
fn test_eval_bool_literal() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::Literal(Literal::Bool(true));
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Bool(true));
}
#[test]
fn test_eval_string_literal() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::Literal(Literal::Str("hello".into()));
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Str("hello".into()));
}
#[test]
fn test_eval_addition() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::BinOp {
op: BinOp::Add,
left: Box::new(Expr::Literal(Literal::Int(3))),
right: Box::new(Expr::Literal(Literal::Int(4))),
};
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(7));
}
#[test]
fn test_eval_multiplication() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::BinOp {
op: BinOp::Mul,
left: Box::new(Expr::Literal(Literal::Int(6))),
right: Box::new(Expr::Literal(Literal::Int(7))),
};
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(42));
}
#[test]
fn test_eval_equality_true() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::BinOp {
op: BinOp::Eq,
left: Box::new(Expr::Literal(Literal::Int(42))),
right: Box::new(Expr::Literal(Literal::Int(42))),
};
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Bool(true));
}
#[test]
fn test_eval_greater_than() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::BinOp {
op: BinOp::Gt,
left: Box::new(Expr::Literal(Literal::Int(5))),
right: Box::new(Expr::Literal(Literal::Int(3))),
};
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Bool(true));
}
#[test]
fn test_eval_activate_empty_graph() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::Activate {
type_name: "Customer".into(),
query: "anything".into(),
};
let result = e.eval_expr(&expr).unwrap();
assert!(matches!(result, EvalValue::NodeList(ref v) if v.is_empty()));
}
#[test]
fn test_eval_activate_with_seeds() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Will Anderson, founding member", 0.9, None);
let mut e = fresh_eval(&g);
let expr = Expr::Activate {
type_name: "Customer".into(),
query: "founding".into(),
};
let result = e.eval_expr(&expr).unwrap();
assert!(matches!(result, EvalValue::NodeList(ref v) if v.len() == 1));
}
#[test]
fn test_eval_let_and_ident() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
e.locals.insert("x".into(), EvalValue::Int(10));
let expr = Expr::Ident("x".into());
assert_eq!(e.eval_expr(&expr).unwrap(), EvalValue::Int(10));
}
#[test]
fn test_eval_reason_empty_graph() {
let g = TestGraph::new();
let mut e = fresh_eval(&g);
let expr = Expr::Call {
func: Box::new(Expr::Ident("reason".into())),
args: vec![Expr::Literal(Literal::Str("Is there a customer?".into()))],
};
let result = e.eval_expr(&expr).unwrap();
match result {
EvalValue::Reasoning(r) => assert_eq!(r.verdict, "Insufficient"),
_ => panic!("expected Reasoning"),
}
}
}
crates/el-test/src/graph.rs
+335
View File
@@ -0,0 +1,335 @@
//! In-memory graph for test seeding and activation.
//!
//! `TestGraph` provides the runtime behind `seed Node { ... }`, `seed Edge { ... }`,
//! `activate T where "query"`, and `reason("hypothesis")` in test blocks.
//!
//! For **unit tests** the graph is in-memory only — no disk, no external DB.
//! For **e2e tests** the graph would delegate to a real Engram database
//! (full implementation when Engram DB Rust client is available).
use std::collections::HashMap;
/// Unique node identifier (simplified UUID representation).
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct NodeId(pub String);
impl NodeId {
pub fn new() -> Self {
// Simple deterministic ID for tests (real impl would use uuid crate)
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(1);
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
NodeId(format!("node-{n:08x}"))
}
}
impl Default for NodeId {
fn default() -> Self {
Self::new()
}
}
impl std::fmt::Display for NodeId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
/// A node in the test graph.
#[derive(Debug, Clone)]
pub struct GraphNode {
pub id: NodeId,
pub node_type: String,
pub content: String,
pub importance: f32,
#[allow(dead_code)]
pub tier: Option<String>,
}
/// A directed edge between two graph nodes.
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct GraphEdge {
pub from: NodeId,
pub to: NodeId,
pub relation: String,
pub weight: f32,
}
/// A node returned by an `activate` query.
#[derive(Debug, Clone, PartialEq)]
pub struct ActivatedNode {
pub id: NodeId,
pub node_type: String,
pub content: String,
pub importance: f32,
}
/// The verdict of a `reason()` call.
#[derive(Debug, Clone, PartialEq)]
pub enum ReasoningVerdict {
/// Evidence found; hypothesis is supported.
Supported,
/// Evidence found; hypothesis is contradicted.
Contradicted,
/// Insufficient evidence to evaluate hypothesis.
Insufficient,
}
impl std::fmt::Display for ReasoningVerdict {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ReasoningVerdict::Supported => write!(f, "Supported"),
ReasoningVerdict::Contradicted => write!(f, "Contradicted"),
ReasoningVerdict::Insufficient => write!(f, "Insufficient"),
}
}
}
/// Result of a `reason()` call.
#[derive(Debug, Clone)]
pub struct ReasoningResult {
pub verdict: ReasoningVerdict,
pub evidence: Vec<ActivatedNode>,
pub confidence: f32,
}
/// The in-memory graph used during unit tests.
///
/// Seeds accumulate nodes and edges. Activation queries search over them
/// using simple substring/keyword matching (a proxy for real embedding search).
pub struct TestGraph {
nodes: HashMap<NodeId, GraphNode>,
edges: Vec<GraphEdge>,
}
impl TestGraph {
/// Create an empty graph (unit test mode — pure in-memory).
pub fn new() -> Self {
Self {
nodes: HashMap::new(),
edges: Vec::new(),
}
}
/// Seed a node into the graph. Returns the node's ID.
pub fn seed_node(
&mut self,
node_type: &str,
content: &str,
importance: f32,
tier: Option<&str>,
) -> NodeId {
let id = NodeId::new();
let node = GraphNode {
id: id.clone(),
node_type: node_type.to_string(),
content: content.to_string(),
importance,
tier: tier.map(String::from),
};
self.nodes.insert(id.clone(), node);
id
}
/// Seed a directed edge between two nodes.
pub fn seed_edge(&mut self, from: NodeId, to: NodeId, relation: &str, weight: f32) {
self.edges.push(GraphEdge {
from,
to,
relation: relation.to_string(),
weight,
});
}
/// Activate — query nodes by type and keyword relevance.
///
/// Matches nodes whose `node_type` equals `node_type` (case-insensitive)
/// and whose content contains any keyword from the query.
/// Results are sorted by importance descending.
pub fn activate(&self, query: &str, node_type: Option<&str>) -> Vec<ActivatedNode> {
let query_lower = query.to_lowercase();
let keywords: Vec<&str> = query_lower.split_whitespace().collect();
let mut results: Vec<ActivatedNode> = self
.nodes
.values()
.filter(|node| {
// Type filter
if let Some(ty) = node_type {
if !node.node_type.eq_ignore_ascii_case(ty) {
return false;
}
}
// If no query keywords, match all nodes of that type
if keywords.is_empty() {
return true;
}
// Keyword relevance: content must contain at least one keyword
let content_lower = node.content.to_lowercase();
keywords.iter().any(|kw| content_lower.contains(kw))
})
.map(|node| ActivatedNode {
id: node.id.clone(),
node_type: node.node_type.clone(),
content: node.content.clone(),
importance: node.importance,
})
.collect();
results.sort_by(|a, b| b.importance.partial_cmp(&a.importance).unwrap_or(std::cmp::Ordering::Equal));
results
}
/// Reason — evaluate a hypothesis against the seeded graph.
///
/// If the graph is empty, returns `Insufficient`.
/// If matching nodes exist, returns `Supported` with those nodes.
pub fn reason(&self, hypothesis: &str) -> ReasoningResult {
if self.nodes.is_empty() {
return ReasoningResult {
verdict: ReasoningVerdict::Insufficient,
evidence: vec![],
confidence: 0.0,
};
}
let evidence = self.activate(hypothesis, None);
if evidence.is_empty() {
ReasoningResult {
verdict: ReasoningVerdict::Insufficient,
evidence: vec![],
confidence: 0.0,
}
} else {
let confidence = evidence.iter().map(|n| n.importance).sum::<f32>()
/ evidence.len() as f32;
ReasoningResult {
verdict: ReasoningVerdict::Supported,
evidence,
confidence,
}
}
}
/// Number of seeded nodes.
pub fn node_count(&self) -> usize {
self.nodes.len()
}
/// Number of seeded edges.
pub fn edge_count(&self) -> usize {
self.edges.len()
}
}
impl Default for TestGraph {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_seed_node_returns_id() {
let mut g = TestGraph::new();
let id = g.seed_node("Customer", "Will Anderson", 0.9, Some("Semantic"));
assert!(!id.0.is_empty());
}
#[test]
fn test_activate_returns_matching_nodes() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Will Anderson, founding member", 0.9, None);
g.seed_node("Customer", "Jane Smith, standard client", 0.6, None);
let results = g.activate("founding", Some("Customer"));
assert_eq!(results.len(), 1);
assert!(results[0].content.contains("Will Anderson"));
}
#[test]
fn test_activate_empty_query_matches_all_of_type() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Alice", 0.9, None);
g.seed_node("Customer", "Bob", 0.8, None);
g.seed_node("Order", "Order #1", 0.7, None);
let results = g.activate("", Some("Customer"));
assert_eq!(results.len(), 2);
}
#[test]
fn test_activate_wrong_type_returns_empty() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Will Anderson", 0.9, None);
let results = g.activate("Will", Some("Order"));
assert!(results.is_empty());
}
#[test]
fn test_activate_sorts_by_importance() {
let mut g = TestGraph::new();
g.seed_node("Item", "low importance apple", 0.3, None);
g.seed_node("Item", "high importance apple", 0.9, None);
g.seed_node("Item", "medium importance apple", 0.6, None);
let results = g.activate("apple", Some("Item"));
assert_eq!(results.len(), 3);
assert!(results[0].importance >= results[1].importance);
assert!(results[1].importance >= results[2].importance);
}
#[test]
fn test_activate_no_type_filter() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Will Anderson", 0.9, None);
g.seed_node("Order", "Will's order", 0.7, None);
let results = g.activate("Will", None);
assert_eq!(results.len(), 2);
}
#[test]
fn test_reason_empty_graph_returns_insufficient() {
let g = TestGraph::new();
let result = g.reason("Is there a customer?");
assert_eq!(result.verdict, ReasoningVerdict::Insufficient);
assert_eq!(result.confidence, 0.0);
}
#[test]
fn test_reason_with_matching_nodes_returns_supported() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Will Anderson, founding member", 0.9, None);
let result = g.reason("founding member");
assert_eq!(result.verdict, ReasoningVerdict::Supported);
assert!(!result.evidence.is_empty());
}
#[test]
fn test_reason_no_match_returns_insufficient() {
let mut g = TestGraph::new();
g.seed_node("Customer", "Will Anderson", 0.9, None);
let result = g.reason("quantum teleportation");
assert_eq!(result.verdict, ReasoningVerdict::Insufficient);
}
#[test]
fn test_seed_edge() {
let mut g = TestGraph::new();
let cust = g.seed_node("Customer", "Will", 0.9, None);
let order = g.seed_node("Order", "Order #1", 0.7, None);
g.seed_edge(cust, order, "Purchased", 0.9);
assert_eq!(g.edge_count(), 1);
}
#[test]
fn test_node_count() {
let mut g = TestGraph::new();
assert_eq!(g.node_count(), 0);
g.seed_node("X", "content", 0.5, None);
assert_eq!(g.node_count(), 1);
g.seed_node("Y", "more content", 0.5, None);
assert_eq!(g.node_count(), 2);
}
}
crates/el-test/src/lib.rs
+33
View File
@@ -0,0 +1,33 @@
//! el-test — Engram language unified testing framework.
//!
//! # Core insight
//!
//! All Engram state is graph nodes. A test seeds the graph and makes
//! assertions. Unit test = seed a few nodes in-memory. E2e test = point at
//! the real Engram database. **The test code is identical. Only the graph
//! differs.** No mocking framework. No dependency injection. One syntax.
//!
//! # Usage
//!
//! ```rust,ignore
//! use el_test::{TestRunner, TestReport};
//!
//! let source = std::fs::read_to_string("tests.el").unwrap();
//! let tests = el_test::discover(&source).unwrap();
//! let runner = TestRunner::new();
//! let report = runner.run_all(&tests, None);
//! report.print();
//! ```
mod discovery;
mod eval;
mod graph;
mod report;
mod runner;
mod types;
pub use discovery::discover;
pub use graph::TestGraph;
pub use report::TestReport;
pub use runner::TestRunner;
pub use types::{AssertionResult, TestCase, TestResult, TestStatus, TestTarget};
crates/el-test/src/report.rs
+319
View File
@@ -0,0 +1,319 @@
//! Test report formatting — human-readable, JSON, and JUnit XML output.
use crate::types::{TestResult, TestStatus};
/// Aggregated report for a set of test runs.
pub struct TestReport {
pub total: u32,
pub passed: u32,
pub failed: u32,
pub skipped: u32,
pub errors: u32,
pub duration_ms: u64,
pub results: Vec<TestResult>,
}
impl TestReport {
/// Build a report from a slice of individual test results.
pub fn from_results(results: Vec<TestResult>) -> Self {
let total = results.len() as u32;
let passed = results.iter().filter(|r| r.status == TestStatus::Pass).count() as u32;
let failed = results.iter().filter(|r| r.status == TestStatus::Fail).count() as u32;
let skipped = results.iter().filter(|r| r.status == TestStatus::Skip).count() as u32;
let errors = results.iter().filter(|r| r.status == TestStatus::Error).count() as u32;
let duration_ms = results.iter().map(|r| r.duration_ms).sum();
Self { total, passed, failed, skipped, errors, duration_ms, results }
}
/// Print a human-readable summary to stdout.
pub fn print(&self) {
let target_label = format!("({}ms total)", self.duration_ms);
println!("\nRunning {} tests...\n", self.total);
for r in &self.results {
let icon = match r.status {
TestStatus::Pass => " ok ",
TestStatus::Fail => " FAIL ",
TestStatus::Skip => " SKIP ",
TestStatus::Error => "ERROR ",
};
println!(" [{icon}] {} ({}ms)", r.name, r.duration_ms);
// Show failing assertions
if r.status == TestStatus::Fail {
for (i, a) in r.assertions.iter().enumerate() {
if !a.passed {
println!(" assert {}", a.expression);
if let Some(actual) = &a.actual {
println!(" actual: {actual}");
}
if let Some(expected) = &a.expected {
println!(" expected: {expected}");
}
println!(" at assertion {}", i + 1);
}
}
}
if let Some(err) = &r.error {
println!(" error: {err}");
}
}
println!("\nResults: {} passed, {} failed, {} skipped {}", self.passed, self.failed, self.skipped, target_label);
if self.errors > 0 {
println!(" ({} error(s) — see above)", self.errors);
}
}
/// Serialize to JSON.
pub fn to_json(&self) -> String {
let results: Vec<serde_json::Value> = self
.results
.iter()
.map(|r| {
let assertions: Vec<serde_json::Value> = r
.assertions
.iter()
.map(|a| {
serde_json::json!({
"expression": a.expression,
"passed": a.passed,
"actual": a.actual,
"expected": a.expected,
})
})
.collect();
serde_json::json!({
"name": r.name,
"target": r.target.to_string(),
"status": r.status.to_string(),
"duration_ms": r.duration_ms,
"assertions": assertions,
"error": r.error,
})
})
.collect();
let report = serde_json::json!({
"total": self.total,
"passed": self.passed,
"failed": self.failed,
"skipped": self.skipped,
"errors": self.errors,
"duration_ms": self.duration_ms,
"results": results,
});
serde_json::to_string_pretty(&report).unwrap_or_else(|_| "{}".to_string())
}
/// Serialize to JUnit XML (for CI integration).
pub fn to_junit_xml(&self) -> String {
let mut xml = String::new();
xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
xml.push_str(&format!(
"<testsuite name=\"engram-lang\" tests=\"{}\" failures=\"{}\" errors=\"{}\" skipped=\"{}\" time=\"{}\">\n",
self.total,
self.failed,
self.errors,
self.skipped,
self.duration_ms as f64 / 1000.0,
));
for r in &self.results {
let classname = "el_test";
let time = r.duration_ms as f64 / 1000.0;
let name_escaped = xml_escape(&r.name);
match r.status {
TestStatus::Pass => {
xml.push_str(&format!(
" <testcase classname=\"{classname}\" name=\"{name_escaped}\" time=\"{time:.3}\"/>\n"
));
}
TestStatus::Fail => {
xml.push_str(&format!(
" <testcase classname=\"{classname}\" name=\"{name_escaped}\" time=\"{time:.3}\">\n"
));
for a in r.assertions.iter().filter(|a| !a.passed) {
let msg = xml_escape(&a.expression);
let details = match (&a.actual, &a.expected) {
(Some(act), Some(exp)) => format!("actual: {act}, expected: {exp}"),
(Some(act), None) => format!("actual: {act}"),
_ => "assertion failed".to_string(),
};
let details_esc = xml_escape(&details);
xml.push_str(&format!(
" <failure message=\"{msg}\">{details_esc}</failure>\n"
));
}
xml.push_str(" </testcase>\n");
}
TestStatus::Skip => {
xml.push_str(&format!(
" <testcase classname=\"{classname}\" name=\"{name_escaped}\" time=\"{time:.3}\">\n <skipped/>\n </testcase>\n"
));
}
TestStatus::Error => {
let err_msg = xml_escape(r.error.as_deref().unwrap_or("unknown error"));
xml.push_str(&format!(
" <testcase classname=\"{classname}\" name=\"{name_escaped}\" time=\"{time:.3}\">\n <error message=\"{err_msg}\"/>\n </testcase>\n"
));
}
}
}
xml.push_str("</testsuite>\n");
xml
}
/// Whether the overall test run passed (no failures or errors).
pub fn is_pass(&self) -> bool {
self.failed == 0 && self.errors == 0
}
}
fn xml_escape(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&apos;")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::{AssertionResult, TestTarget};
fn make_pass(name: &str) -> TestResult {
TestResult {
name: name.to_string(),
target: TestTarget::Unit,
status: TestStatus::Pass,
duration_ms: 5,
assertions: vec![AssertionResult {
expression: "x == 42".into(),
passed: true,
actual: Some("true".into()),
expected: None,
}],
error: None,
}
}
fn make_fail(name: &str) -> TestResult {
TestResult {
name: name.to_string(),
target: TestTarget::Unit,
status: TestStatus::Fail,
duration_ms: 3,
assertions: vec![AssertionResult {
expression: "x == 99".into(),
passed: false,
actual: Some("42".into()),
expected: Some("99".into()),
}],
error: None,
}
}
fn make_skip(name: &str) -> TestResult {
TestResult {
name: name.to_string(),
target: TestTarget::E2e,
status: TestStatus::Skip,
duration_ms: 0,
assertions: vec![],
error: Some("ENGRAM_URL not set".into()),
}
}
#[test]
fn test_report_counts() {
let report = TestReport::from_results(vec![
make_pass("a"),
make_fail("b"),
make_skip("c"),
]);
assert_eq!(report.total, 3);
assert_eq!(report.passed, 1);
assert_eq!(report.failed, 1);
assert_eq!(report.skipped, 1);
}
#[test]
fn test_report_is_pass() {
let report = TestReport::from_results(vec![make_pass("a"), make_pass("b")]);
assert!(report.is_pass());
}
#[test]
fn test_report_is_not_pass_on_fail() {
let report = TestReport::from_results(vec![make_pass("a"), make_fail("b")]);
assert!(!report.is_pass());
}
#[test]
fn test_to_json_valid() {
let report = TestReport::from_results(vec![make_pass("test")]);
let json = report.to_json();
let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON");
assert_eq!(parsed["total"], 1);
assert_eq!(parsed["passed"], 1);
}
#[test]
fn test_to_json_contains_results() {
let report = TestReport::from_results(vec![make_pass("hello")]);
let json = report.to_json();
assert!(json.contains("hello"));
}
#[test]
fn test_to_junit_xml_valid() {
let report = TestReport::from_results(vec![make_pass("a"), make_fail("b")]);
let xml = report.to_junit_xml();
assert!(xml.starts_with("<?xml"));
assert!(xml.contains("<testsuite"));
assert!(xml.contains("</testsuite>"));
}
#[test]
fn test_to_junit_xml_pass_testcase() {
let report = TestReport::from_results(vec![make_pass("arithmetic")]);
let xml = report.to_junit_xml();
assert!(xml.contains("arithmetic"));
assert!(!xml.contains("<failure"));
}
#[test]
fn test_to_junit_xml_fail_testcase() {
let report = TestReport::from_results(vec![make_fail("failing")]);
let xml = report.to_junit_xml();
assert!(xml.contains("<failure"));
}
#[test]
fn test_to_junit_xml_skipped() {
let report = TestReport::from_results(vec![make_skip("e2e")]);
let xml = report.to_junit_xml();
assert!(xml.contains("<skipped"));
}
#[test]
fn test_duration_sum() {
let report = TestReport::from_results(vec![make_pass("a"), make_pass("b")]);
assert_eq!(report.duration_ms, 10); // 5 + 5
}
#[test]
fn test_xml_escape() {
let escaped = super::xml_escape("a < b & c > d \"e\" 'f'");
assert!(escaped.contains("&lt;"));
assert!(escaped.contains("&amp;"));
assert!(escaped.contains("&gt;"));
assert!(escaped.contains("&quot;"));
}
}
crates/el-test/src/runner.rs
+382
View File
@@ -0,0 +1,382 @@
//! Test runner — executes test cases and produces results.
use std::time::Instant;
use el_parser::{SeedStmt, Stmt};
use crate::eval::{evaluate_assert, expr_to_text, Evaluator};
use crate::graph::TestGraph;
use crate::types::{AssertionResult, TestCase, TestResult, TestStatus, TestTarget};
/// Runs test cases and collects results.
pub struct TestRunner;
impl TestRunner {
pub fn new() -> Self {
Self
}
/// Run all tests. E2e tests are skipped if `engram_url` is `None`.
pub fn run_all(&self, tests: &[TestCase], engram_url: Option<&str>) -> Vec<TestResult> {
let mut results = Vec::new();
for test in tests {
match &test.target {
TestTarget::Unit => {
results.push(self.run_unit_test(test));
}
TestTarget::E2e => {
if let Some(url) = engram_url {
results.push(self.run_e2e_test(test, url));
} else {
results.push(TestResult {
name: test.name.clone(),
target: TestTarget::E2e,
status: TestStatus::Skip,
duration_ms: 0,
assertions: vec![],
error: Some("ENGRAM_URL not set; skipping e2e test".into()),
});
}
}
TestTarget::Both => {
results.push(self.run_unit_test(test));
if let Some(url) = engram_url {
results.push(self.run_e2e_test(test, url));
} else {
results.push(TestResult {
name: format!("{} (e2e)", test.name),
target: TestTarget::E2e,
status: TestStatus::Skip,
duration_ms: 0,
assertions: vec![],
error: Some("ENGRAM_URL not set; skipping e2e test".into()),
});
}
}
}
}
results
}
/// Run only unit tests.
pub fn run_unit(&self, tests: &[TestCase]) -> Vec<TestResult> {
tests
.iter()
.filter(|t| matches!(t.target, TestTarget::Unit | TestTarget::Both))
.map(|t| self.run_unit_test(t))
.collect()
}
/// Run only e2e tests.
pub fn run_e2e<'a>(&self, tests: &'a [TestCase], engram_url: &str) -> Vec<TestResult> {
tests
.iter()
.filter(|t| matches!(t.target, TestTarget::E2e | TestTarget::Both))
.map(|t| self.run_e2e_test(t, engram_url))
.collect()
}
/// Run a single test against an in-memory graph.
pub fn run_one(&self, test: &TestCase, engram_url: Option<&str>) -> TestResult {
match (&test.target, engram_url) {
(TestTarget::E2e, Some(url)) => self.run_e2e_test(test, url),
(TestTarget::E2e, None) => TestResult {
name: test.name.clone(),
target: TestTarget::E2e,
status: TestStatus::Skip,
duration_ms: 0,
assertions: vec![],
error: Some("ENGRAM_URL not set; skipping e2e test".into()),
},
_ => self.run_unit_test(test),
}
}
// ── Private ───────────────────────────────────────────────────────────────
fn run_unit_test(&self, test: &TestCase) -> TestResult {
let start = Instant::now();
let mut graph = TestGraph::new();
// Apply seed statements first
for stmt in &test.body {
if let Stmt::Seed(seed, _) = stmt {
apply_seed(&mut graph, seed);
}
}
self.execute_test(test, &graph, TestTarget::Unit, start)
}
fn run_e2e_test(&self, test: &TestCase, _engram_url: &str) -> TestResult {
let start = Instant::now();
// For e2e, we still use an in-memory graph for now (real DB client TBD).
// The distinction is that e2e tests skip the seed step (they use real data).
let graph = TestGraph::new();
self.execute_test(test, &graph, TestTarget::E2e, start)
}
fn execute_test(
&self,
test: &TestCase,
graph: &TestGraph,
target: TestTarget,
start: Instant,
) -> TestResult {
let mut eval = Evaluator::new(graph);
let mut assertions: Vec<AssertionResult> = Vec::new();
let mut error: Option<String> = None;
for stmt in &test.body {
match stmt {
Stmt::Seed(..) => {
// Already processed before eval
}
Stmt::Assert(expr, _) => {
let text = expr_to_text(expr);
let result = evaluate_assert(&mut eval, expr, &text);
assertions.push(result);
}
other => {
if let Err(e) = eval.exec_stmt(other) {
error = Some(e);
break;
}
}
}
}
let duration_ms = start.elapsed().as_millis() as u64;
let all_passed = assertions.iter().all(|a| a.passed);
let status = if error.is_some() {
TestStatus::Error
} else if all_passed {
TestStatus::Pass
} else {
TestStatus::Fail
};
TestResult {
name: test.name.clone(),
target,
status,
duration_ms,
assertions,
error,
}
}
}
impl Default for TestRunner {
fn default() -> Self {
Self::new()
}
}
fn apply_seed(graph: &mut TestGraph, seed: &SeedStmt) {
match seed {
SeedStmt::Node { node_type, content, importance, tier } => {
graph.seed_node(node_type, content, *importance, tier.as_deref());
}
SeedStmt::Edge { from, to, relation, weight } => {
// For edges we use string IDs — in a real impl these would be looked
// up from the graph's node registry. We create placeholder node IDs.
use crate::graph::NodeId;
graph.seed_edge(
NodeId(from.clone()),
NodeId(to.clone()),
relation,
*weight,
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use el_parser::{Expr, Literal, BinOp, Stmt};
use el_lexer::Span;
fn dummy_span() -> Span {
Span::new(0, 0, 1, 1)
}
fn make_assert(expr: Expr) -> Stmt {
Stmt::Assert(expr, dummy_span())
}
fn make_let(name: &str, expr: Expr) -> Stmt {
Stmt::Let {
name: name.to_string(),
type_ann: None,
value: expr,
span: dummy_span(),
}
}
fn int_lit(n: i64) -> Expr {
Expr::Literal(Literal::Int(n))
}
fn str_lit(s: &str) -> Expr {
Expr::Literal(Literal::Str(s.to_string()))
}
fn bool_lit(b: bool) -> Expr {
Expr::Literal(Literal::Bool(b))
}
fn binop(op: BinOp, l: Expr, r: Expr) -> Expr {
Expr::BinOp { op, left: Box::new(l), right: Box::new(r) }
}
fn test_case(name: &str, body: Vec<Stmt>) -> TestCase {
TestCase {
name: name.to_string(),
target: TestTarget::Unit,
body,
}
}
#[test]
fn test_runner_pass() {
let runner = TestRunner::new();
let tc = test_case("arithmetic", vec![
make_let("x", int_lit(6)),
make_let("y", int_lit(7)),
make_let("result", binop(BinOp::Mul, Expr::Ident("x".into()), Expr::Ident("y".into()))),
make_assert(binop(BinOp::Eq, Expr::Ident("result".into()), int_lit(42))),
]);
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Pass);
assert!(result.assertions[0].passed);
}
#[test]
fn test_runner_fail() {
let runner = TestRunner::new();
let tc = test_case("failing", vec![
make_assert(binop(BinOp::Eq, int_lit(1), int_lit(2))),
]);
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Fail);
assert!(!result.assertions[0].passed);
}
#[test]
fn test_runner_multiple_assertions_partial_fail() {
let runner = TestRunner::new();
let tc = test_case("partial", vec![
make_assert(bool_lit(true)),
make_assert(binop(BinOp::Eq, int_lit(1), int_lit(2))), // fails
make_assert(bool_lit(true)),
]);
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Fail);
assert!(result.assertions[0].passed);
assert!(!result.assertions[1].passed);
assert!(result.assertions[2].passed);
}
#[test]
fn test_runner_e2e_skip_without_url() {
let runner = TestRunner::new();
let tc = TestCase {
name: "e2e test".to_string(),
target: TestTarget::E2e,
body: vec![],
};
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Skip);
}
#[test]
fn test_runner_run_all_skips_e2e() {
let runner = TestRunner::new();
let tests = vec![
TestCase { name: "unit".into(), target: TestTarget::Unit, body: vec![] },
TestCase { name: "e2e".into(), target: TestTarget::E2e, body: vec![] },
];
let results = runner.run_all(&tests, None);
assert_eq!(results.len(), 2);
assert_eq!(results[0].status, TestStatus::Pass); // empty = pass
assert_eq!(results[1].status, TestStatus::Skip);
}
#[test]
fn test_runner_run_unit_filters_unit_only() {
let runner = TestRunner::new();
let tests = vec![
TestCase { name: "unit".into(), target: TestTarget::Unit, body: vec![] },
TestCase { name: "e2e".into(), target: TestTarget::E2e, body: vec![] },
];
let results = runner.run_unit(&tests);
assert_eq!(results.len(), 1);
assert_eq!(results[0].name, "unit");
}
#[test]
fn test_runner_empty_test_passes() {
let runner = TestRunner::new();
let tc = test_case("empty", vec![]);
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Pass);
}
#[test]
fn test_runner_with_seed_and_activate() {
use el_parser::SeedStmt;
let runner = TestRunner::new();
let seed = Stmt::Seed(
SeedStmt::Node {
node_type: "Customer".into(),
content: "Will Anderson, founding member".into(),
importance: 0.9,
tier: Some("Semantic".into()),
},
dummy_span(),
);
// activate Customer where "founding"
let activate = Expr::Activate {
type_name: "Customer".into(),
query: "founding".into(),
};
let let_results = make_let("results", activate);
// assert results.len() > 0 => assert results.len() > 0
// We'll call .len() via a Call to Field
let len_call = Expr::Call {
func: Box::new(Expr::Field {
object: Box::new(Expr::Ident("results".into())),
field: "len".into(),
}),
args: vec![],
};
let assert_len = make_assert(binop(BinOp::Gt, len_call, int_lit(0)));
let tc = test_case("seed and activate", vec![seed, let_results, assert_len]);
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Pass);
}
#[test]
fn test_runner_empty_graph_activate_returns_empty() {
let runner = TestRunner::new();
// No seeds — activate should return empty list
let activate = Expr::Activate {
type_name: "Customer".into(),
query: "anything".into(),
};
let let_results = make_let("results", activate);
let len_call = Expr::Call {
func: Box::new(Expr::Field {
object: Box::new(Expr::Ident("results".into())),
field: "len".into(),
}),
args: vec![],
};
let assert_zero = make_assert(binop(BinOp::Eq, len_call, int_lit(0)));
let tc = test_case("empty graph", vec![let_results, assert_zero]);
let result = runner.run_one(&tc, None);
assert_eq!(result.status, TestStatus::Pass);
}
}
crates/el-test/src/types.rs
+84
View File
@@ -0,0 +1,84 @@
//! Core data types for the testing framework.
use el_parser::Stmt;
/// Which graph a test should execute against.
#[derive(Debug, Clone, PartialEq)]
pub enum TestTarget {
/// In-memory graph — default, zero external dependencies.
Unit,
/// Real Engram database pointed at by `ENGRAM_URL` / `ENGRAM_DB_PATH`.
E2e,
/// Run against both unit (in-memory) and e2e (real DB).
Both,
}
impl std::fmt::Display for TestTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TestTarget::Unit => write!(f, "unit"),
TestTarget::E2e => write!(f, "e2e"),
TestTarget::Both => write!(f, "both"),
}
}
}
impl From<el_parser::TestTarget> for TestTarget {
fn from(t: el_parser::TestTarget) -> Self {
match t {
el_parser::TestTarget::Unit => TestTarget::Unit,
el_parser::TestTarget::E2e => TestTarget::E2e,
el_parser::TestTarget::Both => TestTarget::Both,
}
}
}
/// A single test case extracted from an `.el` source file.
pub struct TestCase {
pub name: String,
pub target: TestTarget,
/// The body statements of the test block.
pub body: Vec<Stmt>,
}
/// Status of a single test run.
#[derive(Debug, Clone, PartialEq)]
pub enum TestStatus {
Pass,
Fail,
/// Skipped because the required target is not available.
Skip,
/// Unexpected runtime error (not an assertion failure).
Error,
}
impl std::fmt::Display for TestStatus {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TestStatus::Pass => write!(f, "pass"),
TestStatus::Fail => write!(f, "fail"),
TestStatus::Skip => write!(f, "skip"),
TestStatus::Error => write!(f, "error"),
}
}
}
/// Result of evaluating a single `assert` statement.
#[derive(Debug, Clone)]
pub struct AssertionResult {
/// The `assert` expression as source text.
pub expression: String,
pub passed: bool,
pub actual: Option<String>,
pub expected: Option<String>,
}
/// The full result of one test execution.
pub struct TestResult {
pub name: String,
pub target: TestTarget,
pub status: TestStatus,
pub duration_ms: u64,
pub assertions: Vec<AssertionResult>,
pub error: Option<String>,
}
examples/hello-project/src/tests.el
+57
View File
@@ -0,0 +1,57 @@
// Engram language — example test file
// Run with: el test-file examples/hello-project/src/tests.el
test "basic arithmetic" {
let x: Int = 6
let y: Int = 7
let result: Int = x * y
assert result == 42
}
test "string operations" {
let greeting: String = "Hello"
let name: String = "Engram"
let full: String = greeting + ", " + name
assert full == "Hello, Engram"
}
test "type graph has Point type" {
seed Node {
type: "Point"
content: "A 2D coordinate"
importance: 0.8
tier: Semantic
}
let results = activate Point where "coordinate"
assert results.len() > 0
}
test "empty graph" {
let results = activate Customer where "anything"
assert results.len() == 0
}
test "empty graph returns insufficient" {
let result = reason("Is there a customer named Will?")
assert result.verdict == "Insufficient"
}
test "seeded graph reasoning" {
seed Node {
type: "Customer"
content: "Will Anderson, founding member"
importance: 0.9
tier: Semantic
}
let result = reason("founding member")
assert result.verdict == "Supported"
}
test "production customer lookup" target: e2e {
// No seeds — uses the real Engram database
// (skipped when ENGRAM_URL is not set)
let results = activate Customer where "founding member"
assert results.len() > 0
}