feat: unified testing framework — unit and e2e same syntax, seed-based graph testing, debugger infrastructure

- New crate el-test: test discovery, in-memory graph seeding, assertion evaluator, TestRunner, TestReport with human/JSON/JUnit XML output
- New keywords: test, seed, assert, target — fully integrated into lexer, parser, codegen, type-checker
- Parser extensions: TestDef, SeedStmt, Assert AST nodes; seed blocks handle type: as field name (keyword-as-ident in seed context)
- Debugger: DebugEvent, Debugger, StepMode, StackFrame in el-compiler — breakpoints, step-over, step-into, step-out
- CLI: el test-file <file.el> runs tests; el test integrates with project; el debug attaches debugger; --output json|junit for CI
- 52 new tests in el-test covering discovery, graph seeding, assertion evaluation, pass/fail/error/skip, report generation, JUnit XML
- Example: examples/hello-project/src/tests.el — 6 unit tests pass, 1 e2e test correctly skipped without ENGRAM_URL
This commit is contained in:
Will Anderson
2026-04-27 19:11:59 -05:00
parent 48b72843e1
commit 0a36a454f9
14 changed files with 2177 additions and 14 deletions
+1
View File
@@ -18,6 +18,7 @@ el-seal = { workspace = true }
el-manifest = { workspace = true }
el-registry = { workspace = true }
el-build = { workspace = true }
el-test = { workspace = true }
clap = { workspace = true }
thiserror = { workspace = true }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
+216 -12
View File
@@ -32,6 +32,7 @@ use std::path::PathBuf;
use clap::{Parser, Subcommand};
use el_build::BuildSystem;
use el_compiler::{Compiler, CompilerOptions, Target};
use el_test;
use el_manifest::{BuildTarget, Manifest};
use el_seal::{seal as seal_fn, unseal as unseal_fn, SealedArtifact, DeploymentBinding, SealAlgorithm, SealConfig};
@@ -101,12 +102,50 @@ enum Command {
manifest: Option<PathBuf>,
},
/// Run all tests.
/// Run tests in the current project (reads el.toml).
Test {
/// Only run tests matching this name (substring match).
filter: Option<String>,
/// Also run e2e tests (requires ENGRAM_URL or ENGRAM_DB_PATH).
#[arg(long)]
e2e: bool,
/// Run unit AND e2e tests.
#[arg(long)]
all: bool,
/// Output format: human (default) | json | junit.
#[arg(long, default_value = "human")]
output: String,
/// Path to the project manifest (default: el.toml).
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Run tests from a single .el file (no el.toml required).
TestFile {
/// Source file containing test blocks (*.el).
file: PathBuf,
/// Only run tests matching this name (substring match).
filter: Option<String>,
/// Also run e2e tests (requires ENGRAM_URL or ENGRAM_DB_PATH).
#[arg(long)]
e2e: bool,
/// Run unit AND e2e tests.
#[arg(long)]
all: bool,
/// Output format: human (default) | json | junit.
#[arg(long, default_value = "human")]
output: String,
},
/// Run an Engram source file with the step-debugger attached.
Debug {
/// Source file (*.el).
file: PathBuf,
/// Set a breakpoint at line N.
#[arg(long, value_name = "LINE")]
r#break: Option<u32>,
},
/// Type-check source files without producing artifacts.
Check {
#[arg(long)]
@@ -268,20 +307,51 @@ async fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
run_interpreter(&instructions);
}
Command::Test { manifest } => {
Command::Test { filter, e2e, all, output, manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
let report = bs.test().await?;
println!(
"test: {} passed, {} failed (total {})",
report.passed, report.failed, report.total
);
for f in &report.failures {
eprintln!(" FAIL: {f}");
}
if !report.success() {
std::process::exit(1);
// Find all .el source files in the project
let entry = bs.manifest.build.entry.clone();
let entry_path = manifest_path.parent().unwrap_or(std::path::Path::new(".")).join(&entry);
let source = std::fs::read_to_string(&entry_path)
.map_err(|e| format!("cannot read {}: {e}", entry_path.display()))?;
run_tests_from_source(&source, filter.as_deref(), e2e, all, &output)?;
}
Command::TestFile { file, filter, e2e, all, output } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
run_tests_from_source(&source, filter.as_deref(), e2e, all, &output)?;
}
Command::Debug { file, r#break } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
let opts = CompilerOptions {
target: Target::Debug,
source_path: file.clone(),
..Default::default()
};
let compiled = Compiler::compile(&source, opts)?;
let instructions = el_compiler::Bytecode::deserialize_all(&compiled.artifact)
.unwrap_or_default();
let mut debugger = el_compiler::Debugger::new();
if let Some(line) = r#break {
// Convert line number to a bytecode offset approximation.
// In a full implementation this would use the source map.
// For now we use the line number directly as a placeholder offset.
debugger.add_breakpoint(line as usize);
println!("debugger: breakpoint set at line {line}");
} else {
println!("debugger: breaking on first instruction");
}
println!("debugger: running {} ({} instructions)", file.display(), instructions.len());
run_interpreter_debug(&instructions, &mut debugger);
}
Command::Check { manifest } => {
@@ -628,6 +698,55 @@ fn build_seal_config() -> Result<SealConfig, String> {
})
}
/// Discover tests in source, filter, run, and print results.
fn run_tests_from_source(
source: &str,
filter: Option<&str>,
e2e: bool,
all: bool,
output_fmt: &str,
) -> Result<(), Box<dyn std::error::Error>> {
use el_test::{TestReport, TestRunner};
let mut tests = el_test::discover(source)?;
// Apply name filter
if let Some(f) = filter {
tests.retain(|t| t.name.contains(f));
}
if tests.is_empty() {
println!("no tests found");
return Ok(());
}
let engram_url = std::env::var("ENGRAM_URL").ok().or_else(|| std::env::var("ENGRAM_DB_PATH").ok());
let url_ref = engram_url.as_deref();
let runner = TestRunner::new();
let results = if all {
runner.run_all(&tests, url_ref)
} else if e2e {
runner.run_e2e(&tests, url_ref.unwrap_or(""))
} else {
runner.run_unit(&tests)
};
let report = TestReport::from_results(results);
match output_fmt {
"json" => println!("{}", report.to_json()),
"junit" => println!("{}", report.to_junit_xml()),
_ => report.print(),
}
if !report.is_pass() {
std::process::exit(1);
}
Ok(())
}
/// Minimal interpreter for demonstration.
fn run_interpreter(instructions: &[el_compiler::Bytecode]) {
use el_compiler::{Bytecode, Value};
@@ -732,3 +851,88 @@ fn run_interpreter(instructions: &[el_compiler::Bytecode]) {
ip += 1;
}
}
/// Interpreter with debugger support — emits DebugEvents as it runs.
fn run_interpreter_debug(instructions: &[el_compiler::Bytecode], debugger: &mut el_compiler::Debugger) {
use el_compiler::{Bytecode, Value};
let mut stack: Vec<Value> = Vec::new();
let mut locals: std::collections::HashMap<String, Value> = std::collections::HashMap::new();
let mut ip = 0usize;
while ip < instructions.len() {
// Check if we should pause here
if debugger.should_pause(ip) {
debugger.on_pause(ip, locals.clone());
for event in debugger.drain_events() {
match event {
el_compiler::DebugEvent::Breakpoint { offset, frame } => {
println!("[break] offset={offset} fn={} {}:{}:{}", frame.function_name, frame.source_file, frame.line, frame.col);
}
el_compiler::DebugEvent::Step { frame, locals: step_locals } => {
let var_list: Vec<String> = step_locals.iter()
.map(|(k, v)| format!("{k}={v}"))
.collect();
println!("[step] offset={ip} {}:{} vars=[{}]", frame.line, frame.col, var_list.join(", "));
}
_ => {}
}
}
}
match &instructions[ip] {
Bytecode::Push(v) => stack.push(v.clone()),
Bytecode::Pop => { stack.pop(); }
Bytecode::Add => {
let (b, a) = (stack.pop().unwrap_or(Value::Nil), stack.pop().unwrap_or(Value::Nil));
stack.push(match (a, b) {
(Value::Int(x), Value::Int(y)) => Value::Int(x + y),
(Value::Float(x), Value::Float(y)) => Value::Float(x + y),
(Value::Str(x), Value::Str(y)) => Value::Str(x + &y),
_ => Value::Nil,
});
}
Bytecode::StoreLocal(name) => {
let v = stack.pop().unwrap_or(Value::Nil);
locals.insert(name.clone(), v);
}
Bytecode::LoadLocal(name) => {
let v = locals.get(name).cloned().unwrap_or(Value::Nil);
stack.push(v);
}
Bytecode::Call { name, .. } => {
if name == "print" || name == "println" {
let v = stack.pop().unwrap_or(Value::Nil);
println!("{v}");
stack.push(Value::Nil);
}
}
Bytecode::Jump(offset) => {
let new_ip = (ip as i32 + 1 + offset) as usize;
ip = new_ip;
continue;
}
Bytecode::JumpIf(offset) => {
let cond = stack.pop().unwrap_or(Value::Nil);
if matches!(cond, Value::Bool(true)) {
let new_ip = (ip as i32 + 1 + offset) as usize;
ip = new_ip;
continue;
}
}
Bytecode::JumpIfNot(offset) => {
let cond = stack.pop().unwrap_or(Value::Nil);
if !matches!(cond, Value::Bool(true)) {
let new_ip = (ip as i32 + 1 + offset) as usize;
ip = new_ip;
continue;
}
}
Bytecode::Return | Bytecode::Halt => {
debugger.pop_frame(stack.last().cloned().unwrap_or(Value::Nil));
break;
}
_ => {}
}
ip += 1;
}
}