feat: package manager, build system, native cross-compilation, plugin system

Add three new crates and extend the compiler and CLI toolchain: - el-manifest: el.toml manifest parser using serde + toml crate; supports package info, registry/path/version deps, build config with seal key sources, cross targets, and plugins; Manifest::find_manifest() walks up the directory tree - el-registry: HTTP registry client (reqwest + tokio) for packages.neurontechnologies.ai; PackageMetadata, fetch/download/publish/ search, BLAKE3 checksum verification, local cache at ~/.engram/packages/ - el-build: build orchestrator with incremental builds (BLAKE3 file hashes in .el/build-cache.json), cross-compilation target tagging, dep resolution, plugin registry with on_ast/on_typed_ast/on_bytecode hooks, test runner, fmt/check/clean commands - CrossTarget and NativeTarget enums with triple() and artifact_extension() methods; NativeTarget::Host detects compile-time platform via cfg! macros - Plugin system: CompilerPlugin trait + PluginRegistry; dynamic loading is a marked TODO with clear extension point for libloading - CLI extended with: new, add, remove, update, build --cross, run, test, check, fmt, clean, publish, search, plugin add/remove/list; old single-file commands moved to build-file/seal/unseal subcommands - Fix pre-existing debugger.rs borrow error (unwrap_or temporary lifetime) - Fix checker.rs and codegen.rs to handle TestDef/Seed/Assert Stmt variants - Add spec/language.md sections 12-14: package system, build system, plugin system, cross-compilation targets table 130 tests passing, zero warnings
2026-04-27 19:08:25 -05:00
parent 9ced941590
commit 48b72843e1
31 changed files with 4923 additions and 107 deletions
@@ -121,6 +121,9 @@ impl Codegen {
            Stmt::TypeDef { .. } | Stmt::EnumDef { .. } => {
                // Type and enum definitions are compile-time only; no runtime code.
            }
+            // Test-related statements — skipped during normal compilation.
+            // The el-test crate walks the AST directly rather than running compiled bytecode.
+            Stmt::TestDef { .. } | Stmt::Seed(..) | Stmt::Assert(..) => {}
        }
        Ok(())
    }
@@ -327,7 +330,10 @@ fn stmt_span(stmt: &Stmt) -> el_lexer::Span {
        | Stmt::Expr(_, span)
        | Stmt::FnDef { span, .. }
        | Stmt::TypeDef { span, .. }
-        | Stmt::EnumDef { span, .. } => *span,
+        | Stmt::EnumDef { span, .. }
+        | Stmt::TestDef { span, .. }
+        | Stmt::Seed(_, span)
+        | Stmt::Assert(_, span) => *span,
    }
 }

@@ -0,0 +1,279 @@
+//! Step-debugger infrastructure for the Engram VM.
+//!
+//! The [`Debugger`] is attached to the bytecode interpreter and emits
+//! [`DebugEvent`]s as execution proceeds. IDEs and `el debug` consume these
+//! events to show variable state, call stack, and current source line.
+
+use std::collections::{HashMap, HashSet};
+
+use crate::bytecode::Value;
+
+/// A single frame on the call stack.
+#[derive(Debug, Clone)]
+pub struct StackFrame {
+    pub function_name: String,
+    pub source_file: String,
+    pub line: u32,
+    pub col: u32,
+}
+
+/// Controls how the debugger advances through bytecode.
+#[derive(Debug, Clone, PartialEq)]
+pub enum StepMode {
+    /// Run freely until the next breakpoint.
+    Run,
+    /// Execute exactly one statement, then pause.
+    StepOver,
+    /// Step into function calls (pause on first instruction of callee).
+    StepInto,
+    /// Run until the current frame returns, then pause.
+    StepOut,
+}
+
+/// An event emitted by the VM when in debug mode.
+#[derive(Debug, Clone)]
+pub enum DebugEvent {
+    /// Execution paused at a breakpoint.
+    Breakpoint {
+        offset: usize,
+        frame: StackFrame,
+    },
+    /// Execution paused after a single step.
+    Step {
+        frame: StackFrame,
+        locals: HashMap<String, Value>,
+    },
+    /// A function returned a value.
+    Return {
+        value: Value,
+    },
+    /// The VM encountered a runtime error.
+    Error {
+        message: String,
+        frame: StackFrame,
+    },
+}
+
+/// The debugger attached to a running VM instance.
+///
+/// In debug mode the interpreter queries [`should_pause`] before each
+/// instruction. If it returns `true`, execution stops and a [`DebugEvent`]
+/// is emitted to the registered handler.
+pub struct Debugger {
+    /// Bytecode offsets at which to pause execution.
+    pub breakpoints: HashSet<usize>,
+    /// Current stepping mode.
+    pub step_mode: StepMode,
+    /// Simulated call stack (maintained by the interpreter).
+    pub call_stack: Vec<StackFrame>,
+    /// Snapshot of local variables at the last pause.
+    pub locals: HashMap<String, Value>,
+    /// Events emitted since the last [`drain_events`] call.
+    events: Vec<DebugEvent>,
+}
+
+impl Debugger {
+    /// Create a new debugger that will break on the very first instruction.
+    pub fn new() -> Self {
+        Self {
+            breakpoints: HashSet::new(),
+            step_mode: StepMode::StepOver,
+            call_stack: vec![StackFrame {
+                function_name: "<top>".into(),
+                source_file: "<unknown>".into(),
+                line: 1,
+                col: 1,
+            }],
+            locals: HashMap::new(),
+            events: Vec::new(),
+        }
+    }
+
+    /// Add a breakpoint at a bytecode offset.
+    pub fn add_breakpoint(&mut self, offset: usize) {
+        self.breakpoints.insert(offset);
+    }
+
+    /// Remove a breakpoint.
+    pub fn remove_breakpoint(&mut self, offset: usize) {
+        self.breakpoints.remove(&offset);
+    }
+
+    /// Returns `true` if the debugger should pause at `offset`.
+    pub fn should_pause(&self, offset: usize) -> bool {
+        if self.breakpoints.contains(&offset) {
+            return true;
+        }
+        matches!(self.step_mode, StepMode::StepOver | StepMode::StepInto)
+    }
+
+    /// Called by the interpreter when it pauses at `offset`.
+    pub fn on_pause(&mut self, offset: usize, locals: HashMap<String, Value>) {
+        self.locals = locals.clone();
+        let frame = self.current_frame_cloned();
+        if self.breakpoints.contains(&offset) {
+            self.events.push(DebugEvent::Breakpoint { offset, frame });
+        } else {
+            self.events.push(DebugEvent::Step { frame, locals });
+        }
+    }
+
+    /// Called when a function is entered.
+    pub fn push_frame(&mut self, function_name: String, source_file: String) {
+        self.call_stack.push(StackFrame {
+            function_name,
+            source_file,
+            line: 1,
+            col: 1,
+        });
+    }
+
+    /// Called when a function returns.
+    pub fn pop_frame(&mut self, value: Value) {
+        self.call_stack.pop();
+        self.events.push(DebugEvent::Return { value });
+    }
+
+    /// Record a runtime error.
+    pub fn on_error(&mut self, message: String) {
+        let frame = self.current_frame_cloned();
+        self.events.push(DebugEvent::Error { message, frame });
+    }
+
+    /// Update the source position of the top frame.
+    pub fn update_position(&mut self, line: u32, col: u32) {
+        if let Some(frame) = self.call_stack.last_mut() {
+            frame.line = line;
+            frame.col = col;
+        }
+    }
+
+    /// Drain and return all queued events.
+    pub fn drain_events(&mut self) -> Vec<DebugEvent> {
+        std::mem::take(&mut self.events)
+    }
+
+    /// Current frame clone, or a sentinel if the stack is empty.
+    fn current_frame_cloned(&self) -> StackFrame {
+        self.call_stack.last().cloned().unwrap_or_else(|| StackFrame {
+            function_name: String::new(),
+            source_file: String::new(),
+            line: 0,
+            col: 0,
+        })
+    }
+}
+
+impl Default for Debugger {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_breakpoint_triggers_pause() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::Run; // not stepping — only breakpoints
+        dbg.add_breakpoint(5);
+        assert!(!dbg.should_pause(0));
+        assert!(!dbg.should_pause(4));
+        assert!(dbg.should_pause(5));
+        assert!(!dbg.should_pause(6));
+    }
+
+    #[test]
+    fn test_step_over_always_pauses() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::StepOver;
+        assert!(dbg.should_pause(0));
+        assert!(dbg.should_pause(99));
+    }
+
+    #[test]
+    fn test_step_into_always_pauses() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::StepInto;
+        assert!(dbg.should_pause(0));
+    }
+
+    #[test]
+    fn test_run_mode_no_pause_without_breakpoint() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::Run;
+        assert!(!dbg.should_pause(42));
+    }
+
+    #[test]
+    fn test_remove_breakpoint() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::Run;
+        dbg.add_breakpoint(10);
+        assert!(dbg.should_pause(10));
+        dbg.remove_breakpoint(10);
+        assert!(!dbg.should_pause(10));
+    }
+
+    #[test]
+    fn test_on_pause_emits_step_event() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::StepOver;
+        let mut locals = HashMap::new();
+        locals.insert("x".into(), Value::Int(42));
+        dbg.on_pause(0, locals.clone());
+        let events = dbg.drain_events();
+        assert_eq!(events.len(), 1);
+        assert!(matches!(events[0], DebugEvent::Step { .. }));
+    }
+
+    #[test]
+    fn test_on_pause_at_breakpoint_emits_breakpoint_event() {
+        let mut dbg = Debugger::new();
+        dbg.step_mode = StepMode::Run;
+        dbg.add_breakpoint(7);
+        dbg.on_pause(7, HashMap::new());
+        let events = dbg.drain_events();
+        assert_eq!(events.len(), 1);
+        assert!(matches!(events[0], DebugEvent::Breakpoint { offset: 7, .. }));
+    }
+
+    #[test]
+    fn test_push_pop_frame() {
+        let mut dbg = Debugger::new();
+        dbg.push_frame("my_fn".into(), "test.el".into());
+        assert_eq!(dbg.call_stack.len(), 2);
+        assert_eq!(dbg.call_stack[1].function_name, "my_fn");
+        dbg.pop_frame(Value::Int(0));
+        assert_eq!(dbg.call_stack.len(), 1);
+        let events = dbg.drain_events();
+        assert!(matches!(events[0], DebugEvent::Return { .. }));
+    }
+
+    #[test]
+    fn test_on_error_emits_error_event() {
+        let mut dbg = Debugger::new();
+        dbg.on_error("division by zero".into());
+        let events = dbg.drain_events();
+        assert!(matches!(&events[0], DebugEvent::Error { message, .. } if message == "division by zero"));
+    }
+
+    #[test]
+    fn test_drain_clears_events() {
+        let mut dbg = Debugger::new();
+        dbg.on_error("oops".into());
+        let _ = dbg.drain_events();
+        let events2 = dbg.drain_events();
+        assert!(events2.is_empty());
+    }
+
+    #[test]
+    fn test_update_position() {
+        let mut dbg = Debugger::new();
+        dbg.update_position(10, 5);
+        assert_eq!(dbg.call_stack[0].line, 10);
+        assert_eq!(dbg.call_stack[0].col, 5);
+    }
+}
@@ -24,10 +24,12 @@
 mod bytecode;
 mod codegen;
 mod compiler;
+mod debugger;
 mod error;
 mod source_map;

 pub use bytecode::{Bytecode, Value};
 pub use compiler::{CompileOutput, Compiler, CompilerOptions, Target};
+pub use debugger::{DebugEvent, Debugger, StackFrame, StepMode};
 pub use error::{CompileError, CompileResult};
 pub use source_map::SourceMap;