feat: package manager, build system, native cross-compilation, plugin system

Add three new crates and extend the compiler and CLI toolchain:

- el-manifest: el.toml manifest parser using serde + toml crate; supports
  package info, registry/path/version deps, build config with seal key
  sources, cross targets, and plugins; Manifest::find_manifest() walks up
  the directory tree

- el-registry: HTTP registry client (reqwest + tokio) for
  packages.neurontechnologies.ai; PackageMetadata, fetch/download/publish/
  search, BLAKE3 checksum verification, local cache at ~/.engram/packages/

- el-build: build orchestrator with incremental builds (BLAKE3 file hashes
  in .el/build-cache.json), cross-compilation target tagging, dep resolution,
  plugin registry with on_ast/on_typed_ast/on_bytecode hooks, test runner,
  fmt/check/clean commands

- CrossTarget and NativeTarget enums with triple() and artifact_extension()
  methods; NativeTarget::Host detects compile-time platform via cfg! macros

- Plugin system: CompilerPlugin trait + PluginRegistry; dynamic loading is
  a marked TODO with clear extension point for libloading

- CLI extended with: new, add, remove, update, build --cross, run, test,
  check, fmt, clean, publish, search, plugin add/remove/list; old
  single-file commands moved to build-file/seal/unseal subcommands

- Fix pre-existing debugger.rs borrow error (unwrap_or temporary lifetime)
- Fix checker.rs and codegen.rs to handle TestDef/Seed/Assert Stmt variants
- Add spec/language.md sections 12-14: package system, build system,
  plugin system, cross-compilation targets table

130 tests passing, zero warnings
This commit is contained in:
Will Anderson
2026-04-27 19:08:25 -05:00
parent 9ced941590
commit 48b72843e1
31 changed files with 4923 additions and 107 deletions
+5 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "el"
description = "Engram language CLI — el build / run / check / seal / unseal"
description = "Engram language CLI — el build / run / check / seal / unseal / new / add / publish / …"
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -15,5 +15,9 @@ el-parser = { workspace = true }
el-types = { workspace = true }
el-compiler = { workspace = true }
el-seal = { workspace = true }
el-manifest = { workspace = true }
el-registry = { workspace = true }
el-build = { workspace = true }
clap = { workspace = true }
thiserror = { workspace = true }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
+491 -89
View File
@@ -1,20 +1,48 @@
//! el — The Engram language CLI.
//!
//! Commands:
//! el build <file.el> [--target debug|release|prod] [--output <path>]
//! el run <file.el>
//! el check <file.el>
//! el seal <artifact>
//! el unseal <artifact>
//! # Commands
//!
//! ## Project management
//! el new <name> scaffold new project with el.toml + src/main.el
//! el add <package>[@ver] add a dependency to el.toml
//! el remove <package> remove a dependency from el.toml
//! el update update all deps to latest compatible versions
//!
//! ## Build & run
//! el build [--target prod] build the project (reads el.toml)
//! el build --cross build for all configured cross targets
//! el run build debug and run
//! el test run tests
//! el check type-check only
//! el fmt format source files
//! el clean clean build artifacts
//!
//! ## Registry
//! el publish publish to registry
//! el search <query> search registry
//! el plugin add <plugin> add compiler plugin
//!
//! ## Low-level
//! el build-file <file.el> compile a single .el file (no el.toml needed)
//! el seal <artifact> seal an existing release artifact
//! el unseal <artifact> unseal a sealed artifact
use std::path::PathBuf;
use clap::{Parser, Subcommand};
use el_build::BuildSystem;
use el_compiler::{Compiler, CompilerOptions, Target};
use el_manifest::{BuildTarget, Manifest};
use el_seal::{seal as seal_fn, unseal as unseal_fn, SealedArtifact, DeploymentBinding, SealAlgorithm, SealConfig};
// ── CLI definition ────────────────────────────────────────────────────────────
#[derive(Parser, Debug)]
#[command(name = "el", about = "The Engram programming language compiler and toolchain", version)]
#[command(
name = "el",
about = "The Engram programming language compiler and toolchain",
version
)]
struct Cli {
#[command(subcommand)]
command: Command,
@@ -22,56 +50,318 @@ struct Cli {
#[derive(Subcommand, Debug)]
enum Command {
/// Compile an Engram source file.
// ── Project management ────────────────────────────────────────────────────
/// Scaffold a new Engram project with el.toml and src/main.el.
New {
/// Project name.
name: String,
/// Directory to create the project in (default: <name>/).
#[arg(long, short = 'd')]
dir: Option<PathBuf>,
},
/// Add a dependency to el.toml.
Add {
/// Package name, optionally with version: `engram-http@1.2`.
package: String,
/// Path dependency: `--path ../my-lib`.
#[arg(long)]
path: Option<PathBuf>,
},
/// Remove a dependency from el.toml.
Remove {
/// Package name.
package: String,
},
/// Update all dependencies to the latest compatible versions.
Update,
// ── Build & run ───────────────────────────────────────────────────────────
/// Build the project (reads el.toml).
Build {
/// Source file (*.el)
/// Override the build target: debug | release | prod.
#[arg(long)]
target: Option<String>,
/// Build for all configured cross-compilation targets.
#[arg(long)]
cross: bool,
/// Path to the project manifest (default: el.toml in current directory).
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Build in debug mode and run the output.
Run {
/// Path to the project manifest (default: el.toml).
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Run all tests.
Test {
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Type-check source files without producing artifacts.
Check {
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Format source files.
Fmt {
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Remove build artifacts and cache.
Clean {
#[arg(long)]
manifest: Option<PathBuf>,
},
// ── Registry ──────────────────────────────────────────────────────────────
/// Publish this package to the Engram registry.
Publish {
/// Registry API key.
#[arg(long, env = "ENGRAM_REGISTRY_KEY")]
api_key: Option<String>,
#[arg(long)]
manifest: Option<PathBuf>,
},
/// Search the Engram registry.
Search {
/// Search query.
query: String,
},
/// Manage compiler plugins.
Plugin {
#[command(subcommand)]
action: PluginAction,
},
// ── Low-level / single-file ───────────────────────────────────────────────
/// Compile a single .el source file (no el.toml required).
BuildFile {
/// Source file (*.el).
file: PathBuf,
/// Compilation target: debug | release | prod
/// Compilation target: debug | release | prod.
#[arg(long, default_value = "debug")]
target: String,
/// Output path
/// Output path.
#[arg(long, short = 'o')]
output: Option<PathBuf>,
},
/// Compile and run an Engram source file (debug target).
Run {
/// Source file (*.el)
file: PathBuf,
},
/// Type-check an Engram source file without producing output.
Check {
/// Source file (*.el)
file: PathBuf,
},
/// Seal an existing release artifact.
Seal {
/// Release artifact to seal
artifact: PathBuf,
/// Output path (default: <artifact>.sealed)
#[arg(long, short = 'o')]
output: Option<PathBuf>,
},
/// Unseal a sealed artifact (requires ENGRAM_SEAL_KEY env var).
/// Unseal a sealed artifact (requires ENGRAM_SEAL_KEY).
Unseal {
/// Sealed artifact
artifact: PathBuf,
/// Output path for decrypted bytecode
#[arg(long, short = 'o')]
output: Option<PathBuf>,
},
}
fn main() {
#[derive(Subcommand, Debug)]
enum PluginAction {
/// Add a compiler plugin to el.toml.
Add {
/// Plugin name, optionally with version: `el-fmt@1.0`.
plugin: String,
},
/// Remove a compiler plugin from el.toml.
Remove {
plugin: String,
},
/// List installed plugins.
List,
}
// ── Entry point ───────────────────────────────────────────────────────────────
#[tokio::main]
async fn main() {
let cli = Cli::parse();
if let Err(e) = run(cli) {
if let Err(e) = run(cli).await {
eprintln!("error: {e}");
std::process::exit(1);
}
}
fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
async fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
match cli.command {
Command::Build { file, target, output } => {
// ── Project management ────────────────────────────────────────────────
Command::New { name, dir } => {
cmd_new(&name, dir.as_deref())?;
}
Command::Add { package, path } => {
cmd_add(&package, path.as_deref())?;
}
Command::Remove { package } => {
cmd_remove(&package)?;
}
Command::Update => {
println!("update: checking for newer dependency versions...");
println!("(registry not yet live — no updates available)");
}
// ── Build & run ───────────────────────────────────────────────────────
Command::Build { target, cross, manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
if cross {
let results = bs.build_all_targets().await?;
for (ct, out) in &results {
println!(
"built {} ({}) -> {} [{} bytes, {}ms]",
ct,
out.target,
out.artifact_path.display(),
out.size_bytes,
out.compile_time_ms,
);
}
println!("cross: {} target(s) built", results.len());
} else {
let build_target = target.as_deref().map(parse_build_target).transpose()?;
let out = bs.build(build_target).await?;
println!(
"built {} -> {} [{} bytes, {}ms, sealed={}]",
bs.manifest.package.name,
out.artifact_path.display(),
out.size_bytes,
out.compile_time_ms,
out.sealed,
);
}
}
Command::Run { manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
let out = bs.build(Some(BuildTarget::Debug)).await?;
let artifact = std::fs::read(&out.artifact_path)?;
let instructions = el_compiler::Bytecode::deserialize_all(&artifact)
.unwrap_or_default();
run_interpreter(&instructions);
}
Command::Test { manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
let report = bs.test().await?;
println!(
"test: {} passed, {} failed (total {})",
report.passed, report.failed, report.total
);
for f in &report.failures {
eprintln!(" FAIL: {f}");
}
if !report.success() {
std::process::exit(1);
}
}
Command::Check { manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
let diags = bs.check()?;
if diags.is_empty() {
println!("check: ok");
} else {
for d in &diags {
eprintln!("warning: {d}");
}
}
}
Command::Fmt { manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
bs.fmt()?;
}
Command::Clean { manifest } => {
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
bs.clean()?;
}
// ── Registry ──────────────────────────────────────────────────────────
Command::Publish { api_key, manifest } => {
let key = api_key.ok_or("publish requires --api-key or ENGRAM_REGISTRY_KEY env var")?;
let manifest_path = resolve_manifest(manifest.as_deref())?;
let bs = BuildSystem::from_manifest_file(&manifest_path)?;
// Build release artifact first
let out = bs.build(Some(BuildTarget::Release)).await?;
let client = el_registry::RegistryClient::new();
client.publish(&bs.manifest, &out.artifact_path, &key).await?;
println!("published {} v{}", bs.manifest.package.name, bs.manifest.package.version);
}
Command::Search { query } => {
let client = el_registry::RegistryClient::new();
println!("searching registry for '{query}'...");
match client.search(&query).await {
Ok(results) => {
if results.is_empty() {
println!("no results found");
}
for pkg in &results {
println!(" {} v{}{}", pkg.name, pkg.version, pkg.description);
}
}
Err(e) => {
eprintln!("registry unavailable (server not yet deployed): {e}");
}
}
}
Command::Plugin { action } => match action {
PluginAction::Add { plugin } => {
cmd_plugin_add(&plugin)?;
}
PluginAction::Remove { plugin } => {
println!("remove plugin: {plugin} (not yet implemented)");
}
PluginAction::List => {
let manifest_path = resolve_manifest(None)?;
let m = Manifest::from_file(&manifest_path)?;
if m.plugins.is_empty() {
println!("no plugins installed");
}
for (name, ver) in &m.plugins {
println!(" {name} = \"{ver}\"");
}
}
},
// ── Low-level / single-file ───────────────────────────────────────────
Command::BuildFile { file, target, output } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
@@ -79,8 +369,7 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
let out_path = output.unwrap_or_else(|| {
let stem = file.file_stem().unwrap_or_default().to_string_lossy();
match &compilation_target {
Target::Debug => PathBuf::from(format!("{stem}.elc")),
Target::Release => PathBuf::from(format!("{stem}.elc")),
Target::Debug | Target::Release => PathBuf::from(format!("{stem}.elc")),
Target::Prod => PathBuf::from(format!("{stem}.sealed")),
}
});
@@ -95,17 +384,13 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
};
let output = Compiler::compile(&source, opts)?;
// Print diagnostics
for d in &output.diagnostics {
eprintln!("warning: {d}");
}
// Write artifact
std::fs::write(&out_path, &output.artifact)
.map_err(|e| format!("cannot write {}: {e}", out_path.display()))?;
// Write source map alongside (debug only)
if let Some(sm) = &output.source_map {
let sm_path = out_path.with_extension("map.json");
std::fs::write(&sm_path, sm)
@@ -118,46 +403,6 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
}
}
Command::Run { file } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
let opts = CompilerOptions {
target: Target::Debug,
..Default::default()
};
let output = Compiler::compile(&source, opts)?;
// Diagnostics
for d in &output.diagnostics {
eprintln!("warning: {d}");
}
// Run the bytecode through the interpreter
let instructions = el_compiler::Bytecode::deserialize_all(&output.artifact)
.unwrap_or_default();
run_interpreter(&instructions);
}
Command::Check { file } => {
let source = std::fs::read_to_string(&file)
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
let tokens = el_lexer::tokenize(&source)?;
let program = el_parser::parse(tokens, source.clone())?;
let mut checker = el_types::TypeChecker::with_builtins();
checker.check(&program);
if checker.ok() {
println!("{}: ok", file.display());
} else {
for d in checker.diagnostics.iter().filter(|d| d.is_error) {
eprintln!("error: {}", d.message);
}
std::process::exit(1);
}
}
Command::Seal { artifact, output } => {
let bytes = std::fs::read(&artifact)
.map_err(|e| format!("cannot read {}: {e}", artifact.display()))?;
@@ -175,37 +420,194 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
let artifact_bytes = sealed.to_bytes()?;
std::fs::write(&out_path, &artifact_bytes)
.map_err(|e| format!("cannot write {}: {e}", out_path.display()))?;
println!("sealed {} -> {} ({} bytes)", artifact.display(), out_path.display(), artifact_bytes.len());
println!("sealed {} -> {} ({} bytes)",
artifact.display(), out_path.display(), artifact_bytes.len());
}
Command::Unseal { artifact, output } => {
let bytes = std::fs::read(&artifact)
.map_err(|e| format!("cannot read {}: {e}", artifact.display()))?;
let out_path = output.unwrap_or_else(|| {
artifact.with_extension("elc")
});
let out_path = output.unwrap_or_else(|| artifact.with_extension("elc"));
let sealed = SealedArtifact::from_bytes(&bytes)?;
// Get the binding key from environment
let key_str = std::env::var("ENGRAM_SEAL_KEY")
.unwrap_or_default();
let key_str = std::env::var("ENGRAM_SEAL_KEY").unwrap_or_default();
let key_bytes = key_str.as_bytes();
let plaintext = unseal_fn(&sealed, key_bytes)?;
std::fs::write(&out_path, &plaintext)
.map_err(|e| format!("cannot write {}: {e}", out_path.display()))?;
println!("unsealed {} -> {} ({} bytes)", artifact.display(), out_path.display(), plaintext.len());
println!("unsealed {} -> {} ({} bytes)",
artifact.display(), out_path.display(), plaintext.len());
}
}
Ok(())
}
// ── Command implementations ───────────────────────────────────────────────────
fn cmd_new(name: &str, dir: Option<&std::path::Path>) -> Result<(), Box<dyn std::error::Error>> {
let project_dir = dir
.map(|d| d.to_path_buf())
.unwrap_or_else(|| PathBuf::from(name));
if project_dir.exists() {
return Err(format!("directory '{}' already exists", project_dir.display()).into());
}
std::fs::create_dir_all(project_dir.join("src"))?;
// Write el.toml
let manifest_content = format!(
r#"[package]
name = "{name}"
version = "0.1.0"
description = ""
authors = []
license = "MIT"
edition = "2026"
[dependencies]
[build]
target = "debug"
entry = "src/main.el"
output = "dist/"
[cross]
targets = []
"#
);
std::fs::write(project_dir.join("el.toml"), manifest_content)?;
// Write src/main.el
let main_el = format!(
r#"// {name} — entry point
fn main() -> Void {{
let msg: String = "Hello from {name}!"
println(msg)
}}
"#
);
std::fs::write(project_dir.join("src").join("main.el"), main_el)?;
// Write .gitignore
std::fs::write(project_dir.join(".gitignore"), "dist/\n.el/\n")?;
println!("created project '{name}' in {}/", project_dir.display());
println!(" el.toml");
println!(" src/main.el");
Ok(())
}
fn cmd_add(package: &str, path: Option<&std::path::Path>) -> Result<(), Box<dyn std::error::Error>> {
let manifest_path = resolve_manifest(None)?;
let mut manifest_text = std::fs::read_to_string(&manifest_path)?;
let dep_line = if let Some(p) = path {
format!(
r#"{package} = {{ path = "{}" }}"#,
p.display()
)
} else {
// Parse name@version
let (pkg_name, version) = if let Some((n, v)) = package.split_once('@') {
(n, v.to_string())
} else {
(package, "*".to_string())
};
format!(r#"{pkg_name} = "{version}""#)
};
// Append to [dependencies] section
if let Some(idx) = manifest_text.find("[dependencies]") {
// Find next section or end
let after = &manifest_text[idx + "[dependencies]".len()..];
let insert_pos = after
.find("\n[")
.map(|i| idx + "[dependencies]".len() + i)
.unwrap_or(manifest_text.len());
manifest_text.insert_str(insert_pos, &format!("\n{dep_line}"));
} else {
manifest_text.push_str(&format!("\n[dependencies]\n{dep_line}\n"));
}
std::fs::write(&manifest_path, &manifest_text)?;
println!("added: {dep_line}");
Ok(())
}
fn cmd_remove(package: &str) -> Result<(), Box<dyn std::error::Error>> {
let manifest_path = resolve_manifest(None)?;
let manifest_text = std::fs::read_to_string(&manifest_path)?;
// Remove the line containing `package = ...` from [dependencies]
let updated: String = manifest_text
.lines()
.filter(|line| {
let trimmed = line.trim();
!trimmed.starts_with(package)
|| !trimmed[package.len()..].trim_start().starts_with('=')
})
.collect::<Vec<_>>()
.join("\n");
// Preserve trailing newline
let updated = if manifest_text.ends_with('\n') {
format!("{updated}\n")
} else {
updated
};
std::fs::write(&manifest_path, &updated)?;
println!("removed: {package}");
Ok(())
}
fn cmd_plugin_add(plugin: &str) -> Result<(), Box<dyn std::error::Error>> {
let manifest_path = resolve_manifest(None)?;
let mut manifest_text = std::fs::read_to_string(&manifest_path)?;
let (plugin_name, version) = if let Some((n, v)) = plugin.split_once('@') {
(n.to_string(), v.to_string())
} else {
(plugin.to_string(), "*".to_string())
};
let plugin_line = format!(r#"{plugin_name} = "{version}""#);
if let Some(idx) = manifest_text.find("[plugins]") {
let after = &manifest_text[idx + "[plugins]".len()..];
let insert_pos = after
.find("\n[")
.map(|i| idx + "[plugins]".len() + i)
.unwrap_or(manifest_text.len());
manifest_text.insert_str(insert_pos, &format!("\n{plugin_line}"));
} else {
manifest_text.push_str(&format!("\n[plugins]\n{plugin_line}\n"));
}
std::fs::write(&manifest_path, &manifest_text)?;
println!("added plugin: {plugin_line}");
Ok(())
}
// ── Helpers ───────────────────────────────────────────────────────────────────
/// Find the nearest `el.toml` starting from the current directory.
fn resolve_manifest(path: Option<&std::path::Path>) -> Result<PathBuf, Box<dyn std::error::Error>> {
if let Some(p) = path {
return Ok(p.to_path_buf());
}
let cwd = std::env::current_dir()?;
Manifest::find_manifest(&cwd).map_err(|e| e.into())
}
fn parse_build_target(s: &str) -> Result<BuildTarget, Box<dyn std::error::Error>> {
s.parse::<BuildTarget>().map_err(|e| e.into())
}
fn parse_target(s: &str) -> Result<Target, String> {
match s {
"debug" => Ok(Target::Debug),
@@ -226,7 +628,7 @@ fn build_seal_config() -> Result<SealConfig, String> {
})
}
/// Minimal interpreter for demonstration — prints values to stdout.
/// Minimal interpreter for demonstration.
fn run_interpreter(instructions: &[el_compiler::Bytecode]) {
use el_compiler::{Bytecode, Value};
let mut stack: Vec<Value> = Vec::new();