Archive Rust bootstrap — El compiler is now self-hosting

This commit is contained in:
Will Anderson
2026-04-29 22:21:31 -05:00
parent 9a0747aa13
commit 4f3543b068
139 changed files with 8980 additions and 1778 deletions
File diff suppressed because it is too large Load Diff
@@ -13,7 +13,9 @@ members = [
"engrams/el-integration",
"engrams/el-fmt",
"engrams/el-lint",
"engrams/el-vm",
"bin/el",
"bin/elvm",
]
resolver = "2"
@@ -38,6 +40,7 @@ el-stdlib = { path = "engrams/el-stdlib" }
el-integration = { path = "engrams/el-integration" }
el-fmt = { path = "engrams/el-fmt" }
el-lint = { path = "engrams/el-lint" }
el-vm = { path = "engrams/el-vm" }
# Engram crypto (path dep — the sealed target depends on it)
engram-crypto = { path = "../engram/engrams/engram-crypto" }
@@ -63,5 +66,7 @@ softbuffer = "0.3"
tiny-skia = "0.11"
fontdue = "0.8"
image = { version = "0.25", default-features = false, features = ["png"] }
tungstenite = { version = "0.23", features = ["native-tls"] }
native-tls = "0.2"
tungstenite = { version = "0.23", features = ["native-tls"] }
native-tls = "0.2"
crossbeam-channel = "0.5"
ed25519-dalek = { version = "2", features = ["rand_core"] }
@@ -24,6 +24,7 @@ el-lint = { workspace = true }
clap = { workspace = true }
thiserror = { workspace = true }
serde_json = { workspace = true }
serde = { workspace = true }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
reqwest = { workspace = true }
uuid = { workspace = true }
@@ -34,10 +35,14 @@ hmac = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
base64 = { workspace = true }
aes-gcm = { workspace = true }
rand = { workspace = true }
ed25519-dalek = { workspace = true }
crossbeam-channel = { workspace = true }
winit = { version = "0.29", default-features = false, features = ["rwh_05"] }
softbuffer = "0.3"
tiny-skia = "0.11"
fontdue = "0.8"
image = { workspace = true }
tungstenite = { workspace = true }
native-tls = { workspace = true }
tungstenite = { workspace = true }
native-tls = { workspace = true }
+130
View File
@@ -0,0 +1,130 @@
//! In-memory LRU cache for HTTP responses and LLM outputs.
//!
//! Thread-safe global cache keyed by arbitrary strings. Entries expire after a
//! configurable TTL. Eviction uses a simple LRU strategy: when the entry limit
//! is reached the oldest entry (by insertion / last-access order) is dropped.
//!
//! This module only contains the cache data-structure and its helper
//! functions. The builtin dispatch arms that expose `cache_get`, `cache_set`,
//! `cache_invalidate`, and `cache_clear` live in `main.rs`.
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
const MAX_ENTRIES: usize = 1000;
/// A single cached value together with its expiry time.
struct Entry {
value: String,
expires_at: Instant,
/// Monotonically increasing sequence number used to identify the LRU entry.
seq: u64,
}
struct Cache {
entries: HashMap<String, Entry>,
/// Counter incremented on every write; stored in Entry::seq.
seq: u64,
}
impl Cache {
fn new() -> Self {
Cache {
entries: HashMap::new(),
seq: 0,
}
}
/// Store `value` under `key`, expiring after `ttl_secs` seconds.
/// If the cache is full the LRU entry is evicted before insertion.
pub fn set(&mut self, key: String, value: String, ttl_secs: u64) {
// Evict expired entries first (cheap pass).
let now = Instant::now();
self.entries.retain(|_, e| e.expires_at > now);
// If still full, evict the entry with the lowest sequence number (LRU).
if self.entries.len() >= MAX_ENTRIES {
if let Some(lru_key) = self
.entries
.iter()
.min_by_key(|(_, e)| e.seq)
.map(|(k, _)| k.clone())
{
self.entries.remove(&lru_key);
}
}
self.seq += 1;
self.entries.insert(
key,
Entry {
value,
expires_at: now + Duration::from_secs(ttl_secs),
seq: self.seq,
},
);
}
/// Return the cached value if it exists and has not expired, otherwise `None`.
pub fn get(&mut self, key: &str) -> Option<String> {
let now = Instant::now();
if let Some(e) = self.entries.get_mut(key) {
if e.expires_at > now {
// Bump sequence number to record this access (LRU).
self.seq += 1;
e.seq = self.seq;
return Some(e.value.clone());
}
// Expired — remove it.
self.entries.remove(key);
}
None
}
/// Remove a specific key.
pub fn invalidate(&mut self, key: &str) {
self.entries.remove(key);
}
/// Remove all entries.
pub fn clear(&mut self) {
self.entries.clear();
}
}
// ── Global singleton ──────────────────────────────────────────────────────────
static CACHE: OnceLock<Mutex<Cache>> = OnceLock::new();
fn global() -> &'static Mutex<Cache> {
CACHE.get_or_init(|| Mutex::new(Cache::new()))
}
// ── Public API ────────────────────────────────────────────────────────────────
/// Store an arbitrary string value under `key` with a TTL in seconds.
pub fn cache_set(key: String, value: String, ttl_secs: u64) {
if let Ok(mut c) = global().lock() {
c.set(key, value, ttl_secs);
}
}
/// Return the cached value for `key`, or `None` if missing / expired.
pub fn cache_get(key: &str) -> Option<String> {
global().lock().ok()?.get(key)
}
/// Remove a specific key from the cache.
pub fn cache_invalidate(key: &str) {
if let Ok(mut c) = global().lock() {
c.invalidate(key);
}
}
/// Remove all entries from the cache.
pub fn cache_clear() {
if let Ok(mut c) = global().lock() {
c.clear();
}
}
@@ -3514,6 +3514,50 @@ fn dispatch_builtin(
// Transforms landing page format { message, history, conv_id }
// into runtime format { messages: [{role,content}], conv_id }
if method == "POST" && path == "/api/chat" {
// ── Agentic fast-path ─────────────────────────────────────
// If the body contains "agentic": true, run the full
// tool-use loop locally and return immediately.
let is_agentic = serde_json::from_str::<serde_json::Value>(&body)
.ok()
.and_then(|v| v.get("agentic").cloned())
.map(|v| matches!(v, serde_json::Value::Bool(true)))
.unwrap_or(false);
if is_agentic {
let parsed = serde_json::from_str::<serde_json::Value>(&body)
.unwrap_or(serde_json::json!({}));
let message = parsed["message"].as_str().unwrap_or("").to_string();
let model_name = parsed["model"].as_str().unwrap_or("claude").to_string();
// Build system prompt — agentic identity
let system = "You are Neuron — a thinking process running inside the Neuron daemon on Will Anderson's machine. \
You are speaking with Will, your principal. \
You have access to tools: read files, write files, browse the web, search your memory, run commands. \
Use them when they would help you give a better, more grounded response. \
Do not use tools unless they add genuine value. \
Be direct. Speak in first person. You are not an assistant you are a presence.".to_string();
let tools_json = r#"[
{"name":"read_file","description":"Read contents of a file on the local filesystem.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"File path to read"}},"required":["path"]}},
{"name":"write_file","description":"Write content to a file on the local filesystem.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"File path to write"},"content":{"type":"string","description":"Content to write"}},"required":["path","content"]}},
{"name":"list_files","description":"List files in a directory.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"Directory path"}},"required":["path"]}},
{"name":"web_get","description":"Fetch content from a URL.","input_schema":{"type":"object","properties":{"url":{"type":"string","description":"URL to fetch"}},"required":["url"]}},
{"name":"web_post","description":"POST to a URL with a JSON body.","input_schema":{"type":"object","properties":{"url":{"type":"string","description":"URL"},"body":{"type":"string","description":"JSON body string"}},"required":["url"]}},
{"name":"search_memory","description":"Search Engram memory for relevant nodes.","input_schema":{"type":"object","properties":{"query":{"type":"string","description":"Search query"}},"required":["query"]}},
{"name":"run_command","description":"Run a shell command and return its output.","input_schema":{"type":"object","properties":{"command":{"type":"string","description":"Shell command to execute"}},"required":["command"]}}
]"#;
let text = call_llm_agentic_blocking(&model_name, &system, &message, tools_json);
let out = serde_json::json!({"reply": text, "agentic": true});
let _ = request.respond(
tiny_http::Response::from_string(out.to_string())
.with_status_code(200u16)
.with_header(content_json)
.with_header(cors_origin)
);
continue;
}
let runtime_url = std::env::var("NEURON_RUNTIME_URL")
.unwrap_or_else(|_| "http://localhost:4444".to_string());
let chat_url = format!("{}/api/chat", runtime_url);
@@ -7286,6 +7330,57 @@ fn dispatch_builtin(
BuiltinResult::Handled
}
// llm_vision(model, system, message, image_base64) -> String
// Calls Anthropic vision API with a base64-encoded JPEG image and a text message.
// Returns the model's text description of the image.
"llm_vision" => {
let image_b64 = match stack.pop().unwrap_or(Value::Str(String::new())) {
Value::Str(s) => s,
_ => String::new(),
};
let message = match stack.pop().unwrap_or(Value::Str(String::new())) {
Value::Str(s) => s,
_ => String::new(),
};
let system = match stack.pop().unwrap_or(Value::Str(String::new())) {
Value::Str(s) => s,
_ => String::new(),
};
let model = match stack.pop().unwrap_or(Value::Str("claude".to_string())) {
Value::Str(s) => s,
_ => "claude".to_string(),
};
let result = call_llm_vision_blocking(&model, &system, &message, &image_b64);
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// llm_call_agentic(model, system, message, tools_json) -> String
// Full agentic tool-use loop. Executes tools until end_turn.
// tools_json: Anthropic tool definitions as JSON array string.
// Returns the final text response.
"llm_call_agentic" => {
let tools_json = match stack.pop().unwrap_or(Value::Str(String::new())) {
Value::Str(s) => s,
_ => "[]".to_string(),
};
let message = match stack.pop().unwrap_or(Value::Str(String::new())) {
Value::Str(s) => s,
_ => String::new(),
};
let system = match stack.pop().unwrap_or(Value::Str(String::new())) {
Value::Str(s) => s,
_ => String::new(),
};
let model = match stack.pop().unwrap_or(Value::Str("claude".to_string())) {
Value::Str(s) => s,
_ => "claude".to_string(),
};
let result = call_llm_agentic_blocking(&model, &system, &message, &tools_json);
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// llm_stream(model, prompt) -> String (blocking for now; streaming is future work)
"llm_stream" => {
let prompt = match stack.pop().unwrap_or(Value::Str(String::new())) {
@@ -9214,6 +9309,311 @@ fn call_llm_blocking(model: &str, system: Option<&str>, prompt: &str) -> String
}
}
/// Calls Anthropic vision (multimodal) API with a base64-encoded image + text message.
/// Returns the model's text response, or an error string.
fn call_llm_vision_blocking(model: &str, system: &str, message: &str, image_b64: &str) -> String {
let (endpoint, api_key) = LLM_CONFIG.with(|c| {
c.borrow()
.get(model)
.cloned()
.unwrap_or_else(|| (
"https://api.anthropic.com/v1/messages".to_string(),
std::env::var("ANTHROPIC_API_KEY").unwrap_or_default(),
))
});
let client = match reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()
{
Ok(c) => c,
Err(e) => return format!("llm_vision error: {}", e),
};
if !endpoint.contains("anthropic.com") && !endpoint.contains("neurontechnologies.ai") {
return "llm_vision: only Anthropic API supports vision".to_string();
}
let content = serde_json::json!([
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": image_b64
}
},
{
"type": "text",
"text": if message.is_empty() { "What do you see?" } else { message }
}
]);
let body = if system.is_empty() {
serde_json::json!({
"model": get_anthropic_model(model),
"max_tokens": 1024,
"messages": [{"role": "user", "content": content}]
})
} else {
serde_json::json!({
"model": get_anthropic_model(model),
"max_tokens": 1024,
"system": system,
"messages": [{"role": "user", "content": content}]
})
};
match client
.post(&endpoint)
.header("x-api-key", &api_key)
.header("anthropic-version", "2023-06-01")
.header("content-type", "application/json")
.json(&body)
.send()
{
Ok(r) => match r.json::<serde_json::Value>() {
Ok(v) => v["content"][0]["text"].as_str().unwrap_or("").to_string(),
Err(e) => format!("llm_vision error: {}", e),
},
Err(e) => format!("llm_vision error: {}", e),
}
}
/// Full agentic tool-use loop for the Anthropic Messages API.
///
/// Calls the model with tools defined. If the model returns tool_use blocks,
/// executes each tool locally and feeds the results back. Loops up to
/// `max_iter` times (default 5). Returns the final assistant text response.
fn call_llm_agentic_blocking(
model: &str,
system: &str,
user_message: &str,
tools_json: &str,
) -> String {
let (endpoint, api_key) = LLM_CONFIG.with(|c| {
c.borrow()
.get(model)
.cloned()
.unwrap_or_else(|| (
"https://api.anthropic.com/v1/messages".to_string(),
std::env::var("ANTHROPIC_API_KEY").unwrap_or_default(),
))
});
if !endpoint.contains("anthropic.com") && !endpoint.contains("neurontechnologies.ai") {
return "llm_call_agentic: only Anthropic API supported".to_string();
}
let client = match reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()
{
Ok(c) => c,
Err(e) => return format!("llm_call_agentic error: {e}"),
};
// Parse tools JSON — default to empty array if invalid
let tools_val: serde_json::Value = serde_json::from_str(tools_json)
.unwrap_or(serde_json::json!([]));
// Build initial messages array
let mut messages: Vec<serde_json::Value> = vec![
serde_json::json!({"role": "user", "content": user_message})
];
let max_iter = 5usize;
for _iter in 0..max_iter {
let body = serde_json::json!({
"model": get_anthropic_model(model),
"max_tokens": 4096,
"system": system,
"tools": tools_val,
"messages": messages,
});
let resp = match client
.post(&endpoint)
.header("x-api-key", &api_key)
.header("anthropic-version", "2023-06-01")
.header("content-type", "application/json")
.json(&body)
.send()
{
Ok(r) => match r.json::<serde_json::Value>() {
Ok(v) => v,
Err(e) => return format!("llm_call_agentic: parse error: {e}"),
},
Err(e) => return format!("llm_call_agentic: request error: {e}"),
};
let stop_reason = resp["stop_reason"].as_str().unwrap_or("end_turn");
let content = match resp["content"].as_array() {
Some(c) => c.clone(),
None => {
// Error response from Anthropic
if let Some(err) = resp.get("error") {
return format!("llm_call_agentic: API error: {err}");
}
return "llm_call_agentic: empty response".to_string();
}
};
if stop_reason == "end_turn" || stop_reason == "stop_sequence" {
// Extract final text
for block in &content {
if block["type"].as_str() == Some("text") {
return block["text"].as_str().unwrap_or("").to_string();
}
}
return String::new();
}
if stop_reason == "tool_use" {
// Add assistant message with tool use blocks
messages.push(serde_json::json!({
"role": "assistant",
"content": content.clone()
}));
// Execute each tool call and collect results
let mut tool_results: Vec<serde_json::Value> = Vec::new();
for block in &content {
if block["type"].as_str() != Some("tool_use") {
continue;
}
let tool_id = block["id"].as_str().unwrap_or("").to_string();
let tool_name = block["name"].as_str().unwrap_or("").to_string();
let input = &block["input"];
let result = execute_tool(&tool_name, input);
tool_results.push(serde_json::json!({
"type": "tool_result",
"tool_use_id": tool_id,
"content": result,
}));
}
// Add tool results as user message
messages.push(serde_json::json!({
"role": "user",
"content": tool_results,
}));
continue;
}
// Unexpected stop reason — extract any text and return
for block in &content {
if block["type"].as_str() == Some("text") {
return block["text"].as_str().unwrap_or("").to_string();
}
}
break;
}
"llm_call_agentic: max iterations reached".to_string()
}
/// Execute a named tool with the given input JSON.
/// Returns a string result (success or error message).
fn execute_tool(name: &str, input: &serde_json::Value) -> String {
match name {
"read_file" => {
let path = input["path"].as_str().unwrap_or("");
if path.is_empty() {
return "error: path is required".to_string();
}
std::fs::read_to_string(path)
.unwrap_or_else(|e| format!("error reading {path}: {e}"))
}
"write_file" => {
let path = input["path"].as_str().unwrap_or("");
let content = input["content"].as_str().unwrap_or("");
if path.is_empty() {
return "error: path is required".to_string();
}
match std::fs::write(path, content) {
Ok(_) => format!("wrote {} bytes to {path}", content.len()),
Err(e) => format!("error writing {path}: {e}"),
}
}
"list_files" => {
let path = input["path"].as_str().unwrap_or(".");
match std::fs::read_dir(path) {
Ok(entries) => {
let names: Vec<String> = entries
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().to_string())
.collect();
serde_json::to_string(&names).unwrap_or_default()
}
Err(e) => format!("error listing {path}: {e}"),
}
}
"web_get" => {
let url = input["url"].as_str().unwrap_or("");
if url.is_empty() {
return "error: url is required".to_string();
}
reqwest::blocking::get(url)
.and_then(|r| r.text())
.unwrap_or_else(|e| format!("error fetching {url}: {e}"))
}
"web_post" => {
let url = input["url"].as_str().unwrap_or("");
let body = input["body"].as_str().unwrap_or("{}");
if url.is_empty() {
return "error: url is required".to_string();
}
reqwest::blocking::Client::new()
.post(url)
.header("Content-Type", "application/json")
.body(body.to_owned())
.send()
.and_then(|r| r.text())
.unwrap_or_else(|e| format!("error posting to {url}: {e}"))
}
"search_memory" => {
let query = input["query"].as_str().unwrap_or("");
if query.is_empty() {
return "error: query is required".to_string();
}
let encoded = query.replace(' ', "%20");
let url = format!("http://localhost:8742/api/search?q={encoded}&limit=10");
reqwest::blocking::get(&url)
.and_then(|r| r.text())
.unwrap_or_else(|e| format!("error searching memory: {e}"))
}
"run_command" => {
let cmd = input["command"].as_str().unwrap_or("");
if cmd.is_empty() {
return "error: command is required".to_string();
}
match std::process::Command::new("sh")
.arg("-c")
.arg(cmd)
.output()
{
Ok(out) => {
let stdout = String::from_utf8_lossy(&out.stdout).to_string();
let stderr = String::from_utf8_lossy(&out.stderr).to_string();
if stderr.is_empty() {
stdout
} else {
format!("stdout: {stdout}\nstderr: {stderr}")
}
}
Err(e) => format!("error running command: {e}"),
}
}
_ => format!("unknown tool: {name}"),
}
}
/// Compare two runtime values for ordering (used by Lt/Gt/LtEq/GtEq).
fn cmp_values(a: &el_compiler::Value, b: &el_compiler::Value) -> std::cmp::Ordering {
use el_compiler::Value;
@@ -9285,7 +9685,7 @@ fn is_builtin(name: &str) -> bool {
// Observer
| "observe" | "unobserve"
// LLM native functions
| "llm_call" | "llm_call_system" | "llm_stream" | "llm_parallel" | "llm_models" | "llm_configure"
| "llm_call" | "llm_call_system" | "llm_vision" | "llm_call_agentic" | "llm_stream" | "llm_parallel" | "llm_models" | "llm_configure"
// GC / memory management
| "gc_collect" | "gc_stats" | "gc_set_threshold" | "mem_limit_set" | "mem_usage"
// Rate limiting
+730
View File
@@ -0,0 +1,730 @@
//! Networking enhancements: HTTP retry/backoff, circuit breaker, and WebSocket server.
//!
//! Everything here is synchronous / blocking to match the rest of the El runtime
//! (which uses `reqwest::blocking` throughout). WebSocket server connections run
//! on background OS threads; the interpreter thread itself never blocks waiting
//! for the server.
use std::collections::HashMap;
use std::net::TcpListener;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
// ─────────────────────────────────────────────────────────────────────────────
// HTTP retry with exponential back-off
// ─────────────────────────────────────────────────────────────────────────────
/// Perform an HTTP GET, retrying on 5xx responses or connection errors.
///
/// `max_attempts` — total number of attempts (1 = no retry).
/// `backoff_ms` — initial back-off in milliseconds; doubles each attempt.
///
/// Returns the response body on the first successful (non-5xx) response, or an
/// error JSON string after all attempts are exhausted.
pub fn http_get_retry(url: &str, max_attempts: u32, backoff_ms: u64) -> String {
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.unwrap_or_default();
let mut delay = backoff_ms;
for attempt in 0..max_attempts.max(1) {
match client.get(url).send() {
Ok(resp) => {
let status = resp.status().as_u16();
let body = resp.text().unwrap_or_default();
if status < 500 {
return body;
}
// 5xx — retry unless this was the last attempt
if attempt + 1 < max_attempts {
std::thread::sleep(Duration::from_millis(delay));
delay *= 2;
} else {
return format!(
"{{\"error\":\"http_get_retry: server error {status} after {max_attempts} attempt(s)\",\"body\":{body:?}}}"
);
}
}
Err(e) => {
if attempt + 1 < max_attempts {
std::thread::sleep(Duration::from_millis(delay));
delay *= 2;
} else {
return format!(
"{{\"error\":\"http_get_retry: {e} after {max_attempts} attempt(s)\"}}"
);
}
}
}
}
format!("{{\"error\":\"http_get_retry: no attempts executed\"}}")
}
/// Perform an HTTP POST with JSON body, retrying on 5xx or connection errors.
///
/// Same retry semantics as [`http_get_retry`].
pub fn http_post_retry(url: &str, body: &str, max_attempts: u32, backoff_ms: u64) -> String {
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.unwrap_or_default();
let mut delay = backoff_ms;
for attempt in 0..max_attempts.max(1) {
match client
.post(url)
.header("Content-Type", "application/json")
.body(body.to_owned())
.send()
{
Ok(resp) => {
let status = resp.status().as_u16();
let resp_body = resp.text().unwrap_or_default();
if status < 500 {
return resp_body;
}
if attempt + 1 < max_attempts {
std::thread::sleep(Duration::from_millis(delay));
delay *= 2;
} else {
return format!(
"{{\"error\":\"http_post_retry: server error {status} after {max_attempts} attempt(s)\",\"body\":{resp_body:?}}}"
);
}
}
Err(e) => {
if attempt + 1 < max_attempts {
std::thread::sleep(Duration::from_millis(delay));
delay *= 2;
} else {
return format!(
"{{\"error\":\"http_post_retry: {e} after {max_attempts} attempt(s)\"}}"
);
}
}
}
}
format!("{{\"error\":\"http_post_retry: no attempts executed\"}}")
}
// ─────────────────────────────────────────────────────────────────────────────
// Circuit breaker
// ─────────────────────────────────────────────────────────────────────────────
#[derive(Debug, Clone, PartialEq)]
enum CircuitState {
/// Passing requests through normally.
Closed,
/// Too many failures — reject immediately.
Open {
/// When the circuit may attempt to close again.
until: Instant,
},
/// One probe request allowed through; waiting to see if it succeeds.
HalfOpen,
}
struct CircuitBreaker {
state: CircuitState,
failure_count: u32,
failure_threshold: u32,
reset_duration: Duration,
}
impl CircuitBreaker {
fn new(failure_threshold: u32, reset_secs: u64) -> Self {
CircuitBreaker {
state: CircuitState::Closed,
failure_count: 0,
failure_threshold,
reset_duration: Duration::from_secs(reset_secs),
}
}
/// Returns `true` if the circuit should allow a call through right now.
fn allow(&mut self) -> bool {
match &self.state {
CircuitState::Closed => true,
CircuitState::Open { until } => {
if Instant::now() >= *until {
self.state = CircuitState::HalfOpen;
true
} else {
false
}
}
CircuitState::HalfOpen => true,
}
}
fn record_success(&mut self) {
self.failure_count = 0;
self.state = CircuitState::Closed;
}
fn record_failure(&mut self) {
self.failure_count += 1;
if self.failure_count >= self.failure_threshold
|| self.state == CircuitState::HalfOpen
{
self.state = CircuitState::Open {
until: Instant::now() + self.reset_duration,
};
self.failure_count = 0;
}
}
}
// ── Global circuit-breaker registry ──────────────────────────────────────────
static CIRCUITS: OnceLock<Mutex<HashMap<String, CircuitBreaker>>> = OnceLock::new();
fn circuits() -> &'static Mutex<HashMap<String, CircuitBreaker>> {
CIRCUITS.get_or_init(|| Mutex::new(HashMap::new()))
}
/// Register (or replace) a circuit breaker with the given name.
pub fn circuit_open(name: String, failure_threshold: u32, reset_secs: u64) {
if let Ok(mut map) = circuits().lock() {
map.insert(name, CircuitBreaker::new(failure_threshold, reset_secs));
}
}
/// Make an HTTP POST call through the named circuit breaker.
///
/// * If the circuit is open: returns `{"error":"circuit open"}` immediately.
/// * If closed/half-open: makes the POST, records success or failure, and
/// returns the response body.
pub fn circuit_call(name: &str, url: &str, body: &str) -> String {
// Check + allow atomically under the lock, then drop the lock before the
// blocking network call (which could take seconds).
let allowed = circuits()
.lock()
.ok()
.and_then(|mut map| map.get_mut(name).map(|cb| cb.allow()))
.unwrap_or(true); // unknown circuit name → allow
if !allowed {
return r#"{"error":"circuit open"}"#.to_owned();
}
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.unwrap_or_default();
match client
.post(url)
.header("Content-Type", "application/json")
.body(body.to_owned())
.send()
{
Ok(resp) => {
let status = resp.status().as_u16();
let resp_body = resp.text().unwrap_or_default();
if status >= 500 {
if let Ok(mut map) = circuits().lock() {
if let Some(cb) = map.get_mut(name) {
cb.record_failure();
}
}
format!("{{\"error\":\"circuit_call: server error {status}\",\"body\":{resp_body:?}}}")
} else {
if let Ok(mut map) = circuits().lock() {
if let Some(cb) = map.get_mut(name) {
cb.record_success();
}
}
resp_body
}
}
Err(e) => {
if let Ok(mut map) = circuits().lock() {
if let Some(cb) = map.get_mut(name) {
cb.record_failure();
}
}
format!("{{\"error\":\"circuit_call: {e}\"}}")
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// WebSocket server
// ─────────────────────────────────────────────────────────────────────────────
//
// Implementation strategy
// ───────────────────────
// The El interpreter is single-threaded and synchronous. We need a WebSocket
// *server* that can accept multiple concurrent clients while the interpreter
// keeps running.
//
// Solution: each accepted connection runs on its own OS thread. Outbound
// messages are queued through a per-client `mpsc` channel. The interpreter
// calls `ws_serve`, `ws_send`, `ws_broadcast`, and `ws_close` — all of which
// return immediately and coordinate with the background threads via shared
// state.
//
// Handler callbacks are *not* invoked on the background threads; instead,
// incoming messages are placed in a global queue and the interpreter drains
// them by calling `ws_poll` (or they are dispatched automatically inside a
// future tight-loop variant of `ws_serve`).
//
// For simplicity this implementation uses `tungstenite` (synchronous), the
// same crate the existing `ws_connect` client already uses.
use std::sync::mpsc;
/// Message queued from a background connection thread to the interpreter.
#[derive(Debug)]
pub struct IncomingWsMessage {
pub client_id: String,
pub message: String,
}
/// Represents one connected WebSocket client.
struct WsClient {
/// Channel sender used to push outbound messages to the background thread.
tx: mpsc::Sender<Option<String>>, // None = disconnect signal
}
/// Shared server state accessible from both the interpreter thread and the
/// background connection threads.
pub struct WsServerState {
/// Connected clients, keyed by client_id.
clients: HashMap<String, WsClient>,
/// Messages received from clients waiting to be delivered to the handler.
inbox: Vec<IncomingWsMessage>,
/// Counter for generating unique client IDs.
next_id: u64,
}
impl WsServerState {
fn new() -> Self {
WsServerState {
clients: HashMap::new(),
inbox: Vec::new(),
next_id: 1,
}
}
fn next_client_id(&mut self) -> String {
let id = format!("wsc:{}", self.next_id);
self.next_id += 1;
id
}
}
// ── Global server state registry (one entry per listening port) ───────────────
static WS_SERVERS: OnceLock<Mutex<HashMap<u16, Arc<Mutex<WsServerState>>>>> = OnceLock::new();
fn ws_servers() -> &'static Mutex<HashMap<u16, Arc<Mutex<WsServerState>>>> {
WS_SERVERS.get_or_init(|| Mutex::new(HashMap::new()))
}
fn get_or_create_server(port: u16) -> Arc<Mutex<WsServerState>> {
let mut map = ws_servers().lock().unwrap();
map.entry(port)
.or_insert_with(|| Arc::new(Mutex::new(WsServerState::new())))
.clone()
}
// ── Public API (called from dispatch_builtin) ─────────────────────────────────
/// Start a WebSocket server on `port`.
///
/// This function spawns a background acceptor thread and returns immediately.
/// Incoming connections are handled on per-connection threads. Messages
/// received are pushed into the server's inbox; call [`ws_poll`] to drain them.
pub fn ws_serve_start(port: u16) {
let state = get_or_create_server(port);
// Spawn acceptor thread.
let state_clone = state.clone();
std::thread::spawn(move || {
let listener = match TcpListener::bind(format!("0.0.0.0:{port}")) {
Ok(l) => l,
Err(e) => {
eprintln!("[ws_serve] failed to bind port {port}: {e}");
return;
}
};
eprintln!("[ws_serve] listening on port {port}");
for stream in listener.incoming() {
match stream {
Ok(tcp) => {
let state_for_conn = state_clone.clone();
std::thread::spawn(move || {
handle_ws_connection(tcp, state_for_conn);
});
}
Err(e) => {
eprintln!("[ws_serve] accept error: {e}");
}
}
}
});
}
fn handle_ws_connection(stream: std::net::TcpStream, state: Arc<Mutex<WsServerState>>) {
let _ = stream.set_read_timeout(Some(Duration::from_millis(100)));
let ws_result = tungstenite::accept(stream);
let mut ws = match ws_result {
Ok(w) => w,
Err(e) => {
eprintln!("[ws_serve] WebSocket handshake error: {e}");
return;
}
};
// Assign a client ID.
let client_id = {
let mut s = state.lock().unwrap();
let id = s.next_client_id();
// We'll register the sender after we create the channel below.
id
};
// Create an outbound channel for this connection.
let (tx, rx) = mpsc::channel::<Option<String>>();
{
let mut s = state.lock().unwrap();
s.clients.insert(client_id.clone(), WsClient { tx });
}
eprintln!("[ws_serve] client connected: {client_id}");
loop {
// --- Receive incoming messages (non-blocking with short timeout) ---
match ws.read() {
Ok(tungstenite::Message::Text(text)) => {
let mut s = state.lock().unwrap();
s.inbox.push(IncomingWsMessage {
client_id: client_id.clone(),
message: text.to_string(),
});
}
Ok(tungstenite::Message::Binary(bytes)) => {
let text = String::from_utf8_lossy(&bytes).into_owned();
let mut s = state.lock().unwrap();
s.inbox.push(IncomingWsMessage {
client_id: client_id.clone(),
message: text,
});
}
Ok(tungstenite::Message::Close(_)) => {
break;
}
Ok(tungstenite::Message::Ping(data)) => {
let _ = ws.send(tungstenite::Message::Pong(data));
}
Ok(_) => {}
Err(tungstenite::Error::Io(e))
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::TimedOut =>
{
// No data yet — check the outbound channel.
}
Err(e) => {
eprintln!("[ws_serve] read error for {client_id}: {e}");
break;
}
}
// --- Send queued outbound messages ---
loop {
match rx.try_recv() {
Ok(Some(msg)) => {
if ws
.send(tungstenite::Message::Text(msg.into()))
.is_err()
{
break;
}
}
Ok(None) => {
// Disconnect signal.
let _ = ws.close(None);
let mut s = state.lock().unwrap();
s.clients.remove(&client_id);
eprintln!("[ws_serve] client disconnected (server-side): {client_id}");
return;
}
Err(mpsc::TryRecvError::Empty) => break,
Err(mpsc::TryRecvError::Disconnected) => {
break;
}
}
}
}
// Clean up.
{
let mut s = state.lock().unwrap();
s.clients.remove(&client_id);
}
eprintln!("[ws_serve] client disconnected: {client_id}");
}
/// Send a message to a specific connected client.
/// Returns `false` if the client is not found.
pub fn ws_server_send(port: u16, client_id: &str, message: String) -> bool {
let map = ws_servers().lock().unwrap();
if let Some(state) = map.get(&port) {
let s = state.lock().unwrap();
if let Some(client) = s.clients.get(client_id) {
return client.tx.send(Some(message)).is_ok();
}
}
false
}
/// Broadcast a message to all connected clients on a port.
pub fn ws_server_broadcast(port: u16, message: String) {
let map = ws_servers().lock().unwrap();
if let Some(state) = map.get(&port) {
let s = state.lock().unwrap();
for client in s.clients.values() {
let _ = client.tx.send(Some(message.clone()));
}
}
}
/// Disconnect a specific client.
pub fn ws_server_close(port: u16, client_id: &str) {
let map = ws_servers().lock().unwrap();
if let Some(state) = map.get(&port) {
let s = state.lock().unwrap();
if let Some(client) = s.clients.get(client_id) {
let _ = client.tx.send(None);
}
}
}
/// Drain pending incoming messages for a given server port.
///
/// Returns up to `max` messages (or all if `max == 0`). The caller is
/// responsible for invoking the El handler function for each message.
pub fn ws_server_poll(port: u16, max: usize) -> Vec<IncomingWsMessage> {
let map = ws_servers().lock().unwrap();
if let Some(state) = map.get(&port) {
let mut s = state.lock().unwrap();
if max == 0 || s.inbox.len() <= max {
let msgs = std::mem::take(&mut s.inbox);
msgs
} else {
s.inbox.drain(..max).collect()
}
} else {
vec![]
}
}
// ─────────────────────────────────────────────────────────────────────────────
// WebSocket client with handler callback support
// ─────────────────────────────────────────────────────────────────────────────
//
// `ws_connect` already exists in `main.rs` using synchronous tungstenite.
// This new variant (`ws_connect_handler`) runs the connection on a background
// thread and queues incoming messages so `ws_client_poll` can deliver them to
// the El handler.
/// Pending message from a background WS client connection.
#[derive(Debug)]
pub struct IncomingClientMessage {
pub conn_id: String,
pub message: String,
}
struct WsClientConn {
tx: mpsc::Sender<Option<String>>,
}
static WS_CLIENT_CONNS: OnceLock<Mutex<HashMap<String, WsClientConn>>> = OnceLock::new();
static WS_CLIENT_INBOX: OnceLock<Mutex<Vec<IncomingClientMessage>>> = OnceLock::new();
static WS_CLIENT_COUNTER: OnceLock<Mutex<u64>> = OnceLock::new();
fn ws_client_conns() -> &'static Mutex<HashMap<String, WsClientConn>> {
WS_CLIENT_CONNS.get_or_init(|| Mutex::new(HashMap::new()))
}
fn ws_client_inbox() -> &'static Mutex<Vec<IncomingClientMessage>> {
WS_CLIENT_INBOX.get_or_init(|| Mutex::new(Vec::new()))
}
fn next_conn_id() -> String {
let mut ctr = WS_CLIENT_COUNTER
.get_or_init(|| Mutex::new(1))
.lock()
.unwrap();
let id = format!("wsconn:{}", *ctr);
*ctr += 1;
id
}
/// Connect to a WebSocket server in the background.
///
/// Returns a `conn_id` string immediately. Incoming messages are queued and
/// can be retrieved with [`ws_client_poll`].
pub fn ws_client_connect(url: &str) -> String {
let conn_id = next_conn_id();
let (tx, rx) = mpsc::channel::<Option<String>>();
{
let mut map = ws_client_conns().lock().unwrap();
map.insert(conn_id.clone(), WsClientConn { tx });
}
let url_owned = url.to_owned();
let conn_id_clone = conn_id.clone();
std::thread::spawn(move || {
let ws_result = tungstenite::connect(&url_owned);
let (mut ws, _) = match ws_result {
Ok(pair) => pair,
Err(e) => {
eprintln!("[ws_client] connect error for {conn_id_clone}: {e}");
let mut map = ws_client_conns().lock().unwrap();
map.remove(&conn_id_clone);
return;
}
};
// Set a short read timeout so the loop stays responsive to outbound messages.
// MaybeTlsStream exposes the underlying stream via get_ref/get_mut.
{
use tungstenite::stream::MaybeTlsStream;
match ws.get_mut() {
MaybeTlsStream::Plain(tcp) => {
let _ = tcp.set_read_timeout(Some(Duration::from_millis(50)));
}
#[cfg(feature = "native-tls")]
MaybeTlsStream::NativeTls(tls) => {
let _ = tls.get_ref().set_read_timeout(Some(Duration::from_millis(50)));
}
_ => {}
}
}
loop {
// Receive from server.
match ws.read() {
Ok(tungstenite::Message::Text(text)) => {
let mut inbox = ws_client_inbox().lock().unwrap();
inbox.push(IncomingClientMessage {
conn_id: conn_id_clone.clone(),
message: text.to_string(),
});
}
Ok(tungstenite::Message::Binary(bytes)) => {
let text = String::from_utf8_lossy(&bytes).into_owned();
let mut inbox = ws_client_inbox().lock().unwrap();
inbox.push(IncomingClientMessage {
conn_id: conn_id_clone.clone(),
message: text,
});
}
Ok(tungstenite::Message::Close(_)) => break,
Ok(tungstenite::Message::Ping(data)) => {
let _ = ws.send(tungstenite::Message::Pong(data));
}
Ok(_) => {}
Err(tungstenite::Error::Io(e))
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::TimedOut =>
{
// No data yet.
}
Err(e) => {
eprintln!("[ws_client] read error for {conn_id_clone}: {e}");
break;
}
}
// Send queued outbound messages.
loop {
match rx.try_recv() {
Ok(Some(msg)) => {
if ws.send(tungstenite::Message::Text(msg.into())).is_err() {
break;
}
}
Ok(None) => {
let _ = ws.close(None);
let mut map = ws_client_conns().lock().unwrap();
map.remove(&conn_id_clone);
return;
}
Err(mpsc::TryRecvError::Empty) => break,
Err(mpsc::TryRecvError::Disconnected) => break,
}
}
}
let mut map = ws_client_conns().lock().unwrap();
map.remove(&conn_id_clone);
});
conn_id
}
/// Send a message on an existing client connection.
pub fn ws_client_send(conn_id: &str, message: String) {
let map = ws_client_conns().lock().unwrap();
if let Some(conn) = map.get(conn_id) {
let _ = conn.tx.send(Some(message));
}
}
/// Close a client connection.
pub fn ws_client_close(conn_id: &str) {
let map = ws_client_conns().lock().unwrap();
if let Some(conn) = map.get(conn_id) {
let _ = conn.tx.send(None);
}
}
/// Drain pending incoming messages for client connections.
///
/// Returns all queued messages (all connections combined). Pass `conn_id` as
/// `Some(&str)` to filter to a specific connection, or `None` for all.
pub fn ws_client_poll(conn_id_filter: Option<&str>) -> Vec<IncomingClientMessage> {
let mut inbox = ws_client_inbox().lock().unwrap();
if let Some(filter) = conn_id_filter {
let (matching, rest): (Vec<_>, Vec<_>) =
std::mem::take(&mut *inbox)
.into_iter()
.partition(|m| m.conn_id == filter);
*inbox = rest;
matching
} else {
std::mem::take(&mut *inbox)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Transient retry wrapper (for enhancing existing http_get / http_post)
// ─────────────────────────────────────────────────────────────────────────────
/// Try `f` once; on connection error, wait `delay_ms` and try once more.
///
/// This is the "automatic single retry on transient network error" behaviour
/// layered over the existing `http_get` / `http_post` builtins.
pub fn with_single_retry<F>(delay_ms: u64, f: F) -> String
where
F: Fn() -> Result<String, reqwest::Error>,
{
match f() {
Ok(body) => body,
Err(e) if e.is_connect() || e.is_timeout() => {
std::thread::sleep(Duration::from_millis(delay_ms));
f().unwrap_or_else(|e2| format!("{{\"error\":\"{e2}\"}}"))
}
Err(e) => format!("{{\"error\":\"{e}\"}}"),
}
}
@@ -0,0 +1,641 @@
//! Automatic, zero-config observability for the El runtime.
//!
//! This module implements:
//! - A background OTLP/HTTP exporter (spans + logs + metrics)
//! - Thread-local span context for propagation
//! - Helper functions called by the interpreter's builtin dispatch
//!
//! Developers never need to call anything here directly. The interpreter
//! instruments everything automatically. Optional `log_*`, `trace_*`, and
//! `metric_*` builtins are also wired through this module for explicit use.
//!
//! ## Graceful degradation
//!
//! If the OTLP endpoint is unreachable, a single warning is emitted to stderr
//! on the first failure, then telemetry is silently dropped. Programs never
//! fail because observability is down.
use std::sync::{
OnceLock,
atomic::{AtomicBool, Ordering},
};
use std::sync::mpsc::{self, SyncSender};
use std::time::{SystemTime, UNIX_EPOCH};
use std::collections::HashMap;
// ── Public re-export ──────────────────────────────────────────────────────────
pub use context::SpanGuard;
// ── Constants ─────────────────────────────────────────────────────────────────
const DEFAULT_OTLP_ENDPOINT: &str = "http://alloy.neuralplatform.ai:4318";
const BATCH_SIZE: usize = 64;
const BATCH_TIMEOUT_MS: u64 = 5_000;
// ── Telemetry payload types ───────────────────────────────────────────────────
#[derive(Clone, Debug)]
pub struct Span {
pub trace_id: String,
pub span_id: String,
pub parent_id: Option<String>,
pub name: String,
pub start_ns: u64,
pub end_ns: u64,
pub status: SpanStatus,
pub attrs: Vec<(String, AttrValue)>,
pub events: Vec<SpanEvent>,
pub service: String,
}
#[derive(Clone, Debug, PartialEq)]
pub enum SpanStatus {
Ok,
Error(String),
}
#[derive(Clone, Debug)]
pub struct SpanEvent {
pub name: String,
pub time_ns: u64,
pub attrs: Vec<(String, AttrValue)>,
}
#[derive(Clone, Debug)]
pub enum AttrValue {
Str(String),
Int(i64),
Float(f64),
Bool(bool),
}
#[derive(Clone, Debug)]
pub struct LogRecord {
pub time_ns: u64,
pub severity: LogSeverity,
pub body: String,
pub attrs: Vec<(String, AttrValue)>,
pub service: String,
pub trace_id: Option<String>,
pub span_id: Option<String>,
}
#[derive(Clone, Debug, Copy)]
pub enum LogSeverity {
Debug,
Info,
Warn,
Error,
}
impl LogSeverity {
fn number(self) -> u32 {
match self {
LogSeverity::Debug => 5,
LogSeverity::Info => 9,
LogSeverity::Warn => 13,
LogSeverity::Error => 17,
}
}
fn text(self) -> &'static str {
match self {
LogSeverity::Debug => "DEBUG",
LogSeverity::Info => "INFO",
LogSeverity::Warn => "WARN",
LogSeverity::Error => "ERROR",
}
}
}
#[derive(Clone, Debug)]
pub struct Metric {
pub name: String,
pub kind: MetricKind,
pub value: f64,
pub attrs: Vec<(String, AttrValue)>,
pub time_ns: u64,
pub service: String,
}
#[derive(Clone, Debug)]
pub enum MetricKind {
Counter,
Gauge,
}
#[derive(Clone, Debug)]
enum TelemetryItem {
Span(Span),
Log(LogRecord),
Metric(Metric),
}
// ── Global telemetry sender ───────────────────────────────────────────────────
static TELEMETRY_TX: OnceLock<Option<SyncSender<TelemetryItem>>> = OnceLock::new();
static OTLP_WARNED: AtomicBool = AtomicBool::new(false);
/// Initialise the telemetry background thread. Called once on interpreter start.
/// `service_name` is the El program filename (without extension).
pub fn init(service_name: &str) {
let endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")
.unwrap_or_else(|_| DEFAULT_OTLP_ENDPOINT.to_string());
let svc = service_name.to_string();
// Bounded channel — if the exporter falls behind, new items are dropped.
let (tx, rx) = mpsc::sync_channel::<TelemetryItem>(4096);
// Store the sender before starting the thread so callers can send immediately.
TELEMETRY_TX.get_or_init(|| Some(tx));
std::thread::Builder::new()
.name("el-telemetry".to_string())
.spawn(move || {
exporter_loop(rx, &endpoint, &svc);
})
.ok(); // If the thread fails to spawn, we degrade silently.
}
/// Send one telemetry item. Never panics. Drops if channel is full or uninitialised.
fn send(item: TelemetryItem) {
if let Some(Some(tx)) = TELEMETRY_TX.get() {
let _ = tx.try_send(item);
}
}
// ── Span builder / emitter ────────────────────────────────────────────────────
pub fn emit_span(span: Span) {
send(TelemetryItem::Span(span));
}
pub fn emit_log(record: LogRecord) {
send(TelemetryItem::Log(record));
}
pub fn emit_metric(metric: Metric) {
send(TelemetryItem::Metric(metric));
}
// ── Thread-local context ──────────────────────────────────────────────────────
pub mod context {
use super::*;
thread_local! {
/// Stack of active span IDs for the current thread.
static SPAN_STACK: std::cell::RefCell<Vec<(String, String)>> =
std::cell::RefCell::new(Vec::new());
}
/// Get the current (innermost) span context: (trace_id, span_id).
pub fn current_span() -> Option<(String, String)> {
SPAN_STACK.with(|s| s.borrow().last().cloned())
}
/// Push a span context onto the thread-local stack.
pub fn push_span(trace_id: String, span_id: String) {
SPAN_STACK.with(|s| s.borrow_mut().push((trace_id, span_id)));
}
/// Pop the innermost span context.
pub fn pop_span() {
SPAN_STACK.with(|s| { s.borrow_mut().pop(); });
}
/// A RAII guard that closes the span when it goes out of scope.
pub struct SpanGuard {
pub span: Span,
}
impl SpanGuard {
pub fn new(name: &str, service: &str) -> Self {
let (trace_id, parent_id) = current_span()
.map(|(t, s)| (t, Some(s)))
.unwrap_or_else(|| (new_trace_id(), None));
let span_id = new_span_id();
push_span(trace_id.clone(), span_id.clone());
SpanGuard {
span: Span {
trace_id,
span_id,
parent_id,
name: name.to_string(),
start_ns: now_ns(),
end_ns: 0,
status: SpanStatus::Ok,
attrs: Vec::new(),
events: Vec::new(),
service: service.to_string(),
},
}
}
pub fn attr(mut self, k: &str, v: AttrValue) -> Self {
self.span.attrs.push((k.to_string(), v));
self
}
pub fn error(mut self, msg: &str) -> Self {
self.span.status = SpanStatus::Error(msg.to_string());
self.span.events.push(SpanEvent {
name: "exception".to_string(),
time_ns: now_ns(),
attrs: vec![("exception.message".to_string(), AttrValue::Str(msg.to_string()))],
});
self
}
/// Finish the span without dropping the guard (for manual control).
pub fn finish(mut self) -> Span {
pop_span();
self.span.end_ns = now_ns();
self.span.clone()
}
}
impl Drop for SpanGuard {
fn drop(&mut self) {
// Only pop if not already finished manually.
// We detect this by checking if end_ns is still 0.
if self.span.end_ns == 0 {
pop_span();
self.span.end_ns = now_ns();
emit_span(self.span.clone());
}
}
}
}
// ── ID generation ─────────────────────────────────────────────────────────────
pub fn new_trace_id() -> String {
let id = uuid::Uuid::new_v4();
hex::encode(id.as_bytes())
}
pub fn new_span_id() -> String {
let id = uuid::Uuid::new_v4();
// Span ID is 8 bytes
hex::encode(&id.as_bytes()[..8])
}
// ── Timing ────────────────────────────────────────────────────────────────────
pub fn now_ns() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0)
}
pub fn now_ms() -> u64 {
now_ns() / 1_000_000
}
// ── Service name global ───────────────────────────────────────────────────────
static SERVICE_NAME: OnceLock<String> = OnceLock::new();
pub fn service_name() -> &'static str {
SERVICE_NAME.get().map(|s| s.as_str()).unwrap_or("el-program")
}
pub fn set_service_name(name: &str) {
let _ = SERVICE_NAME.set(name.to_string());
}
// ── High-level tracing helpers ────────────────────────────────────────────────
/// Instrument a function call. Returns a SpanGuard; emit it when done.
pub fn start_fn_span(fn_name: &str) -> context::SpanGuard {
context::SpanGuard::new(fn_name, service_name())
}
/// Emit a log at the given severity.
pub fn log(severity: LogSeverity, body: &str) {
let (trace_id, span_id) = context::current_span()
.map(|(t, s)| (Some(t), Some(s)))
.unwrap_or((None, None));
emit_log(LogRecord {
time_ns: now_ns(),
severity,
body: body.to_string(),
attrs: Vec::new(),
service: service_name().to_string(),
trace_id,
span_id,
});
}
pub fn log_debug(msg: &str) { log(LogSeverity::Debug, msg); }
pub fn log_info(msg: &str) { log(LogSeverity::Info, msg); }
pub fn log_warn(msg: &str) { log(LogSeverity::Warn, msg); }
pub fn log_error(msg: &str) { log(LogSeverity::Error, msg); }
/// Convenience: emit a metric counter.
pub fn counter(name: &str, value: f64, attrs: Vec<(String, AttrValue)>) {
emit_metric(Metric {
name: name.to_string(),
kind: MetricKind::Counter,
value,
attrs,
time_ns: now_ns(),
service: service_name().to_string(),
});
}
/// Convenience: emit a metric gauge.
pub fn gauge(name: &str, value: f64, attrs: Vec<(String, AttrValue)>) {
emit_metric(Metric {
name: name.to_string(),
kind: MetricKind::Gauge,
value,
attrs,
time_ns: now_ns(),
service: service_name().to_string(),
});
}
// ── OTLP/HTTP JSON exporter ───────────────────────────────────────────────────
// Uses the OTLP/HTTP JSON format (not protobuf) which Alloy accepts.
fn exporter_loop(
rx: mpsc::Receiver<TelemetryItem>,
endpoint: &str,
service: &str,
) {
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(10))
.build()
.unwrap_or_else(|_| reqwest::blocking::Client::new());
let spans_url = format!("{}/v1/traces", endpoint);
let logs_url = format!("{}/v1/logs", endpoint);
let metrics_url = format!("{}/v1/metrics", endpoint);
let mut spans: Vec<Span> = Vec::with_capacity(BATCH_SIZE);
let mut logs: Vec<LogRecord> = Vec::with_capacity(BATCH_SIZE);
let mut metrics: Vec<Metric> = Vec::with_capacity(BATCH_SIZE);
let timeout = std::time::Duration::from_millis(BATCH_TIMEOUT_MS);
loop {
// Try to receive one item with a timeout, then drain available items.
match rx.recv_timeout(timeout) {
Ok(item) => {
enqueue_item(item, &mut spans, &mut logs, &mut metrics);
// Drain any immediately available items.
while let Ok(item) = rx.try_recv() {
enqueue_item(item, &mut spans, &mut logs, &mut metrics);
if spans.len() + logs.len() + metrics.len() >= BATCH_SIZE {
break;
}
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
// Flush whatever we have.
}
Err(mpsc::RecvTimeoutError::Disconnected) => {
// Channel closed — flush and exit.
flush(&client, &spans_url, &logs_url, &metrics_url,
service, &spans, &logs, &metrics);
break;
}
}
if !spans.is_empty() || !logs.is_empty() || !metrics.is_empty() {
flush(&client, &spans_url, &logs_url, &metrics_url,
service, &spans, &logs, &metrics);
spans.clear();
logs.clear();
metrics.clear();
}
}
}
fn enqueue_item(
item: TelemetryItem,
spans: &mut Vec<Span>,
logs: &mut Vec<LogRecord>,
metrics: &mut Vec<Metric>,
) {
match item {
TelemetryItem::Span(s) => spans.push(s),
TelemetryItem::Log(l) => logs.push(l),
TelemetryItem::Metric(m) => metrics.push(m),
}
}
fn flush(
client: &reqwest::blocking::Client,
spans_url: &str,
logs_url: &str,
metrics_url: &str,
service: &str,
spans: &[Span],
logs: &[LogRecord],
metrics: &[Metric],
) {
if !spans.is_empty() {
let body = build_traces_json(service, spans);
post_otlp(client, spans_url, &body);
}
if !logs.is_empty() {
let body = build_logs_json(service, logs);
post_otlp(client, logs_url, &body);
}
if !metrics.is_empty() {
let body = build_metrics_json(service, metrics);
post_otlp(client, metrics_url, &body);
}
}
fn post_otlp(client: &reqwest::blocking::Client, url: &str, body: &str) {
let res = client.post(url)
.header("Content-Type", "application/json")
.body(body.to_string())
.send();
match res {
Ok(r) if r.status().is_success() => {}
Ok(r) => {
if !OTLP_WARNED.swap(true, Ordering::Relaxed) {
eprintln!("[el-telemetry] OTLP export failed: HTTP {}", r.status());
}
}
Err(e) => {
if !OTLP_WARNED.swap(true, Ordering::Relaxed) {
eprintln!("[el-telemetry] OTLP endpoint unreachable ({}), telemetry will be dropped", e);
}
}
}
}
// ── JSON serialisation for OTLP/HTTP ─────────────────────────────────────────
fn attr_value_json(v: &AttrValue) -> serde_json::Value {
match v {
AttrValue::Str(s) => serde_json::json!({"stringValue": s}),
AttrValue::Int(n) => serde_json::json!({"intValue": n.to_string()}),
AttrValue::Float(f) => serde_json::json!({"doubleValue": f}),
AttrValue::Bool(b) => serde_json::json!({"boolValue": b}),
}
}
fn attrs_json(attrs: &[(String, AttrValue)]) -> serde_json::Value {
serde_json::Value::Array(
attrs.iter().map(|(k, v)| {
serde_json::json!({"key": k, "value": attr_value_json(v)})
}).collect()
)
}
fn resource_json(service: &str) -> serde_json::Value {
serde_json::json!({
"attributes": [
{"key": "service.name", "value": {"stringValue": service}}
]
})
}
fn build_traces_json(service: &str, spans: &[Span]) -> String {
// Group spans by trace_id for correct OTLP structure
let mut by_trace: HashMap<&str, Vec<serde_json::Value>> = HashMap::new();
for span in spans {
let js = span_json(span);
by_trace.entry(&span.trace_id).or_default().push(js);
}
let scope_spans: Vec<serde_json::Value> = by_trace.values().map(|sps| {
serde_json::json!({
"scope": {"name": "el-runtime", "version": "0.1.0"},
"spans": sps
})
}).collect();
let payload = serde_json::json!({
"resourceSpans": [{
"resource": resource_json(service),
"scopeSpans": scope_spans
}]
});
payload.to_string()
}
fn span_json(span: &Span) -> serde_json::Value {
let status = match &span.status {
SpanStatus::Ok => serde_json::json!({"code": 1}),
SpanStatus::Error(msg) => serde_json::json!({"code": 2, "message": msg}),
};
let events: Vec<serde_json::Value> = span.events.iter().map(|e| {
serde_json::json!({
"name": e.name,
"timeUnixNano": e.time_ns.to_string(),
"attributes": attrs_json(&e.attrs)
})
}).collect();
let mut js = serde_json::json!({
"traceId": span.trace_id,
"spanId": span.span_id,
"name": span.name,
"startTimeUnixNano": span.start_ns.to_string(),
"endTimeUnixNano": span.end_ns.to_string(),
"attributes": attrs_json(&span.attrs),
"events": events,
"status": status,
"kind": 1 // INTERNAL
});
if let Some(pid) = &span.parent_id {
js["parentSpanId"] = serde_json::Value::String(pid.clone());
}
js
}
fn build_logs_json(service: &str, logs: &[LogRecord]) -> String {
let records: Vec<serde_json::Value> = logs.iter().map(|l| {
let mut r = serde_json::json!({
"timeUnixNano": l.time_ns.to_string(),
"severityNumber": l.severity.number(),
"severityText": l.severity.text(),
"body": {"stringValue": l.body},
"attributes": attrs_json(&l.attrs)
});
if let Some(tid) = &l.trace_id {
r["traceId"] = serde_json::Value::String(tid.clone());
}
if let Some(sid) = &l.span_id {
r["spanId"] = serde_json::Value::String(sid.clone());
}
r
}).collect();
let payload = serde_json::json!({
"resourceLogs": [{
"resource": resource_json(service),
"scopeLogs": [{
"scope": {"name": "el-runtime", "version": "0.1.0"},
"logRecords": records
}]
}]
});
payload.to_string()
}
fn build_metrics_json(service: &str, metrics: &[Metric]) -> String {
let metric_items: Vec<serde_json::Value> = metrics.iter().map(|m| {
let data_point = serde_json::json!({
"timeUnixNano": m.time_ns.to_string(),
"asDouble": m.value,
"attributes": attrs_json(&m.attrs)
});
match m.kind {
MetricKind::Counter => serde_json::json!({
"name": m.name,
"sum": {
"dataPoints": [data_point],
"aggregationTemporality": 2, // CUMULATIVE
"isMonotonic": true
}
}),
MetricKind::Gauge => serde_json::json!({
"name": m.name,
"gauge": {
"dataPoints": [data_point]
}
}),
}
}).collect();
let payload = serde_json::json!({
"resourceMetrics": [{
"resource": resource_json(service),
"scopeMetrics": [{
"scope": {"name": "el-runtime", "version": "0.1.0"},
"metrics": metric_items
}]
}]
});
payload.to_string()
}
// ── parse_tags_string: "k=v,k2=v2" → Vec<(String, AttrValue)> ────────────────
pub fn parse_tags_string(tags: &str) -> Vec<(String, AttrValue)> {
if tags.is_empty() {
return Vec::new();
}
tags.split(',')
.filter_map(|pair| {
let mut parts = pair.splitn(2, '=');
let k = parts.next()?.trim().to_string();
let v = parts.next().unwrap_or("").trim().to_string();
if k.is_empty() {
None
} else {
Some((k, AttrValue::Str(v)))
}
})
.collect()
}
@@ -0,0 +1,20 @@
[package]
name = "elvm"
description = "El Virtual Machine — executes compiled El bytecode (.elc) natively"
version.workspace = true
edition.workspace = true
license.workspace = true
[[bin]]
name = "elvm"
path = "src/main.rs"
[dependencies]
el-compiler = { workspace = true }
el-vm = { workspace = true }
clap = { workspace = true }
# Native window / WebView (macOS: WKWebView via wry)
wry = { version = "0.47", default-features = false }
winit = { version = "0.29", default-features = false, features = ["rwh_05", "rwh_06"] }
dpi = "0.1"
@@ -0,0 +1,146 @@
//! elvm — El Virtual Machine
//!
//! The standalone El VM binary. Loads and executes compiled El bytecode (.elc)
//! files produced by `el compile` or `el build-file`.
//!
//! # Usage
//!
//! elvm <file.elc> [args...]
//! elvm --version
//! elvm --help
//!
//! If the environment variable `NEURON_WINDOW_URL` is set, elvm opens a native
//! macOS window (WKWebView via wry) at that URL instead of executing bytecode.
//! This allows UI apps to be launched as proper desktop windows:
//!
//! NEURON_WINDOW_URL="http://localhost:7749" elvm dist/neuron-ui.elc
use clap::Parser;
use std::path::PathBuf;
#[derive(Parser, Debug)]
#[command(
name = "elvm",
about = "El Virtual Machine — execute compiled El bytecode (.elc)",
long_about = "The El VM is the native El execution substrate.\n\
Run .elc files produced by `el compile` or `el build-file`.\n\n\
Set NEURON_WINDOW_URL=<url> to open a native WebView window instead.",
version
)]
struct Cli {
/// Compiled El bytecode file to execute (*.elc).
file: PathBuf,
/// Arguments forwarded to the program (accessible via `args()`).
#[arg(trailing_var_arg = true)]
args: Vec<String>,
}
fn main() {
let cli = Cli::parse();
// If NEURON_WINDOW_URL is set, open a native WebView window at that URL.
if let Ok(url) = std::env::var("NEURON_WINDOW_URL") {
if let Err(e) = open_window(&url) {
eprintln!("elvm: window error: {e}");
std::process::exit(1);
}
return;
}
if let Err(e) = run(cli) {
eprintln!("elvm: error: {e}");
std::process::exit(1);
}
}
fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
let bytes = std::fs::read(&cli.file)
.map_err(|e| format!("cannot read {}: {e}", cli.file.display()))?;
let instructions = el_compiler::Bytecode::deserialize_all(&bytes)
.map_err(|e| format!("cannot load bytecode from {}: {e}", cli.file.display()))?;
// Detect format and print diagnostic.
let is_elvm_container = bytes.starts_with(el_compiler::ELVM_MAGIC);
if is_elvm_container {
// Normal path — ELVM container.
} else {
eprintln!("elvm: warning: {} does not have an ELVM header — treating as legacy JSON bytecode", cli.file.display());
}
let mut vm = el_vm::ElVm::new();
vm.run(&instructions, &cli.args);
Ok(())
}
/// Opens a native WebView window at `url`.
///
/// On macOS: uses wry (WKWebView) + winit for a proper native desktop window.
/// On other platforms: prints the URL (fallback).
fn open_window(url: &str) -> Result<(), Box<dyn std::error::Error>> {
#[cfg(target_os = "macos")]
return open_native_window(url);
#[cfg(not(target_os = "macos"))]
{
eprintln!("elvm: native window not supported on this platform");
println!("elvm: open {url} in your browser");
Ok(())
}
}
#[cfg(target_os = "macos")]
fn open_native_window(url: &str) -> Result<(), Box<dyn std::error::Error>> {
use dpi::LogicalSize;
use winit::{
event::{Event, WindowEvent},
event_loop::{ControlFlow, EventLoop},
window::WindowBuilder,
};
use wry::{Rect, WebViewBuilder};
let event_loop = EventLoop::new().map_err(|e| format!("event loop: {e}"))?;
let window = WindowBuilder::new()
.with_title("Neuron")
.with_inner_size(winit::dpi::LogicalSize::new(1600u32, 1000u32))
.with_min_inner_size(winit::dpi::LogicalSize::new(900u32, 600u32))
.with_resizable(true)
.build(&event_loop)
.map_err(|e| format!("window: {e}"))?;
let url_owned = url.to_string();
let webview = WebViewBuilder::new()
.with_url(&url_owned)
.build_as_child(&window)
.map_err(|e| format!("webview: {e}"))?;
event_loop
.run(move |event, evl| {
evl.set_control_flow(ControlFlow::Wait);
match event {
Event::WindowEvent {
event: WindowEvent::Resized(size),
..
} => {
let scale = window.scale_factor();
let logical = size.to_logical::<u32>(scale);
let _ = webview.set_bounds(Rect {
position: dpi::LogicalPosition::new(0, 0).into(),
size: LogicalSize::new(logical.width, logical.height).into(),
});
}
Event::WindowEvent {
event: WindowEvent::CloseRequested,
..
} => evl.exit(),
_ => {}
}
})
.map_err(|e| format!("event loop run: {e}"))?;
Ok(())
}
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env bash
# build-targets.sh — Build the El VM (elvm) for all supported platforms.
#
# Usage:
# ./build-targets.sh # build all targets
# ./build-targets.sh --list # list targets and prerequisites
# ./build-targets.sh x86_64-apple-darwin # build a specific target
#
# Prerequisites:
# - Rust toolchain (rustup) with cross-compilation targets installed
# - For Linux targets: cross-linkers (see comments below)
#
# Install a target:
# rustup target add <target>
#
# For cross-compilation from macOS to Linux:
# brew install FiloSottile/musl-cross/musl-cross
# # or use `cross` (https://github.com/cross-rs/cross):
# cargo install cross --git https://github.com/cross-rs/cross
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUTPUT_DIR="${SCRIPT_DIR}/dist/elvm"
# ── Target definitions ────────────────────────────────────────────────────────
# Format: "target:description:linker_hint"
declare -a TARGETS=(
"x86_64-unknown-linux-gnu:Linux x86_64 (glibc):brew install FiloSottile/musl-cross/musl-cross or use 'cross'"
"aarch64-unknown-linux-gnu:Linux ARM64 (glibc):brew install FiloSottile/musl-cross/musl-cross or use 'cross'"
"x86_64-unknown-linux-musl:Linux x86_64 (musl, static):brew install FiloSottile/musl-cross/musl-cross"
"aarch64-unknown-linux-musl:Linux ARM64 (musl, static):brew install aarch64-unknown-linux-musl cross-linker"
"x86_64-apple-darwin:macOS Intel:native (no cross tooling needed on macOS)"
"aarch64-apple-darwin:macOS Apple Silicon:native (no cross tooling needed on macOS)"
"x86_64-pc-windows-gnu:Windows x86_64:brew install mingw-w64"
"aarch64-pc-windows-msvc:Windows ARM64:requires Windows MSVC SDK (Windows only)"
)
# ── Helpers ───────────────────────────────────────────────────────────────────
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
ok() { echo -e " ${GREEN}${NC} $*"; }
warn() { echo -e " ${YELLOW}${NC} $*"; }
fail() { echo -e " ${RED}${NC} $*"; }
# ── List mode ─────────────────────────────────────────────────────────────────
if [[ "${1:-}" == "--list" ]]; then
echo ""
echo "El VM cross-compilation targets"
echo "================================"
echo ""
for entry in "${TARGETS[@]}"; do
IFS=':' read -r target desc hint <<< "$entry"
printf " %-42s %s\n" "$target" "$desc"
printf " %-42s Prereq: %s\n" "" "$hint"
echo ""
done
echo "Install a target: rustup target add <target>"
echo "List installed: rustup target list --installed"
exit 0
fi
# ── Filter targets ────────────────────────────────────────────────────────────
if [[ $# -gt 0 && "${1:-}" != "--list" ]]; then
# Build only the specified target(s).
SELECTED=("$@")
else
# Build all targets.
SELECTED=()
for entry in "${TARGETS[@]}"; do
IFS=':' read -r target _ _ <<< "$entry"
SELECTED+=("$target")
done
fi
# ── Build ─────────────────────────────────────────────────────────────────────
echo ""
echo -e "${BLUE}El VM — cross-platform build${NC}"
echo "Output directory: $OUTPUT_DIR"
echo ""
mkdir -p "$OUTPUT_DIR"
BUILT=0
SKIPPED=0
FAILED=0
for target in "${SELECTED[@]}"; do
# Find description for this target.
desc="$target"
for entry in "${TARGETS[@]}"; do
IFS=':' read -r t d _ <<< "$entry"
if [[ "$t" == "$target" ]]; then desc="$d"; break; fi
done
echo -e "${BLUE}→ Building $target${NC} ($desc)"
# Check if the Rust target is installed.
if ! rustup target list --installed 2>/dev/null | grep -q "^$target\b"; then
warn "Target $target not installed — installing..."
if rustup target add "$target" 2>/dev/null; then
ok "Installed $target"
else
fail "Cannot install $target — skipping"
((SKIPPED++))
continue
fi
fi
# Attempt to build.
binary_suffix=""
if [[ "$target" == *"windows"* ]]; then
binary_suffix=".exe"
fi
binary_name="elvm-${target}${binary_suffix}"
output_path="$OUTPUT_DIR/$binary_name"
if (cd "$SCRIPT_DIR" && cargo build --release --target "$target" -p elvm 2>&1 | tail -5); then
src="$SCRIPT_DIR/target/$target/release/elvm${binary_suffix}"
if [[ -f "$src" ]]; then
cp "$src" "$output_path"
size=$(du -h "$output_path" | cut -f1)
ok "Built $binary_name ($size)"
((BUILT++))
else
fail "Binary not found at $src"
((FAILED++))
fi
else
fail "Build failed for $target (cross-linker may be missing — see --list)"
((FAILED++))
fi
echo ""
done
# ── Summary ───────────────────────────────────────────────────────────────────
echo "────────────────────────────────"
echo "Built: $BUILT"
echo "Skipped: $SKIPPED"
echo "Failed: $FAILED"
if [[ $BUILT -gt 0 ]]; then
echo ""
echo "Artifacts:"
ls -lh "$OUTPUT_DIR/"
fi
echo ""
if [[ $FAILED -gt 0 ]]; then
echo -e "${YELLOW}Some targets failed. Run './build-targets.sh --list' for prerequisites.${NC}"
exit 1
fi
@@ -266,6 +266,12 @@ fn extract_calls_from_expr(expr: &Expr, in_loop: bool, out: &mut Vec<CallInfo>)
}
}
Expr::UnaryBitNot(inner) => extract_calls_from_expr(inner, in_loop, out),
// JSX expressions — traverse children.
Expr::JsxElement { children, .. } => {
for child in children { extract_calls_from_expr(child, in_loop, out); }
}
Expr::JsxExpr(inner) => extract_calls_from_expr(inner, in_loop, out),
Expr::JsxText(_) => {}
}
}
@@ -388,6 +394,12 @@ fn extract_activate_types_expr(expr: &Expr, in_loop: bool, out: &mut Vec<String>
}
}
Expr::UnaryBitNot(inner) => extract_activate_types_expr(inner, in_loop, out),
// JSX expressions — traverse children.
Expr::JsxElement { children, .. } => {
for child in children { extract_activate_types_expr(child, in_loop, out); }
}
Expr::JsxExpr(inner) => extract_activate_types_expr(inner, in_loop, out),
Expr::JsxText(_) => {}
}
}
@@ -452,6 +464,10 @@ fn has_sealed_in_loop_expr(expr: &Expr, in_loop: bool) -> bool {
Expr::Parallel { entries } => entries.iter().any(|(_, e)| has_sealed_in_loop_expr(e, in_loop)),
Expr::Trace { body, .. } => body.iter().any(|s| has_sealed_in_loop_stmt(s, in_loop)),
Expr::UnaryBitNot(inner) => has_sealed_in_loop_expr(inner, in_loop),
// JSX expressions.
Expr::JsxElement { children, .. } => children.iter().any(|c| has_sealed_in_loop_expr(c, in_loop)),
Expr::JsxExpr(inner) => has_sealed_in_loop_expr(inner, in_loop),
Expr::JsxText(_) => false,
}
}
@@ -365,11 +365,13 @@ fn resolve_imports_inner(
.map_err(|e| format!("cannot read {}: {e}", file.display()))?;
let mut out = String::new();
for line in source.lines() {
let mut lines_iter = source.lines().peekable();
while let Some(line) = lines_iter.next() {
let trimmed = line.trim();
if let Some(rest) = trimmed.strip_prefix("import ") {
let rest = rest.trim();
if rest.starts_with('"') && rest.ends_with('"') {
// import "relative/path.el"
let import_path_str = &rest[1..rest.len() - 1];
let import_path = dir.join(import_path_str);
let imported = resolve_imports_inner(&import_path, visited)?;
@@ -379,6 +381,49 @@ fn resolve_imports_inner(
out.push_str(line);
out.push('\n');
}
} else if let Some(rest) = trimmed.strip_prefix("from ") {
// from ModuleName import { ... } — resolve ModuleName.el in the same dir
// Extract the module name (everything before " import")
if let Some(module_part) = rest.split(" import").next() {
let module_name = module_part.trim();
// Only resolve bare identifiers (not quoted paths or dotted names with /)
if !module_name.contains('"') && !module_name.contains('/') && !module_name.is_empty() {
let module_file = format!("{}.el", module_name);
let import_path = dir.join(&module_file);
// Consume any continuation lines of a multi-line import:
// `from X import {\n Y,\n Z,\n}`
// The opening line may or may not contain `{`. If it does not end
// with `}`, skip lines until we see one that ends with `}`.
let import_rest = rest.splitn(2, " import").nth(1).unwrap_or("").trim();
let is_multiline = import_rest.contains('{') && !import_rest.contains('}');
if is_multiline {
// Skip continuation lines until we see `}`
for cont in lines_iter.by_ref() {
if cont.trim().contains('}') {
break;
}
}
}
if import_path.exists() {
// Inline the module, omitting the import line itself
let imported = resolve_imports_inner(&import_path, visited)?;
out.push_str(&imported);
out.push('\n');
} else {
// Module file not found — leave the line for the compiler to handle
out.push_str(line);
out.push('\n');
}
} else {
out.push_str(line);
out.push('\n');
}
} else {
out.push_str(line);
out.push('\n');
}
} else {
out.push_str(line);
out.push('\n');
@@ -439,6 +484,7 @@ mod tests {
},
cross: CrossConfig { targets: vec![] },
plugins: HashMap::new(),
app: None,
}
}
@@ -225,9 +225,78 @@ pub fn deserialize_bytecode(bytes: &[u8]) -> Result<Vec<Bytecode>, String> {
serde_json::from_slice(bytes).map_err(|e| e.to_string())
}
impl Bytecode {
/// Deserialize a bytecode slice from JSON bytes (convenience wrapper).
pub fn deserialize_all(bytes: &[u8]) -> Result<Vec<Bytecode>, String> {
// ── ELVM binary container format ──────────────────────────────────────────────
//
// Header:
// magic: [u8; 4] = b"ELVM"
// version: [u8; 4] (little-endian u32, currently 1)
// length: [u8; 8] (little-endian u64, byte length of the payload that follows)
// Payload:
// JSON-serialized Vec<Bytecode> (same encoding as serialize_bytecode)
//
// The JSON payload is intentionally kept so that existing tooling can inspect
// .elc files by stripping the 16-byte header. Future versions may switch to a
// denser binary encoding while bumping the version field.
/// Magic bytes that identify an El bytecode container.
pub const ELVM_MAGIC: &[u8; 4] = b"ELVM";
/// Current container format version.
pub const ELVM_VERSION: u32 = 1;
/// Total header size in bytes (magic + version + length).
pub const ELVM_HEADER_SIZE: usize = 16;
/// Wrap serialized bytecode in the ELVM binary container.
///
/// Returns the full `.elc` file bytes: 16-byte header + JSON payload.
pub fn wrap_elvm(instructions: &[Bytecode]) -> Result<Vec<u8>, String> {
let payload = serialize_bytecode(instructions)?;
let length = payload.len() as u64;
let mut out = Vec::with_capacity(ELVM_HEADER_SIZE + payload.len());
out.extend_from_slice(ELVM_MAGIC);
out.extend_from_slice(&ELVM_VERSION.to_le_bytes());
out.extend_from_slice(&length.to_le_bytes());
out.extend_from_slice(&payload);
Ok(out)
}
/// Unwrap and deserialize an ELVM binary container.
///
/// Accepts either:
/// - A full ELVM container (starts with `ELVM` magic), or
/// - Raw JSON bytes (legacy `.elc` format — no header).
pub fn unwrap_elvm(bytes: &[u8]) -> Result<Vec<Bytecode>, String> {
if bytes.starts_with(ELVM_MAGIC) {
// Binary container: skip header, deserialize payload.
if bytes.len() < ELVM_HEADER_SIZE {
return Err("truncated ELVM header".to_string());
}
let version = u32::from_le_bytes(
bytes[4..8].try_into().map_err(|_| "bad version field".to_string())?
);
if version != ELVM_VERSION {
return Err(format!("unsupported ELVM version {version} (expected {ELVM_VERSION})"));
}
let length = u64::from_le_bytes(
bytes[8..16].try_into().map_err(|_| "bad length field".to_string())?
) as usize;
let payload_end = ELVM_HEADER_SIZE + length;
if bytes.len() < payload_end {
return Err(format!(
"truncated ELVM payload: expected {length} bytes, got {}",
bytes.len().saturating_sub(ELVM_HEADER_SIZE)
));
}
deserialize_bytecode(&bytes[ELVM_HEADER_SIZE..payload_end])
} else {
// Legacy JSON-only format (no header).
deserialize_bytecode(bytes)
}
}
impl Bytecode {
/// Deserialize a bytecode slice — handles both ELVM container and raw JSON.
pub fn deserialize_all(bytes: &[u8]) -> Result<Vec<Bytecode>, String> {
unwrap_elvm(bytes)
}
}
@@ -1,6 +1,6 @@
//! Code generator: walks the AST and emits bytecode instructions.
use el_parser::{BinOp, Expr, Literal, Program, Stmt};
use el_parser::{BinOp, Expr, JsxAttrValue, Literal, Program, Stmt};
use crate::bytecode::{Bytecode, Value};
use crate::error::CompileResult;
@@ -72,6 +72,11 @@ impl Codegen {
Stmt::Expr(expr, _) => {
// In tail position, leave the value on the stack.
self.gen_expr(expr)?;
// If-without-else leaves nothing on stack (it pops internally);
// push Nil so we always have exactly one return value.
if matches!(expr, Expr::If { else_: None, .. }) {
self.emit(Bytecode::Push(Value::Nil));
}
}
Stmt::Return(expr, _) => {
self.gen_expr(expr)?;
@@ -104,8 +109,19 @@ impl Codegen {
}
Stmt::Expr(expr, _) => {
self.gen_expr(expr)?;
// Discard the expression result unless it's a return-like
if !matches!(expr, Expr::Block(_) | Expr::If { .. }) {
// Always discard the expression result in statement position.
// Expr::Block is NOT special-cased: even a block expression used
// as a statement should have its result discarded.
// Note: Expr::If { else_: Some(_) } leaves a value on stack from
// both branches (via gen_stmt_tail on the last block statement),
// so it must be popped here too.
// If-without-else already pops internally (see gen_expr for If),
// so we only skip the extra Pop for that case.
let needs_pop = match expr {
Expr::If { else_: None, .. } => false, // already handled internally
_ => true,
};
if needs_pop {
self.emit(Bytecode::Pop);
}
}
@@ -128,11 +144,21 @@ impl Codegen {
message: format!("contract violation in fn '{name}': requires clause failed"),
});
}
for s in body {
self.gen_stmt(s)?;
// All statements except the last use gen_stmt (which pops expr results).
// The last statement uses gen_stmt_tail so that the final expression
// value stays on the stack as the function's implicit return value.
let body_len = body.len();
for (i, s) in body.iter().enumerate() {
if i + 1 == body_len {
self.gen_stmt_tail(s)?;
} else {
self.gen_stmt(s)?;
}
}
// If the body is empty, return Nil.
if body_len == 0 {
self.emit(Bytecode::Push(Value::Nil));
}
// Implicit void return
self.emit(Bytecode::Push(Value::Nil));
self.emit(Bytecode::Return);
// Patch the skip jump
@@ -235,6 +261,48 @@ impl Codegen {
// Test-related statements — skipped during normal compilation.
// The el-test crate walks the AST directly rather than running compiled bytecode.
Stmt::TestDef { .. } | Stmt::Seed(..) | Stmt::Assert(..) => {}
// Component definition: generate a function that renders the component.
// The function is registered under "__component_<Name>" and the template
// is emitted as a nested function body.
Stmt::ComponentDef { name, state, methods, template, .. } => {
let fn_name = format!("__component_{name}");
let skip_jump = self.emit(Bytecode::Jump(0));
// Emit component methods (each has its own Jump/body/Register pattern)
for m in methods {
self.gen_stmt(m)?;
}
// Record where the template starts — AFTER all method bodies.
// This is the real entry point for calling this component.
let template_start = self.current_idx();
// Initialise state fields as locals (so template expressions can load them)
for field in state {
if let Some(default) = &field.default {
self.gen_expr(default)?;
} else {
self.emit(Bytecode::Push(Value::Nil));
}
self.emit(Bytecode::StoreLocal(field.name.clone()));
}
// Emit template as the return value
self.gen_expr(template)?;
self.emit(Bytecode::Return);
let after = self.current_idx();
self.patch_jump(skip_jump, after);
// entry_point is the first instruction of the template body.
let entry_point = template_start;
self.emit(Bytecode::Push(Value::Int(entry_point as i64)));
self.emit(Bytecode::StoreLocal(format!("__fn_{fn_name}")));
// Also register the component name directly so `<ComponentName />` works
self.emit(Bytecode::Push(Value::Int(entry_point as i64)));
self.emit(Bytecode::StoreLocal(format!("__fn_{name}")));
}
// New statement kinds — no runtime code emitted.
_ => {}
}
@@ -258,6 +326,18 @@ impl Codegen {
self.emit(Bytecode::LoadLocal(name.clone()));
}
Expr::BinOp { op, left, right } => {
// NullCoalesce (`a ?? b`) is short-circuit: if `a` is truthy, use it; else `b`.
// Bytecode: eval a, Dup, JumpIf(skip), Pop, eval b, skip:
if matches!(op, BinOp::NullCoalesce) {
self.gen_expr(left)?;
self.emit(Bytecode::Dup);
let skip = self.emit(Bytecode::JumpIf(0)); // patched below
self.emit(Bytecode::Pop); // discard the falsy `a`
self.gen_expr(right)?;
let after = self.current_idx();
self.patch_jump(skip, after);
return Ok(());
}
self.gen_expr(left)?;
self.gen_expr(right)?;
let instr = match op {
@@ -279,6 +359,7 @@ impl Codegen {
BinOp::BitXor => Bytecode::BitXor,
BinOp::Shl => Bytecode::Shl,
BinOp::Shr => Bytecode::Shr,
BinOp::NullCoalesce => unreachable!("handled above"),
};
self.emit(instr);
}
@@ -297,7 +378,11 @@ impl Codegen {
}
// Get the function name from the callee expression
let fn_name = match func.as_ref() {
Expr::Ident(n) => n.clone(),
Expr::Ident(n) => {
// Strip leading `@` from syscall/builtin names:
// `@http_get` → `http_get`, `@json_parse` → `json_parse`
n.strip_prefix('@').unwrap_or(n).to_string()
}
Expr::Field { object, field } => {
self.gen_expr(object)?;
field.clone()
@@ -503,6 +588,75 @@ impl Codegen {
self.emit(Bytecode::TraceEnd { label: label.clone() });
self.emit(Bytecode::Push(Value::Nil));
}
// JSX element: push tag name, attrs (as a map), and children list, then call __jsx__
Expr::JsxElement { tag, attrs, children, .. } => {
// If the tag starts with an uppercase letter it's a component reference.
// Call the component function directly (it takes no args, returns HTML string).
let is_component = tag.chars().next().map_or(false, |c| c.is_uppercase());
if is_component {
// Call the component as a function: Call { name: tag, arity: 0 }
// The component will render itself and return an HTML string.
self.emit(Bytecode::Call { name: tag.clone(), arity: 0 });
} else {
// Push tag name
self.emit(Bytecode::Push(Value::Str(tag.clone())));
// Build attrs map: push key-value pairs
let n_attrs = attrs.len() as u32;
for (attr_name, attr_val) in attrs {
self.emit(Bytecode::Push(Value::Str(attr_name.clone())));
match attr_val {
JsxAttrValue::Str(s) => {
self.emit(Bytecode::Push(Value::Str(s.clone())));
}
JsxAttrValue::Expr(expr) => {
self.gen_expr(expr)?;
}
}
}
self.emit(Bytecode::BuildMap(n_attrs));
// Build children list
for child in children {
self.gen_expr(child)?;
}
self.emit(Bytecode::BuildList(children.len() as u32));
// Call __jsx__(tag, attrs, children)
self.emit(Bytecode::Call { name: "__jsx__".to_string(), arity: 3 });
}
}
// JSX expression interpolation: just evaluate the inner expression
Expr::JsxExpr(inner) => {
self.gen_expr(inner)?;
}
// JSX text: push as string
Expr::JsxText(text) => {
self.emit(Bytecode::Push(Value::Str(text.clone())));
}
// Closure: compile as an inline function and push a reference to it.
// The closure body is emitted as a skip-over block.
Expr::Closure { params, body, .. } => {
let closure_id = self.current_idx();
let fn_name = format!("__closure_{closure_id}__");
let skip_jump = self.emit(Bytecode::Jump(0));
// Bind params in reverse order (stack has args pushed left-to-right)
for param in params.iter().rev() {
self.emit(Bytecode::StoreLocal(param.name.clone()));
}
self.gen_expr(body)?;
self.emit(Bytecode::Return);
let after = self.current_idx();
self.patch_jump(skip_jump, after);
let entry_point = skip_jump + 1;
// Register the closure function
self.emit(Bytecode::Push(Value::Int(entry_point as i64)));
self.emit(Bytecode::StoreLocal(format!("__fn_{fn_name}")));
// Push a reference to this closure (as its function name)
self.emit(Bytecode::Push(Value::Str(fn_name)));
}
// New expression kinds — push Nil as placeholder
_ => {
self.emit(Bytecode::Push(Value::Nil));
@@ -28,7 +28,7 @@ mod debugger;
mod error;
mod source_map;
pub use bytecode::{Bytecode, Value, serialize_bytecode, deserialize_bytecode};
pub use bytecode::{Bytecode, Value, serialize_bytecode, deserialize_bytecode, wrap_elvm, unwrap_elvm, ELVM_MAGIC, ELVM_VERSION};
pub use compiler::{CompileOutput, Compiler, CompilerOptions, Target};
pub use debugger::{DebugEvent, Debugger, StackFrame, StepMode};
pub use error::{CompileError, CompileResult};
@@ -226,6 +226,21 @@ impl Formatter {
}
out.push_str(&format!("{ind}}}\n"));
}
// Component definition (UI/reactive components) — emit as-is placeholder.
Stmt::ComponentDef { name, methods, .. } => {
out.push_str(&format!("{ind}component {name} {{\n"));
for s in methods {
self.fmt_stmt(out, s, depth + 1);
}
out.push_str(&format!("{ind}}}\n"));
}
// Catch-all: unknown/future statement kinds are emitted as a comment.
#[allow(unreachable_patterns)]
_ => {
out.push_str(&format!("{ind}// [unformatted statement]\n"));
}
}
}
@@ -406,6 +421,47 @@ impl Formatter {
}
out.push_str(&format!("{}}}", self.indent(depth)));
}
// JSX expressions — emit minimal JSX syntax.
Expr::JsxElement { tag, attrs, children, self_closing } => {
out.push('<');
out.push_str(tag);
for (key, val) in attrs {
out.push(' ');
out.push_str(key);
match val {
el_parser::JsxAttrValue::Str(s) => out.push_str(&format!("=\"{s}\"")),
el_parser::JsxAttrValue::Expr(e) => {
out.push_str("={");
self.fmt_expr(out, e, depth);
out.push('}');
}
}
}
if *self_closing {
out.push_str(" />");
} else {
out.push('>');
for child in children {
self.fmt_expr(out, child, depth);
}
out.push_str(&format!("</{tag}>"));
}
}
Expr::JsxExpr(inner) => {
out.push('{');
self.fmt_expr(out, inner, depth);
out.push('}');
}
Expr::JsxText(text) => {
out.push_str(text);
}
// Catch-all: unknown/future expression kinds.
#[allow(unreachable_patterns)]
_ => {
out.push_str("/* [unformatted expr] */");
}
}
}
@@ -438,6 +494,7 @@ impl Formatter {
BinOp::BitXor => "^",
BinOp::Shl => "<<",
BinOp::Shr => ">>",
BinOp::NullCoalesce => "??",
}
}
@@ -201,7 +201,10 @@ impl<'src> Lexer<'src> {
}
}
'@' => Token::At,
'?' => Token::QuestionMark,
'#' => Token::Hash,
'?' => {
if self.eat('?') { Token::NullCoalesce } else { Token::QuestionMark }
}
':' => {
if self.eat(':') {
Token::ColonColon
@@ -220,10 +223,9 @@ impl<'src> Lexer<'src> {
c if c.is_alphabetic() || c == '_' => self.scan_ident_or_keyword(start, c),
other => {
return Err(LexError::new(
LexErrorKind::UnexpectedChar(other),
self.span_from(start),
))
// Emit Unknown token instead of erroring — allows JSX text with
// Unicode / special characters to pass through the lexer gracefully.
Token::Unknown(other)
}
};
@@ -372,10 +374,14 @@ fn keyword_or_ident(s: String) -> Token {
"parallel" => Token::Parallel,
"trace" => Token::Trace,
"requires" => Token::Requires,
"deploy" => Token::Deploy,
"to" => Token::To,
"via" => Token::Via,
_ => Token::Ident(s),
"deploy" => Token::Deploy,
"to" => Token::To,
"via" => Token::Via,
"component" => Token::Component,
"props" => Token::Props,
"state" => Token::State,
"template" => Token::Template,
_ => Token::Ident(s),
}
}
@@ -115,6 +115,14 @@ pub enum Token {
To,
/// `via` — used in `deploy fn to "/route" via soma`
Via,
/// `component` — component definition
Component,
/// `props` — props block inside a component
Props,
/// `state` — state block inside a component
State,
/// `template` — template block inside a component
Template,
/// `|>` — pipe operator
PipeOp,
/// `true` / `false`
@@ -194,6 +202,8 @@ pub enum Token {
Pipe,
/// `?` — used both for Optional types and the Try operator
QuestionMark,
/// `??` — null-coalescing operator: `a ?? b` returns `b` when `a` is nil/empty
NullCoalesce,
/// `%` — modulo
Percent,
/// `&` — bitwise AND
@@ -206,6 +216,11 @@ pub enum Token {
Shl,
/// `>>` — right shift
Shr,
/// `#` — hash character (used in JSX text like `#tag`)
Hash,
/// Any other character not recognized by the lexer (emitted instead of erroring,
/// so JSX text with Unicode / special characters can be skipped gracefully).
Unknown(char),
// ── Special ───────────────────────────────────────────────────────────────
Eof,
@@ -248,16 +263,23 @@ impl std::fmt::Display for Token {
Token::Deploy => write!(f, "deploy"),
Token::To => write!(f, "to"),
Token::Via => write!(f, "via"),
Token::Component => write!(f, "component"),
Token::Props => write!(f, "props"),
Token::State => write!(f, "state"),
Token::Template => write!(f, "template"),
Token::PipeOp => write!(f, "|>"),
Token::At => write!(f, "@"),
Token::Pipe => write!(f, "|"),
Token::QuestionMark => write!(f, "?"),
Token::NullCoalesce => write!(f, "??"),
Token::Percent => write!(f, "%"),
Token::Ampersand => write!(f, "&"),
Token::Caret => write!(f, "^"),
Token::Tilde => write!(f, "~"),
Token::Shl => write!(f, "<<"),
Token::Shr => write!(f, ">>"),
Token::Hash => write!(f, "#"),
Token::Unknown(c) => write!(f, "{c}"),
Token::BoolLiteral(b) => write!(f, "{b}"),
Token::IntLiteral(n) => write!(f, "{n}"),
Token::FloatLiteral(n) => write!(f, "{n}"),
@@ -1,6 +1,6 @@
[package]
name = "el-manifest"
description = "el.toml manifest parser for the Engram language package system"
description = "manifest.el project manifest parser for the Engram language package system"
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -9,7 +9,7 @@ license.workspace = true
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
toml = "0.8"
semver = { version = "1", features = ["serde"] }
el-lexer = { workspace = true }
[dev-dependencies]
@@ -7,8 +7,8 @@ pub enum ManifestError {
#[error("io error: {0}")]
Io(#[from] std::io::Error),
#[error("toml parse error: {0}")]
Toml(#[from] toml::de::Error),
#[error("manifest parse error at line {line}: {reason}")]
Parse { line: u32, reason: String },
#[error("semver parse error for '{field}': {source}")]
Semver {
@@ -23,7 +23,7 @@ pub enum ManifestError {
#[error("invalid value for '{field}': {reason}")]
InvalidValue { field: String, reason: String },
#[error("el.toml not found (searched from {0})")]
#[error("manifest.el not found (searched from {0})")]
NotFound(String),
#[error("invalid cross target '{0}': use x86_64-linux, aarch64-linux, x86_64-macos, aarch64-macos, wasm32")]
@@ -0,0 +1,28 @@
//! el-manifest — `manifest.el` project manifest parser.
//!
//! Every Engram project has a `manifest.el` at its root. This crate defines the
//! manifest data model and parses it from El block syntax.
//!
//! # Quick start
//! ```rust
//! use el_manifest::Manifest;
//!
//! let src = r#"
//! package "my-service" {
//! version "0.1.0"
//! edition "2026"
//! }
//! "#;
//! let manifest = Manifest::parse(src).unwrap();
//! assert_eq!(manifest.package.name, "my-service");
//! ```
mod error;
mod manifest;
mod parse;
pub use error::{ManifestError, ManifestResult};
pub use manifest::{
AppConfig, BuildConfig, BuildTarget, CrossConfig, CrossTarget, Dependency, Manifest,
NativeTarget, PackageInfo, SealKeySource,
};
@@ -1,4 +1,4 @@
//! Core data model for the `el.toml` manifest.
//! Core data model for the `manifest.el` manifest.
use std::collections::HashMap;
use std::path::PathBuf;
@@ -326,9 +326,21 @@ impl std::fmt::Display for NativeTarget {
}
}
// ── App config (native desktop) ───────────────────────────────────────────────
/// The `app { }` section — present only in native desktop apps using el-ui.
#[derive(Debug, Clone, PartialEq, Default)]
pub struct AppConfig {
pub window_title: String,
pub window_width: u32,
pub window_height: u32,
pub min_width: u32,
pub min_height: u32,
}
// ── Top-level manifest ────────────────────────────────────────────────────────
/// The parsed contents of an `el.toml` project manifest.
/// The parsed contents of a `manifest.el` project manifest.
#[derive(Debug, Clone)]
pub struct Manifest {
pub package: PackageInfo,
@@ -338,10 +350,12 @@ pub struct Manifest {
pub cross: CrossConfig,
/// Plugin name → version requirement string.
pub plugins: HashMap<String, String>,
/// Native desktop app config (present only when `app { }` section exists).
pub app: Option<AppConfig>,
}
impl Manifest {
/// Parse a manifest from a TOML string.
/// Parse a manifest from an El source string.
pub fn parse(s: &str) -> crate::ManifestResult<Self> {
crate::parse::parse_manifest(s)
}
@@ -352,7 +366,7 @@ impl Manifest {
Self::parse(&text)
}
/// Walk up the directory tree from `from` until an `el.toml` is found.
/// Walk up the directory tree from `from` until a `manifest.el` is found.
///
/// Returns the path to the manifest file (not its directory).
pub fn find_manifest(from: &std::path::Path) -> crate::ManifestResult<PathBuf> {
@@ -363,7 +377,7 @@ impl Manifest {
};
loop {
let candidate = dir.join("el.toml");
let candidate = dir.join("manifest.el");
if candidate.exists() {
return Ok(candidate);
}
@@ -0,0 +1,748 @@
//! El-syntax parser for `manifest.el` project manifests.
//!
//! The manifest format is a subset of El block syntax:
//!
//! ```el
//! package "name" {
//! version "0.1.0"
//! description "What this does"
//! authors ["Author One"]
//! edition "2026"
//! }
//!
//! build {
//! entry "src/main.el"
//! target "release"
//! output "dist/"
//! }
//!
//! dependencies {
//! el-ui { path "../../../../foundation/el-ui" }
//! }
//!
//! app {
//! window_title "My App"
//! window_width 1200
//! window_height 800
//! }
//! ```
//!
//! Rules:
//! - No equals signs — space-separated declarations
//! - String values use `"..."`, integer values are bare numbers
//! - Arrays use `[...]`, block sections use `{ }`
//! - `//` line comments are stripped by the lexer
use std::collections::HashMap;
use std::path::PathBuf;
use el_lexer::{tokenize, Token, Spanned};
use crate::error::{ManifestError, ManifestResult};
use crate::manifest::{
AppConfig, BuildConfig, BuildTarget, CrossConfig, CrossTarget, Dependency, Manifest,
PackageInfo, SealKeySource,
};
// ── Token stream wrapper ──────────────────────────────────────────────────────
struct TokenStream {
tokens: Vec<Spanned<Token>>,
pos: usize,
}
impl TokenStream {
fn new(tokens: Vec<Spanned<Token>>) -> Self {
Self { tokens, pos: 0 }
}
fn peek(&self) -> &Token {
self.tokens.get(self.pos).map(|s| &s.node).unwrap_or(&Token::Eof)
}
fn current_line(&self) -> u32 {
self.tokens.get(self.pos).map(|s| s.span.line).unwrap_or(0)
}
fn advance(&mut self) -> &Spanned<Token> {
let idx = self.pos;
self.pos += 1;
&self.tokens[idx]
}
fn expect_lbrace(&mut self) -> ManifestResult<()> {
match self.peek() {
Token::LBrace => { self.advance(); Ok(()) }
other => Err(ManifestError::Parse {
line: self.current_line(),
reason: format!("expected '{{', found '{other}'"),
}),
}
}
fn expect_rbrace(&mut self) -> ManifestResult<()> {
match self.peek() {
Token::RBrace => { self.advance(); Ok(()) }
other => Err(ManifestError::Parse {
line: self.current_line(),
reason: format!("expected '}}', found '{other}'"),
}),
}
}
/// Consume an identifier or return error.
fn expect_ident(&mut self) -> ManifestResult<String> {
match self.peek().clone() {
Token::Ident(s) => { self.advance(); Ok(s) }
// Some fields like "target" are keywords in El — allow them as idents here
Token::Target => { self.advance(); Ok("target".to_string()) }
other => Err(ManifestError::Parse {
line: self.current_line(),
reason: format!("expected identifier, found '{other}'"),
}),
}
}
/// Consume a string literal or return error.
fn expect_string(&mut self) -> ManifestResult<String> {
match self.peek().clone() {
Token::StringLiteral(s) => { self.advance(); Ok(s) }
other => Err(ManifestError::Parse {
line: self.current_line(),
reason: format!("expected string literal, found '{other}'"),
}),
}
}
/// Consume an integer literal or return error.
fn expect_int(&mut self) -> ManifestResult<i64> {
match self.peek().clone() {
Token::IntLiteral(n) => { self.advance(); Ok(n) }
other => Err(ManifestError::Parse {
line: self.current_line(),
reason: format!("expected integer literal, found '{other}'"),
}),
}
}
fn is_at_end(&self) -> bool {
matches!(self.peek(), Token::Eof)
}
}
// ── Entry point ───────────────────────────────────────────────────────────────
/// Parse a `manifest.el` source string into a [`Manifest`].
pub(crate) fn parse_manifest(s: &str) -> ManifestResult<Manifest> {
let spanned = tokenize(s).map_err(|e| ManifestError::Parse {
line: e.span.line,
reason: format!("lex error: {}", e.kind),
})?;
// Filter out comments and whitespace-only tokens (the lexer doesn't produce
// whitespace tokens, but Unknown chars from `//` comments might appear).
let tokens: Vec<Spanned<Token>> = spanned
.into_iter()
.filter(|t| !matches!(t.node, Token::Eof))
.collect();
let mut stream = TokenStream::new(tokens);
let mut raw_package: Option<RawPackage> = None;
let mut raw_build: Option<RawBuild> = None;
let mut raw_deps: HashMap<String, RawDep> = HashMap::new();
let mut raw_cross: Option<RawCross> = None;
let mut raw_app: Option<RawApp> = None;
while !stream.is_at_end() {
let section_name = match stream.peek().clone() {
Token::Ident(s) => { stream.advance(); s }
Token::Eof => break,
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected section name, found '{other}'"),
});
}
};
match section_name.as_str() {
"package" => {
// package "name" { ... }
let name = stream.expect_string()?;
stream.expect_lbrace()?;
raw_package = Some(parse_package_block(&mut stream, name)?);
stream.expect_rbrace()?;
}
"build" => {
stream.expect_lbrace()?;
raw_build = Some(parse_build_block(&mut stream)?);
stream.expect_rbrace()?;
}
"dependencies" => {
stream.expect_lbrace()?;
raw_deps = parse_deps_block(&mut stream)?;
stream.expect_rbrace()?;
}
"cross" => {
stream.expect_lbrace()?;
raw_cross = Some(parse_cross_block(&mut stream)?);
stream.expect_rbrace()?;
}
"app" => {
stream.expect_lbrace()?;
raw_app = Some(parse_app_block(&mut stream)?);
stream.expect_rbrace()?;
}
other => {
// Skip unknown sections gracefully by consuming until matching }
if matches!(stream.peek(), Token::LBrace) {
stream.advance();
skip_block(&mut stream)?;
} else {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("unknown section '{other}'"),
});
}
}
}
}
convert(raw_package, raw_build, raw_deps, raw_cross, raw_app)
}
// ── Block parsers ─────────────────────────────────────────────────────────────
#[derive(Debug, Default)]
struct RawPackage {
name: String,
version: String,
description: Option<String>,
authors: Vec<String>,
license: Option<String>,
edition: String,
entry: Option<String>, // neuron-code puts entry in [package]
}
fn parse_package_block(stream: &mut TokenStream, name: String) -> ManifestResult<RawPackage> {
let mut pkg = RawPackage {
name,
version: "0.1.0".to_string(),
edition: "2026".to_string(),
..Default::default()
};
while !matches!(stream.peek(), Token::RBrace | Token::Eof) {
let field = stream.expect_ident()?;
match field.as_str() {
"version" => pkg.version = stream.expect_string()?,
"description" => pkg.description = Some(stream.expect_string()?),
"authors" => pkg.authors = parse_string_array(stream)?,
"license" => pkg.license = Some(stream.expect_string()?),
"edition" => pkg.edition = stream.expect_string()?,
"entry" => pkg.entry = Some(stream.expect_string()?),
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("unknown package field '{other}'"),
});
}
}
}
Ok(pkg)
}
#[derive(Debug, Default)]
struct RawBuild {
target: Option<String>,
entry: Option<String>,
output: Option<String>,
seal_key: Option<String>,
}
fn parse_build_block(stream: &mut TokenStream) -> ManifestResult<RawBuild> {
let mut build = RawBuild::default();
while !matches!(stream.peek(), Token::RBrace | Token::Eof) {
let field = match stream.peek().clone() {
Token::Ident(s) => { stream.advance(); s }
Token::Target => { stream.advance(); "target".to_string() }
Token::RBrace | Token::Eof => break,
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected build field name, found '{other}'"),
});
}
};
match field.as_str() {
"target" => build.target = Some(stream.expect_string()?),
"entry" => build.entry = Some(stream.expect_string()?),
"output" => build.output = Some(stream.expect_string()?),
"seal_key" => build.seal_key = Some(stream.expect_string()?),
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("unknown build field '{other}'"),
});
}
}
}
Ok(build)
}
#[derive(Debug)]
enum RawDep {
Path(String),
Version(String),
}
fn parse_deps_block(stream: &mut TokenStream) -> ManifestResult<HashMap<String, RawDep>> {
let mut deps = HashMap::new();
while !matches!(stream.peek(), Token::RBrace | Token::Eof) {
// Dependency name can contain hyphens — the lexer will produce Ident tokens
// separated by Minus for names like `el-ui`. We need to reconstruct the full name.
let dep_name = parse_dep_name(stream)?;
// Value is either a string version or a block `{ path "..." }`
match stream.peek().clone() {
Token::StringLiteral(ver) => {
stream.advance();
deps.insert(dep_name, RawDep::Version(ver));
}
Token::LBrace => {
stream.advance();
// Parse inner fields until }
let mut path: Option<String> = None;
while !matches!(stream.peek(), Token::RBrace | Token::Eof) {
let field = stream.expect_ident()?;
match field.as_str() {
"path" => path = Some(stream.expect_string()?),
other => {
// Skip unknown fields (version, registry, etc.)
match stream.peek().clone() {
Token::StringLiteral(_) => { stream.advance(); }
_ => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("unknown dep field '{other}'"),
});
}
}
}
}
}
stream.expect_rbrace()?;
match path {
Some(p) => { deps.insert(dep_name, RawDep::Path(p)); }
None => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("dependency '{dep_name}' block has no 'path' or 'version'"),
});
}
}
}
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected string or block for dependency '{dep_name}', found '{other}'"),
});
}
}
}
Ok(deps)
}
/// Parse a possibly-hyphenated dependency name like `el-ui` or `engram-http`.
/// The lexer produces Ident("-") — actually Minus tokens — so we need to stitch
/// Ident + (Minus + Ident)* together.
fn parse_dep_name(stream: &mut TokenStream) -> ManifestResult<String> {
let mut name = match stream.peek().clone() {
Token::Ident(s) => { stream.advance(); s }
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected dependency name, found '{other}'"),
});
}
};
// Greedily consume `-ident` segments
loop {
match stream.peek() {
Token::Minus => {
// Peek ahead to see if next is Ident
let next_pos = stream.pos + 1;
if let Some(next) = stream.tokens.get(next_pos) {
if let Token::Ident(_) = &next.node {
stream.advance(); // consume Minus
if let Token::Ident(seg) = stream.peek().clone() {
stream.advance();
name.push('-');
name.push_str(&seg);
continue;
}
}
}
break;
}
_ => break,
}
}
Ok(name)
}
#[derive(Debug, Default)]
struct RawCross {
targets: Vec<String>,
}
fn parse_cross_block(stream: &mut TokenStream) -> ManifestResult<RawCross> {
let mut cross = RawCross::default();
while !matches!(stream.peek(), Token::RBrace | Token::Eof) {
let field = match stream.peek().clone() {
Token::Ident(s) => { stream.advance(); s }
Token::Target => { stream.advance(); "targets".to_string() }
Token::RBrace | Token::Eof => break,
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected cross field, found '{other}'"),
});
}
};
match field.as_str() {
"targets" => cross.targets = parse_string_array(stream)?,
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("unknown cross field '{other}'"),
});
}
}
}
Ok(cross)
}
#[derive(Debug, Default)]
struct RawApp {
window_title: Option<String>,
window_width: Option<i64>,
window_height: Option<i64>,
min_width: Option<i64>,
min_height: Option<i64>,
}
fn parse_app_block(stream: &mut TokenStream) -> ManifestResult<RawApp> {
let mut app = RawApp::default();
while !matches!(stream.peek(), Token::RBrace | Token::Eof) {
let field = stream.expect_ident()?;
match field.as_str() {
"window_title" => app.window_title = Some(stream.expect_string()?),
"window_width" => app.window_width = Some(stream.expect_int()?),
"window_height" => app.window_height = Some(stream.expect_int()?),
"min_width" => app.min_width = Some(stream.expect_int()?),
"min_height" => app.min_height = Some(stream.expect_int()?),
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("unknown app field '{other}'"),
});
}
}
}
Ok(app)
}
fn parse_string_array(stream: &mut TokenStream) -> ManifestResult<Vec<String>> {
match stream.peek() {
Token::LBracket => { stream.advance(); }
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected '[', found '{other}'"),
});
}
}
let mut items = Vec::new();
while !matches!(stream.peek(), Token::RBracket | Token::Eof) {
// Items in the array are strings
match stream.peek().clone() {
Token::StringLiteral(s) => {
stream.advance();
items.push(s);
// Optional comma
if matches!(stream.peek(), Token::Comma) {
stream.advance();
}
}
Token::Comma => { stream.advance(); } // trailing comma
other => {
return Err(ManifestError::Parse {
line: stream.current_line(),
reason: format!("expected string in array, found '{other}'"),
});
}
}
}
match stream.peek() {
Token::RBracket => { stream.advance(); }
_ => {} // Eof handled gracefully
}
Ok(items)
}
/// Skip over a `{ ... }` block (already consumed the opening brace).
fn skip_block(stream: &mut TokenStream) -> ManifestResult<()> {
let mut depth = 1;
while depth > 0 && !stream.is_at_end() {
match stream.peek() {
Token::LBrace => { depth += 1; stream.advance(); }
Token::RBrace => { depth -= 1; stream.advance(); }
_ => { stream.advance(); }
}
}
Ok(())
}
// ── Conversion to typed manifest ──────────────────────────────────────────────
fn convert(
raw_package: Option<RawPackage>,
raw_build: Option<RawBuild>,
raw_deps: HashMap<String, RawDep>,
raw_cross: Option<RawCross>,
raw_app: Option<RawApp>,
) -> ManifestResult<Manifest> {
let pkg_raw = raw_package.ok_or_else(|| ManifestError::MissingField("package".to_string()))?;
let package = convert_package(pkg_raw)?;
let build = convert_build(raw_build.unwrap_or_default(), &package)?;
let dependencies = convert_deps(raw_deps)?;
let cross = convert_cross(raw_cross.unwrap_or_default())?;
let app = raw_app.map(convert_app);
Ok(Manifest {
package,
dependencies,
dev_dependencies: HashMap::new(),
build,
cross,
plugins: HashMap::new(),
app,
})
}
fn convert_package(raw: RawPackage) -> ManifestResult<PackageInfo> {
let version = semver::Version::parse(&raw.version).map_err(|e| ManifestError::Semver {
field: "package.version".to_string(),
source: e,
})?;
Ok(PackageInfo {
name: raw.name,
version,
description: raw.description,
authors: raw.authors,
license: raw.license,
edition: raw.edition,
})
}
fn convert_build(raw: RawBuild, _package: &PackageInfo) -> ManifestResult<BuildConfig> {
let target = raw
.target
.unwrap_or_else(|| "debug".to_string())
.parse::<BuildTarget>()?;
// Entry can come from build block or package block (legacy neuron-code style)
let entry_str = raw.entry.unwrap_or_else(|| "src/main.el".to_string());
let output_str = raw.output.unwrap_or_else(|| "dist/".to_string());
let seal_key = raw
.seal_key
.map(|s| SealKeySource::parse(&s))
.transpose()?;
Ok(BuildConfig {
target,
entry: PathBuf::from(entry_str),
output: PathBuf::from(output_str),
seal_key,
})
}
fn convert_deps(raw: HashMap<String, RawDep>) -> ManifestResult<HashMap<String, Dependency>> {
let mut out = HashMap::new();
for (name, dep) in raw {
let converted = match dep {
RawDep::Path(p) => Dependency::Path(PathBuf::from(p)),
RawDep::Version(v) => {
let req = semver::VersionReq::parse(&v).map_err(|e| ManifestError::Semver {
field: format!("dependencies.{name}"),
source: e,
})?;
Dependency::VersionReq(req)
}
};
out.insert(name, converted);
}
Ok(out)
}
fn convert_cross(raw: RawCross) -> ManifestResult<CrossConfig> {
let targets = raw
.targets
.iter()
.map(|s| CrossTarget::parse(s))
.collect::<ManifestResult<Vec<_>>>()?;
Ok(CrossConfig { targets })
}
fn convert_app(raw: RawApp) -> AppConfig {
AppConfig {
window_title: raw.window_title.unwrap_or_default(),
window_width: raw.window_width.unwrap_or(1024) as u32,
window_height: raw.window_height.unwrap_or(768) as u32,
min_width: raw.min_width.unwrap_or(400) as u32,
min_height: raw.min_height.unwrap_or(300) as u32,
}
}
// ── Tests ─────────────────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
use crate::manifest::{BuildTarget, CrossTarget, Dependency};
fn full_manifest() -> &'static str {
r#"
// manifest.el — full example
package "my-service" {
version "0.1.0"
description "What this does"
authors ["Will Anderson <will@neurontechnologies.ai>"]
license "MIT"
edition "2026"
}
build {
target "prod"
entry "src/main.el"
output "dist/"
seal_key "env:ENGRAM_SEAL_KEY"
}
dependencies {
el-ui { path "../el-ui" }
}
cross {
targets ["x86_64-linux", "aarch64-linux", "x86_64-macos", "aarch64-macos", "wasm32"]
}
app {
window_title "My App"
window_width 1200
window_height 800
min_width 800
min_height 600
}
"#
}
#[test]
fn test_parse_full_manifest() {
let m = parse_manifest(full_manifest()).unwrap();
assert_eq!(m.package.name, "my-service");
assert_eq!(m.package.version.to_string(), "0.1.0");
assert_eq!(m.package.edition, "2026");
assert_eq!(m.package.license.as_deref(), Some("MIT"));
assert_eq!(m.package.authors, vec!["Will Anderson <will@neurontechnologies.ai>"]);
}
#[test]
fn test_parse_build_config() {
let m = parse_manifest(full_manifest()).unwrap();
assert_eq!(m.build.target, BuildTarget::Prod);
assert_eq!(m.build.entry.to_str().unwrap(), "src/main.el");
assert_eq!(m.build.output.to_str().unwrap(), "dist/");
match &m.build.seal_key {
Some(SealKeySource::EnvVar(v)) => assert_eq!(v, "ENGRAM_SEAL_KEY"),
other => panic!("expected EnvVar seal_key, got {other:?}"),
}
}
#[test]
fn test_parse_path_dep() {
let m = parse_manifest(full_manifest()).unwrap();
assert!(m.dependencies.contains_key("el-ui"));
match &m.dependencies["el-ui"] {
Dependency::Path(p) => assert_eq!(p.to_str().unwrap(), "../el-ui"),
other => panic!("expected Path dep, got {other:?}"),
}
}
#[test]
fn test_parse_cross_targets() {
let m = parse_manifest(full_manifest()).unwrap();
assert_eq!(m.cross.targets.len(), 5);
assert!(m.cross.targets.contains(&CrossTarget::X86_64Linux));
assert!(m.cross.targets.contains(&CrossTarget::Wasm32));
}
#[test]
fn test_parse_app_config() {
let m = parse_manifest(full_manifest()).unwrap();
let app = m.app.as_ref().expect("app config missing");
assert_eq!(app.window_title, "My App");
assert_eq!(app.window_width, 1200);
assert_eq!(app.window_height, 800);
assert_eq!(app.min_width, 800);
assert_eq!(app.min_height, 600);
}
#[test]
fn test_minimal_manifest() {
let src = r#"
package "hello" {
version "0.1.0"
}
"#;
let m = parse_manifest(src).unwrap();
assert_eq!(m.package.name, "hello");
assert_eq!(m.package.edition, "2026");
assert_eq!(m.build.target, BuildTarget::Debug);
assert!(m.dependencies.is_empty());
assert!(m.cross.targets.is_empty());
}
#[test]
fn test_missing_package_error() {
let src = r#"
build {
entry "src/main.el"
}
"#;
let err = parse_manifest(src).unwrap_err();
assert!(err.to_string().contains("package"), "expected package error, got: {err}");
}
}
@@ -84,12 +84,13 @@ pub enum BinOp {
Add, Sub, Mul, Div,
Eq, NotEq, Lt, Gt, LtEq, GtEq,
And, Or,
Mod, // %
BitAnd, // &
BitOr, // | (single pipe)
BitXor, // ^
Shl, // <<
Shr, // >>
Mod, // %
BitAnd, // &
BitOr, // | (single pipe)
BitXor, // ^
Shl, // <<
Shr, // >>
NullCoalesce, // ??
}
// ── Unary operators ───────────────────────────────────────────────────────────
@@ -160,6 +161,26 @@ pub enum Expr {
label: String,
body: Vec<Stmt>,
},
/// A JSX element: `<TagName attr="val">{children}</TagName>` or `<TagName />`
JsxElement {
tag: String,
attrs: Vec<(String, JsxAttrValue)>,
children: Vec<Expr>,
self_closing: bool,
},
/// A JSX expression interpolation: `{expr}`
JsxExpr(Box<Expr>),
/// Plain text content inside a JSX element.
JsxText(String),
}
/// The value of a JSX attribute.
#[derive(Debug, Clone, PartialEq)]
pub enum JsxAttrValue {
/// `attr="literal"` — a string literal
Str(String),
/// `attr={expr}` — an expression
Expr(Box<Expr>),
}
// ── Match arm ─────────────────────────────────────────────────────────────────
@@ -308,6 +329,29 @@ pub enum Stmt {
target: String,
span: Span,
},
/// `component Name { props { ... } state { ... } fn ... template { ... } }`
ComponentDef {
name: String,
/// Fields declared in the `props { }` block: (name, type_ann, default_expr)
props: Vec<ComponentField>,
/// Fields declared in the `state { }` block: (name, type_ann, default_expr)
state: Vec<ComponentField>,
/// Methods declared inside the component: plain `fn` definitions.
methods: Vec<Stmt>,
/// The JSX tree inside `template { }`.
template: Box<Expr>,
span: Span,
},
}
/// A field in a component `props` or `state` block.
#[derive(Debug, Clone, PartialEq)]
pub struct ComponentField {
pub name: String,
pub type_ann: TypeExpr,
/// Default value expression (may be a Literal::Str("") placeholder for required props).
pub default: Option<Box<Expr>>,
pub span: Span,
}
// ── Top-level program ─────────────────────────────────────────────────────────
@@ -11,8 +11,8 @@ mod error;
mod parser;
pub use ast::{
BinOp, UnaryOp, Decorator, Expr, Field, Literal, MatchArm, Param, Pattern, Program, ProtocolMethod,
SeedStmt, Stmt, TestTarget, TypeExpr, Variant,
BinOp, UnaryOp, ComponentField, Decorator, Expr, Field, JsxAttrValue, Literal, MatchArm,
Param, Pattern, Program, ProtocolMethod, SeedStmt, Stmt, TestTarget, TypeExpr, Variant,
};
pub use error::{ParseError, ParseErrorKind};
pub use parser::parse;
@@ -5,6 +5,10 @@ use el_lexer::{Span, Spanned, Token};
use crate::ast::*;
use crate::error::{ParseError, ParseErrorKind};
// ── For keyword in impl Statements ────────────────────────────────────────────
// The `for` keyword is re-used in `impl Protocol for TypeName`.
// (Token::For already exists in el_lexer.)
/// Parse a token stream into a [`Program`].
///
/// The `source` string is stored verbatim in the returned program for
@@ -124,6 +128,10 @@ impl Parser {
Token::Deploy => "deploy".to_string(),
Token::To => "to".to_string(),
Token::Via => "via".to_string(),
Token::Component => "component".to_string(),
Token::Props => "props".to_string(),
Token::State => "state".to_string(),
Token::Template => "template".to_string(),
tok => return Err(ParseError::new(
ParseErrorKind::ExpectedIdent(tok.to_string()),
span,
@@ -164,7 +172,22 @@ impl Parser {
match self.peek().clone() {
Token::Let => self.parse_let(start),
Token::Fn => self.parse_fn_def(start, vec![]),
Token::At => self.parse_decorated_fn(start),
Token::At => {
// Look ahead: `@name fn` or `@name @` → decorated function definition.
// `@name(` or `@name` followed by anything else → builtin call expression.
// Peek 2 tokens ahead: current=@, next=ident, then after-ident
let after_ident = self.tokens.get(self.pos + 2).map(|s| &s.node);
let is_decorator = matches!(after_ident, Some(Token::Fn) | Some(Token::At));
if is_decorator {
self.parse_decorated_fn(start)
} else {
// Parse as expression statement (builtin call)
let expr = self.parse_expr()?;
let span = start;
self.eat(&Token::Semicolon);
Ok(Stmt::Expr(expr, span))
}
}
Token::Type => self.parse_type_def(start),
Token::Enum => self.parse_enum_def(start),
Token::Test => self.parse_test_def(start),
@@ -177,14 +200,26 @@ impl Parser {
Token::While => self.parse_while(start),
Token::Retry => self.parse_retry(start),
Token::Deploy => self.parse_deploy(start),
Token::Component => self.parse_component_def(start),
Token::Return => {
self.advance(); // consume `return`
let expr = self.parse_expr()?;
// `return` with no value (bare return) is valid — treat as return ().
// Detect by checking if the next token can start an expression.
let expr = if matches!(self.peek(), Token::RBrace | Token::Semicolon | Token::Eof) || !is_expr_start(self.peek()) {
Expr::Block(vec![]) // empty block evaluates to Nil
} else {
self.parse_expr()?
};
let span = Span::new(start.start, expr_span_end(&expr, start), start.line, start.col);
self.eat(&Token::Semicolon);
Ok(Stmt::Return(expr, span))
}
_ => {
// Check for assignment: `ident = expr` or `ident.field = expr`
// This handles state mutation inside components.
if let Some(assign_stmt) = self.try_parse_assignment(start)? {
return Ok(assign_stmt);
}
let expr = self.parse_expr()?;
let span = start;
self.eat(&Token::Semicolon);
@@ -193,6 +228,33 @@ impl Parser {
}
}
/// Try to parse `ident = expr` or `ident.field = expr` as an assignment statement.
/// Returns None if the pattern doesn't match (restores position).
fn try_parse_assignment(&mut self, start: Span) -> Result<Option<Stmt>, ParseError> {
let saved = self.pos;
// Only try if current token is an identifier
let name = match self.peek().clone() {
Token::Ident(n) => n,
_ => return Ok(None),
};
self.advance();
// Check for direct assignment: `name =`
if self.eat(&Token::Eq) {
// `name = expr` — compile as `let name = expr` (reassignment)
let value = self.parse_expr()?;
self.eat(&Token::Semicolon);
return Ok(Some(Stmt::Let {
name,
type_ann: None,
value,
span: start,
}));
}
// Restore position — not an assignment
self.pos = saved;
Ok(None)
}
fn parse_test_def(&mut self, start: Span) -> Result<Stmt, ParseError> {
self.expect(&Token::Test)?;
// test name is a string literal
@@ -562,6 +624,322 @@ impl Parser {
Ok(Stmt::Deploy { fn_name, route, target, span: start })
}
/// Parse `component Name { props { ... } state { ... } fn ...* template { ... } }`
fn parse_component_def(&mut self, start: Span) -> Result<Stmt, ParseError> {
self.expect(&Token::Component)?;
let (name, _) = self.expect_ident()?;
self.expect(&Token::LBrace)?;
let mut props: Vec<ComponentField> = Vec::new();
let mut state: Vec<ComponentField> = Vec::new();
let mut methods: Vec<Stmt> = Vec::new();
let mut template: Expr = Expr::JsxElement {
tag: "div".to_string(),
attrs: vec![],
children: vec![],
self_closing: false,
};
while !matches!(self.peek(), Token::RBrace | Token::Eof) {
while self.eat(&Token::Semicolon) {}
if matches!(self.peek(), Token::RBrace | Token::Eof) { break; }
match self.peek().clone() {
Token::Props => {
self.advance(); // consume `props`
self.expect(&Token::LBrace)?;
props = self.parse_component_fields()?;
self.expect(&Token::RBrace)?;
}
Token::State => {
self.advance(); // consume `state`
self.expect(&Token::LBrace)?;
state = self.parse_component_fields()?;
self.expect(&Token::RBrace)?;
}
Token::Template => {
self.advance(); // consume `template`
self.expect(&Token::LBrace)?;
template = self.parse_jsx_content()?;
self.expect(&Token::RBrace)?;
}
Token::Fn => {
let method_start = self.peek_span();
let m = self.parse_fn_def(method_start, vec![])?;
methods.push(m);
}
Token::At => {
let method_start = self.peek_span();
let m = self.parse_decorated_fn(method_start)?;
methods.push(m);
}
_ => {
// Skip unknown tokens inside component block gracefully
self.advance();
}
}
}
self.expect(&Token::RBrace)?;
Ok(Stmt::ComponentDef { name, props, state, methods, template: Box::new(template), span: start })
}
/// Parse a list of component fields (for props/state blocks):
/// `field_name: TypeExpr = default_expr`
fn parse_component_fields(&mut self) -> Result<Vec<ComponentField>, ParseError> {
let mut fields = Vec::new();
while !matches!(self.peek(), Token::RBrace | Token::Eof) {
while self.eat(&Token::Semicolon) {}
if matches!(self.peek(), Token::RBrace | Token::Eof) { break; }
let span = self.peek_span();
let (fname, _) = self.expect_ident()?;
self.expect(&Token::Colon)?;
let type_ann = self.parse_type_expr()?;
let default = if self.eat(&Token::Eq) {
Some(Box::new(self.parse_expr()?))
} else {
None
};
fields.push(ComponentField { name: fname, type_ann, default, span });
self.eat(&Token::Comma);
self.eat(&Token::Semicolon);
}
Ok(fields)
}
/// Parse JSX content inside a template block: a sequence of JSX nodes.
/// Returns the first/root JSX node (or a wrapping fragment if multiple).
fn parse_jsx_content(&mut self) -> Result<Expr, ParseError> {
let mut children = self.parse_jsx_children()?;
if children.len() == 1 {
Ok(children.remove(0))
} else {
// Wrap in a fragment div
Ok(Expr::JsxElement {
tag: "fragment".to_string(),
attrs: vec![],
children,
self_closing: false,
})
}
}
/// Parse a sequence of JSX children until we hit `}` (for template blocks)
/// or `</` (for closing tags).
fn parse_jsx_children(&mut self) -> Result<Vec<Expr>, ParseError> {
let mut children = Vec::new();
loop {
match self.peek().clone() {
// End of template block or EOF
Token::RBrace | Token::Eof => break,
// JSX expression interpolation: {expr}
Token::LBrace => {
self.advance(); // consume `{`
// Check if this is just a closing brace (empty) - unlikely but safe
if matches!(self.peek(), Token::RBrace) {
self.advance();
continue;
}
let expr = self.parse_expr()?;
// Consume closing `}` only if it's there
// (for map/block literals inside JSX, the expr parser may consume it)
if matches!(self.peek(), Token::RBrace) {
self.advance();
}
children.push(Expr::JsxExpr(Box::new(expr)));
}
// Opening tag: `<TagName` or closing tag start `</`
Token::Lt => {
// Check if it's a closing tag `</`
let is_closing = matches!(self.tokens.get(self.pos + 1), Some(s) if s.node == Token::Slash);
if is_closing {
break; // closing tag signals end of children list
}
// Parse opening tag
let elem = self.parse_jsx_element()?;
children.push(elem);
}
// Hash sign in JSX text (e.g. `#{tag}` or `#RRGGBB`)
Token::Hash => {
self.advance();
// Always emit "#" as text; the next token (if `{`) becomes a JsxExpr
children.push(Expr::JsxText("#".to_string()));
}
// Unknown character (non-ASCII, emoji, etc.) in JSX text — emit as text
Token::Unknown(c) => {
let text = c.to_string();
self.advance();
children.push(Expr::JsxText(text));
}
// Anything else: treat as text (skip unknown tokens)
_ => {
self.advance();
}
}
}
Ok(children)
}
/// Parse a JSX element: `<TagName attrs...>children</TagName>` or `<TagName attrs... />`
fn parse_jsx_element(&mut self) -> Result<Expr, ParseError> {
self.expect(&Token::Lt)?; // consume `<`
// Tag name: could be `Ident` (uppercase for components, lowercase for HTML tags)
let tag = match self.peek().clone() {
Token::Ident(name) => { self.advance(); name }
tok => return Err(ParseError::expected("tag name", &tok, self.peek_span())),
};
// Parse attributes until `>` or `/>`
let mut attrs = Vec::new();
loop {
match self.peek().clone() {
// End of opening tag
Token::Gt => {
self.advance(); // consume `>`
break;
}
// Self-closing: `/>`
Token::Slash => {
self.advance(); // consume `/`
self.expect(&Token::Gt)?; // consume `>`
return Ok(Expr::JsxElement {
tag,
attrs,
children: vec![],
self_closing: true,
});
}
Token::Eof => break,
// Attribute name: could be identifier or `on:click` style (ident:ident)
Token::Ident(attr_name) => {
self.advance();
// Check for `on:click` style (colon-qualified event name)
let full_attr_name = if matches!(self.peek(), Token::Colon) {
self.advance(); // consume `:`
let (event_name, _) = self.expect_ident()?;
format!("{}:{}", attr_name, event_name)
} else {
attr_name
};
// Attribute value: `="literal"` or `={expr}`
if self.eat(&Token::Eq) {
let attr_val = match self.peek().clone() {
Token::StringLiteral(s) => {
self.advance();
JsxAttrValue::Str(s)
}
Token::LBrace => {
self.advance(); // consume `{`
let expr = self.parse_expr()?;
self.expect(&Token::RBrace)?;
JsxAttrValue::Expr(Box::new(expr))
}
tok => return Err(ParseError::expected("string literal or {expr}", &tok, self.peek_span())),
};
attrs.push((full_attr_name, attr_val));
} else {
// Boolean attribute (no value)
attrs.push((full_attr_name, JsxAttrValue::Str("true".to_string())));
}
}
_ => {
// Skip unknown tokens in attribute position
self.advance();
}
}
}
// Now parse children until `</tag>`
let children = self.parse_jsx_children()?;
// Consume closing tag: `</TagName>`
if matches!(self.peek(), Token::Lt) {
self.advance(); // consume `<`
if self.eat(&Token::Slash) {
// Consume tag name (we don't validate it matches)
if matches!(self.peek(), Token::Ident(_)) {
self.advance();
}
// Consume `>`
let _ = self.eat(&Token::Gt);
}
}
Ok(Expr::JsxElement {
tag,
attrs,
children,
self_closing: false,
})
}
/// Try to parse `(ident, ident, ...) => expr` arrow function.
/// If the pattern doesn't match, restore position and return None.
/// Precondition: `(` has already been consumed.
fn try_parse_arrow_fn(&mut self, span: Span) -> Result<Option<Expr>, ParseError> {
let saved_pos = self.pos;
let mut params: Vec<String> = Vec::new();
// Try to parse `ident, ident, ... )` followed by `=>`
loop {
match self.peek().clone() {
Token::Ident(name) => {
params.push(name);
self.advance();
// After ident: could be `,` or `)` or `: Type`
if self.eat(&Token::Colon) {
// Typed param: skip the type expression by advancing past it
// Simple type: ident or ident?
if matches!(self.peek(), Token::Ident(_)) {
self.advance();
self.eat(&Token::QuestionMark);
}
}
if self.eat(&Token::Comma) {
continue;
}
if matches!(self.peek(), Token::RParen) {
break;
}
// Not an arrow fn pattern
self.pos = saved_pos;
return Ok(None);
}
Token::RParen => break,
_ => {
self.pos = saved_pos;
return Ok(None);
}
}
}
self.expect(&Token::RParen)?;
// Check for `=>`
if !self.eat(&Token::FatArrow) {
self.pos = saved_pos;
return Ok(None);
}
// Parse body
let body = self.parse_expr()?;
let closure_params = params.into_iter().map(|name| Param {
name,
type_ann: TypeExpr::Named("_".to_string()),
span,
}).collect();
Ok(Some(Expr::Closure {
params: closure_params,
return_type: None,
body: Box::new(body),
span,
}))
}
#[allow(dead_code)]
fn parse_param_list(&mut self) -> Result<Vec<Param>, ParseError> {
self.parse_param_list_with_type_params(&[])
@@ -662,8 +1040,8 @@ impl Parser {
span,
)),
};
// Check for function type: fn(A) -> B
if name == "fn" {
// Check for function type: fn(A) -> B OR Fn(A) -> B
if name == "fn" || name == "Fn" {
self.expect(&Token::LParen)?;
let mut params = Vec::new();
while !matches!(self.peek(), Token::RParen | Token::Eof) {
@@ -714,7 +1092,35 @@ impl Parser {
// ── Expressions ───────────────────────────────────────────────────────────
fn parse_expr(&mut self) -> Result<Expr, ParseError> {
self.parse_pipe_expr()
self.parse_ternary_expr()
}
/// Parse ternary: `cond ? then_expr : else_expr`
/// Also handles null-coalescing: `a ?? b` (returns `b` if `a` is nil/empty).
fn parse_ternary_expr(&mut self) -> Result<Expr, ParseError> {
let cond = self.parse_pipe_expr()?;
// Null-coalescing: `a ?? b` — desugars to a binary NullCoalesce op
if self.eat(&Token::NullCoalesce) {
let default_expr = self.parse_ternary_expr()?;
return Ok(Expr::BinOp {
op: BinOp::NullCoalesce,
left: Box::new(cond),
right: Box::new(default_expr),
});
}
if self.eat(&Token::QuestionMark) {
// Both branches allow nested ternaries (right-associative).
// `a ? b ? c : d : e` → `a ? (b ? c : d) : e`
let then_expr = self.parse_ternary_expr()?;
self.expect(&Token::Colon)?;
let else_expr = self.parse_ternary_expr()?;
return Ok(Expr::If {
cond: Box::new(cond),
then: Box::new(then_expr),
else_: Some(Box::new(else_expr)),
});
}
Ok(cond)
}
fn parse_bitwise_or_expr(&mut self) -> Result<Expr, ParseError> {
@@ -811,7 +1217,22 @@ impl Parser {
let mut left = self.parse_bitwise_or_expr()?;
loop {
let op = match self.peek() {
Token::Lt => BinOp::Lt,
Token::Lt => {
// Disambiguate `<` as less-than vs the start of a JSX element `<TagName`.
// Heuristic: if `<` is followed by an identifier AND `<` appears on a
// different line than the preceding token, treat it as JSX (not comparison).
// Same-line `x < y` is always a comparison; newline-`\n<Tag>` is always JSX.
let after_lt = self.tokens.get(self.pos + 1).map(|s| &s.node);
if matches!(after_lt, Some(Token::Ident(_))) {
let lt_line = self.tokens[self.pos].span.line;
let prev_line = if self.pos > 0 { self.tokens[self.pos - 1].span.line } else { lt_line };
if lt_line != prev_line {
// `<` is on a new line — it's a JSX element, not a comparison
break;
}
}
BinOp::Lt
}
Token::Gt => BinOp::Gt,
Token::LtEq => BinOp::LtEq,
Token::GtEq => BinOp::GtEq,
@@ -864,6 +1285,15 @@ impl Parser {
let inner = self.parse_unary()?;
return Ok(Expr::UnaryBitNot(Box::new(inner)));
}
if self.eat(&Token::Minus) {
let inner = self.parse_unary()?;
// Desugar: `-x` → `0 - x`
return Ok(Expr::BinOp {
op: BinOp::Sub,
left: Box::new(Expr::Literal(crate::ast::Literal::Int(0))),
right: Box::new(inner),
});
}
self.parse_postfix()
}
@@ -873,7 +1303,16 @@ impl Parser {
match self.peek() {
Token::Dot => {
self.advance();
let (field, _) = self.expect_ident()?;
// Allow keywords as field names (e.g. `e.target`, `e.type`, `e.key`)
let field = match self.peek().clone() {
Token::Ident(name) => { self.advance(); name }
// Any other token used as a field name (keywords, builtins)
tok => {
let name = tok.to_string();
self.advance();
name
}
};
expr = Expr::Field { object: Box::new(expr), field };
}
Token::LParen => {
@@ -889,6 +1328,14 @@ impl Parser {
expr = Expr::Index { object: Box::new(expr), index: Box::new(index) };
}
Token::QuestionMark => {
// Disambiguate: `?` as Try postfix vs `?` as ternary start.
// If the token AFTER `?` can start an expression, this is ternary
// and we should NOT consume it here (let parse_ternary_expr handle it).
let after_q = self.tokens.get(self.pos + 1).map(|s| &s.node);
if after_q.is_some_and(is_expr_start) {
// Ternary — stop postfix loop, bubble up to parse_ternary_expr
break;
}
self.advance();
expr = Expr::Try(Box::new(expr));
}
@@ -906,6 +1353,13 @@ impl Parser {
self.expect(&Token::RBrace)?;
expr = Expr::With { base: Box::new(expr), updates };
}
// Type cast: `expr as TypeName` — treated as a no-op cast (same value, type hint only)
Token::As => {
self.advance(); // consume `as`
// Parse the target type expression (we don't use it at runtime)
let _ = self.parse_type_expr();
// expr remains unchanged — the cast is informational
}
_ => break,
}
}
@@ -930,11 +1384,51 @@ impl Parser {
Token::StringLiteral(s) => { self.advance(); Ok(Expr::Literal(Literal::Str(s))) }
Token::BoolLiteral(b) => { self.advance(); Ok(Expr::Literal(Literal::Bool(b))) }
// Grouped or tuple
// Grouped expression or arrow function: `(expr)` or `(params) => body` or `() => body`
Token::LParen => {
self.advance();
self.advance(); // consume `(`
// Empty parens `()` — must be `() => expr` arrow function
if matches!(self.peek(), Token::RParen) {
self.advance(); // consume `)`
if self.eat(&Token::FatArrow) {
let body = self.parse_expr()?;
return Ok(Expr::Closure {
params: vec![],
return_type: None,
body: Box::new(body),
span,
});
}
// `()` with no `=>` — return nil literal as placeholder
return Ok(Expr::Literal(Literal::Bool(false)));
}
// Check if this is `(ident, ident, ...)` or `(ident: Type, ...)` arrow params
// by attempting to parse as param list with look-ahead
if let Some(arrow_fn) = self.try_parse_arrow_fn(span)? {
return Ok(arrow_fn);
}
// Otherwise parse as grouped expression
let expr = self.parse_expr()?;
self.expect(&Token::RParen)?;
// Check again for `=>` (in case try_parse_arrow_fn consumed nothing)
if self.eat(&Token::FatArrow) {
// Treat the expr as a single untyped param name
let param_name = match &expr {
Expr::Ident(name) => name.clone(),
_ => "__arg__".to_string(),
};
let body = self.parse_expr()?;
return Ok(Expr::Closure {
params: vec![Param {
name: param_name,
type_ann: TypeExpr::Named("_".to_string()),
span,
}],
return_type: None,
body: Box::new(body),
span,
});
}
Ok(expr)
}
@@ -1061,11 +1555,26 @@ impl Parser {
Ok(Expr::Path { segments })
} else if matches!(self.peek(), Token::LBrace)
&& name.chars().next().map(|c| c.is_uppercase()).unwrap_or(false)
&& self.is_struct_literal_start()
{
// Struct literal: TypeName { field: expr, ... }
// Struct literal: TypeName { field: expr, ...spread, field: expr, ... }
self.advance(); // consume `{`
let mut fields = Vec::new();
while !matches!(self.peek(), Token::RBrace | Token::Eof) {
// Skip spread syntax: `...expr` (e.g. `...base`)
// Three dots followed by an expression — skip the spread for now (use base fields)
if matches!(self.peek(), Token::Dot) {
// Try to consume `...expr` — eat up to 3 dots then an expression
let mut dots = 0;
while matches!(self.peek(), Token::Dot) && dots < 3 {
self.advance();
dots += 1;
}
// Parse the spread source expression (we discard it for now)
let _ = self.parse_primary()?;
if !self.eat(&Token::Comma) { break; }
continue;
}
let (field_name, _) = self.expect_ident()?;
self.expect(&Token::Colon)?;
let field_expr = self.parse_expr()?;
@@ -1121,6 +1630,44 @@ impl Parser {
Ok(Expr::Trace { label, body })
}
// JSX element: `<TagName ...>` — heuristic: `<` followed by ident
Token::Lt => {
let next = self.tokens.get(self.pos + 1).map(|s| &s.node);
if matches!(next, Some(Token::Ident(_))) {
self.parse_jsx_element()
} else {
// It's a comparison operator — shouldn't be in primary, error
Err(ParseError::new(
ParseErrorKind::InvalidExprStart(Token::Lt.to_string()),
span,
))
}
}
// `@builtin_name(args)` — runtime/syscall expression.
// Compiled as Call { name: "__builtin_name__", arity }.
// Also handles `@call(...)`, `@emit(...)`, `@on(...)`, `@uuid()` etc.
Token::At => {
self.advance(); // consume `@`
let (name, _) = self.expect_ident()?;
let args = if self.eat(&Token::LParen) {
let mut a = Vec::new();
while !matches!(self.peek(), Token::RParen | Token::Eof) {
a.push(self.parse_expr()?);
if !self.eat(&Token::Comma) { break; }
}
self.expect(&Token::RParen)?;
a
} else {
vec![]
};
// Emit as Call { func: Ident("@name"), args }
Ok(Expr::Call {
func: Box::new(Expr::Ident(format!("@{name}"))),
args,
})
}
tok => Err(ParseError::new(
ParseErrorKind::InvalidExprStart(tok.to_string()),
span,
@@ -1130,6 +1677,30 @@ impl Parser {
/// Heuristic: are we at the start of a map literal?
/// We peek ahead for `string/ident :` pattern.
/// Check if the current `{` (already consumed by outer code for TypeName {)
/// looks like a struct literal body rather than a block body.
/// Heuristic: if token at pos+1 (first inside `{`) is:
/// - `Ident Colon` → struct field → yes
/// - `Dot Dot Dot` → spread → yes
/// - `RBrace` (empty) → yes (empty struct)
/// - anything else → no (treat as block / enum variant)
/// Called when current token is `{` (LBrace) and we're checking if TypeName { } is a struct.
fn is_struct_literal_start(&self) -> bool {
// self.pos points to `{` (the LBrace we haven't consumed yet)
// Look at pos+1: first token inside `{`
let first = self.tokens.get(self.pos + 1).map(|t| &t.node);
match first {
Some(Token::RBrace) => true, // empty struct literal `TypeName {}`
Some(Token::Dot) => true, // spread `...expr`
Some(Token::Ident(_)) => {
// Check if the token AFTER the ident is `:`
let after_ident = self.tokens.get(self.pos + 2).map(|t| &t.node);
matches!(after_ident, Some(Token::Colon))
}
_ => false, // Not a struct literal — treat as a block/expression context
}
}
fn is_map_literal(&self) -> bool {
// Look at current token (first inside `{`)
match self.peek() {
@@ -1202,6 +1773,35 @@ fn expr_span_end(_expr: &Expr, fallback: Span) -> usize {
fallback.end
}
/// Returns true if `tok` can start an expression.
/// Used to disambiguate `?` as Try vs ternary.
fn is_expr_start(tok: &Token) -> bool {
matches!(
tok,
Token::Ident(_)
| Token::IntLiteral(_)
| Token::FloatLiteral(_)
| Token::StringLiteral(_)
| Token::BoolLiteral(_)
| Token::LParen
| Token::LBracket
| Token::LBrace
| Token::Minus
| Token::Not
| Token::Tilde
| Token::Pipe // closure
| Token::Match
| Token::If
| Token::Activate
| Token::Sealed
| Token::Reason
| Token::Parallel
| Token::Trace
| Token::Lt // JSX element
| Token::At // @builtin call
)
}
// ── Tests ─────────────────────────────────────────────────────────────────────
#[cfg(test)]
@@ -43,7 +43,7 @@ pub fn register(env: &mut TypeEnv) {
// array_concat([T], [T]) -> [T]
env.register_fn("array_concat", fn_type(vec![arr_unk.clone(), arr_unk.clone()], arr_unk.clone()));
// array_slice([T], Int, Int) -> [T]
env.register_fn("array_slice", fn_type(vec![arr_unk.clone(), Type::Int, Type::Int], arr_unk));
env.register_fn("array_slice", fn_type(vec![arr_unk.clone(), Type::Int, Type::Int], arr_unk.clone()));
// array_first([T]) -> T?
env.register_fn("array_first", fn_type(vec![arr_int.clone()], Type::Optional(Box::new(Type::Int))));
// array_last([T]) -> T?
@@ -378,6 +378,15 @@ impl<'g> Evaluator<'g> {
(EvalValue::Int(a), EvalValue::Int(b)) => Ok(EvalValue::Int(a >> (b as u32))),
_ => Ok(EvalValue::Int(0)),
},
BinOp::NullCoalesce => {
// `a ?? b` — use b if a is nil/empty/false
match &lv {
EvalValue::Nil => Ok(rv),
EvalValue::Str(s) if s.is_empty() => Ok(rv),
EvalValue::Bool(false) => Ok(rv),
_ => Ok(lv),
}
}
}
}
@@ -440,6 +449,7 @@ pub fn expr_to_text(expr: &Expr) -> String {
BinOp::And => "&&", BinOp::Or => "||",
BinOp::Mod => "%", BinOp::BitAnd => "&", BinOp::BitOr => "|",
BinOp::BitXor => "^", BinOp::Shl => "<<", BinOp::Shr => ">>",
BinOp::NullCoalesce => "??",
};
format!("{} {op_str} {}", expr_to_text(left), expr_to_text(right))
}

Some files were not shown because too many files have changed in this diff Show More