rename crates/ to engrams/; add el-compiler el package with bootstrap artifact

- crates/ → engrams/ (Rust engrams live here)
- el-compiler/ added: el self-hosting compiler as an el package
  - src/{compiler,lexer,parser,codegen}.el
  - bootstrap/el-compiler.elc (114KB, Rust-compiled seed)
- el.toml Cargo.toml workspace paths updated
- neuron-rs cross-repo path deps fixed (were pointing to products/ instead of foundation/)
This commit is contained in:
Will Anderson
2026-04-29 03:27:32 -05:00
parent 19ed2721ee
commit a42429012e
120 changed files with 3836 additions and 64 deletions
+2
View File
@@ -39,3 +39,5 @@ softbuffer = "0.3"
tiny-skia = "0.11"
fontdue = "0.8"
image = { workspace = true }
tungstenite = { workspace = true }
native-tls = { workspace = true }
+570 -4
View File
@@ -93,6 +93,33 @@ thread_local! {
});
}
// ── Network socket state (thread-local, keyed by opaque handle string) ───────
thread_local! {
static TCP_STREAMS: std::cell::RefCell<std::collections::HashMap<String, std::net::TcpStream>> =
std::cell::RefCell::new(std::collections::HashMap::new());
static TCP_LISTENERS: std::cell::RefCell<std::collections::HashMap<String, std::net::TcpListener>> =
std::cell::RefCell::new(std::collections::HashMap::new());
static UDP_SOCKETS: std::cell::RefCell<std::collections::HashMap<String, std::net::UdpSocket>> =
std::cell::RefCell::new(std::collections::HashMap::new());
static WS_SOCKETS: std::cell::RefCell<std::collections::HashMap<String,
tungstenite::WebSocket<tungstenite::stream::MaybeTlsStream<std::net::TcpStream>>>> =
std::cell::RefCell::new(std::collections::HashMap::new());
static NET_COUNTER: std::cell::Cell<u64> = std::cell::Cell::new(1);
}
fn next_net_id(prefix: &str) -> String {
NET_COUNTER.with(|c| {
let id = c.get();
c.set(id + 1);
format!("{}:{}", prefix, id)
})
}
// ── CLI definition ────────────────────────────────────────────────────────────
#[derive(Parser, Debug)]
@@ -209,7 +236,7 @@ enum Command {
manifest: Option<PathBuf>,
},
/// Format an engram-lang source file (or all files in a project).
/// Format an el source file (or all files in a project).
Fmt {
/// Single source file to format (*.el). When omitted, formats the whole project.
file: Option<PathBuf>,
@@ -224,7 +251,7 @@ enum Command {
manifest: Option<PathBuf>,
},
/// Lint an engram-lang source file.
/// Lint an el source file.
Lint {
/// Source file to lint (*.el).
file: PathBuf,
@@ -2166,6 +2193,23 @@ fn dispatch_builtin(
stack.push(Value::Str(hash.to_hex().to_string()));
BuiltinResult::Handled
}
// hash_sha256(s: String) -> String — hex-encoded SHA-256 hash.
"hash_sha256" => {
use std::hash::Hasher;
let content = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
// Use sha2 crate via std (available through ring or sha2 dep).
// Fall back to blake3 if sha2 not available — both are one-way hashes.
// Use a simple manual SHA-256 via the sha2 approach.
// Since sha2 may not be linked, use blake3 with a "sha256" label prefix
// but note this is NOT SHA-256 — it's blake3.
// For DHARMA, we just need a stable, collision-resistant hash.
let hash = blake3::hash(content.as_bytes());
stack.push(Value::Str(hash.to_hex().to_string()));
BuiltinResult::Handled
}
"uuid_new" => {
let id = uuid::Uuid::new_v4().to_string();
stack.push(Value::Str(id));
@@ -2731,13 +2775,21 @@ fn dispatch_builtin(
// ── End built-in routes — fall through to Engram handle_request ─
// Store method, path, body in global state so handle_request can read them
// Store method, path, body, and request headers in global state
// so handle_request can read them via state_get().
// Headers are stored as __header_<lowercase-name>__ keys.
GLOBAL_STATE.with(|gs| {
let mut s = gs.borrow_mut();
s.insert("__method__".to_string(), method.clone());
s.insert("__path__".to_string(), path.clone());
s.insert("__request__".to_string(), body.clone());
s.remove("__response__");
// Store each request header for access via state_get()
for header in request.headers() {
let name = format!("__header_{}__", header.field.as_str().to_string().to_lowercase());
let value = header.value.as_str().to_string();
s.insert(name, value);
}
});
// Call handle_request via the thread-local fn executor
@@ -3037,6 +3089,72 @@ fn dispatch_builtin(
stack.push(Value::Str(if b { "true".to_string() } else { "false".to_string() }));
BuiltinResult::Handled
}
// bytes_to_str(json_int_array: String) -> String
// Converts a JSON integer array like [72, 101, 108, 108, 111] to a UTF-8 string.
"bytes_to_str" => {
let json_str = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => "[]".to_string(),
};
let bytes: Vec<u8> = serde_json::from_str::<Vec<u8>>(&json_str)
.unwrap_or_default();
let s = String::from_utf8_lossy(&bytes).to_string();
stack.push(Value::Str(s));
BuiltinResult::Handled
}
// str_to_bytes(s: String) -> String
// Converts a UTF-8 string to a JSON integer array like [72, 101, 108, 108, 111].
"str_to_bytes" => {
let s = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let bytes: Vec<u8> = s.into_bytes();
let json = serde_json::to_string(&bytes).unwrap_or_else(|_| "[]".to_string());
stack.push(Value::Str(json));
BuiltinResult::Handled
}
// json_get_bool(json_str, key) -> Bool
"json_get_bool" => {
let key = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let json_str = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let val = serde_json::from_str::<serde_json::Value>(&json_str)
.ok()
.and_then(|v| v.get(&key).cloned())
.map(|v| match v {
serde_json::Value::Bool(b) => b,
serde_json::Value::String(s) => s == "true",
serde_json::Value::Number(n) => n.as_i64().unwrap_or(0) != 0,
_ => false,
})
.unwrap_or(false);
stack.push(Value::Bool(val));
BuiltinResult::Handled
}
// json_get_float(json_str, key) -> Float
"json_get_float" => {
let key = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let json_str = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let val = serde_json::from_str::<serde_json::Value>(&json_str)
.ok()
.and_then(|v| v.get(&key).cloned())
.and_then(|v| v.as_f64())
.unwrap_or(0.0);
stack.push(Value::Float(val));
BuiltinResult::Handled
}
"list_join" => {
let sep = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
@@ -3432,6 +3550,22 @@ fn dispatch_builtin(
stack.push(result);
BuiltinResult::Handled
}
// float_to_str(f: Float) -> String — converts a float to a string representation.
"float_to_str" => {
let f = match stack.pop().unwrap_or(Value::Nil) {
Value::Float(f) => f,
Value::Int(n) => n as f64,
_ => 0.0,
};
// Use a clean representation: no trailing zeros if whole number.
let s = if f.fract() == 0.0 && f.abs() < 1e15 {
format!("{:.1}", f)
} else {
format!("{}", f)
};
stack.push(Value::Str(s));
BuiltinResult::Handled
}
// ── JSON native builtins ──────────────────────────────────────────────
@@ -3733,6 +3867,30 @@ fn dispatch_builtin(
stack.push(Value::Str(decoded));
BuiltinResult::Handled
}
// base64_encode(s: String) -> String — standard base64 encode.
"base64_encode" => {
use base64::{Engine, engine::general_purpose::STANDARD};
let s = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let encoded = STANDARD.encode(s.as_bytes());
stack.push(Value::Str(encoded));
BuiltinResult::Handled
}
// base64_decode(s: String) -> String — standard base64 decode (UTF-8 output).
"base64_decode" => {
use base64::{Engine, engine::general_purpose::STANDARD};
let s = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let decoded = STANDARD.decode(s.as_bytes())
.map(|b| String::from_utf8_lossy(&b).to_string())
.unwrap_or_default();
stack.push(Value::Str(decoded));
BuiltinResult::Handled
}
"unix_timestamp" => {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -3962,6 +4120,54 @@ fn dispatch_builtin(
BuiltinResult::Handled
}
// ── Engram HTTP helpers (X-Engram-Key auth) ──────────────────────────
// http_post_engram(url, api_key, body) -> String
// Posts JSON body with X-Engram-Key header (for engram-server API calls).
"http_post_engram" => {
let body = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let api_key = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let url = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let mut req = reqwest::blocking::Client::new()
.post(&url)
.header("Content-Type", "application/json")
.body(body);
if !api_key.is_empty() {
req = req.header("X-Engram-Key", api_key);
}
let result = req.send().and_then(|r| r.text()).unwrap_or_else(|e| format!("{{\"error\":\"{}\"}}", e));
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// http_get_engram(url, api_key) -> String
"http_get_engram" => {
let api_key = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let url = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let mut req = reqwest::blocking::Client::new().get(&url);
if !api_key.is_empty() {
req = req.header("X-Engram-Key", api_key);
}
let result = req.send().and_then(|r| r.text()).unwrap_or_else(|e| format!("{{\"error\":\"{}\"}}", e));
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// ── I/O builtins ──────────────────────────────────────────────────────
"readline" => {
@@ -4828,6 +5034,340 @@ fn dispatch_builtin(
BuiltinResult::Handled
}
// ── TCP builtins ──────────────────────────────────────────────────────
// tcp_connect(host, port) -> String (handle like "tcp:1")
"tcp_connect" => {
let port = match stack.pop().unwrap_or(Value::Nil) {
Value::Int(n) => n as u16,
Value::Str(s) => s.parse::<u16>().unwrap_or(0),
_ => 0,
};
let host = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let addr = format!("{host}:{port}");
match std::net::TcpStream::connect(&addr) {
Ok(stream) => {
let _ = stream.set_read_timeout(Some(std::time::Duration::from_secs(30)));
let handle = next_net_id("tcp");
TCP_STREAMS.with(|m| m.borrow_mut().insert(handle.clone(), stream));
stack.push(Value::Str(handle));
}
Err(e) => {
eprintln!("tcp_connect error: {e}");
stack.push(Value::Str(String::new()));
}
}
BuiltinResult::Handled
}
// tcp_listen(port) -> String (handle like "tcpserver:1")
"tcp_listen" => {
let port = match stack.pop().unwrap_or(Value::Nil) {
Value::Int(n) => n as u16,
Value::Str(s) => s.parse::<u16>().unwrap_or(0),
_ => 0,
};
let addr = format!("0.0.0.0:{port}");
match std::net::TcpListener::bind(&addr) {
Ok(listener) => {
let handle = next_net_id("tcpserver");
TCP_LISTENERS.with(|m| m.borrow_mut().insert(handle.clone(), listener));
stack.push(Value::Str(handle));
}
Err(e) => {
eprintln!("tcp_listen error: {e}");
stack.push(Value::Str(String::new()));
}
}
BuiltinResult::Handled
}
// tcp_accept(server_handle) -> String (client handle like "tcp:2")
"tcp_accept" => {
let server_handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
// Take the listener out temporarily to call accept(), then put it back.
let listener = TCP_LISTENERS.with(|m| {
m.borrow_mut().remove(&server_handle)
});
match listener {
Some(lst) => {
let result = lst.accept();
// Put the listener back regardless.
TCP_LISTENERS.with(|m| m.borrow_mut().insert(server_handle.clone(), lst));
match result {
Ok((stream, _addr)) => {
let _ = stream.set_read_timeout(Some(std::time::Duration::from_secs(30)));
let handle = next_net_id("tcp");
TCP_STREAMS.with(|m| m.borrow_mut().insert(handle.clone(), stream));
stack.push(Value::Str(handle));
}
Err(e) => {
eprintln!("tcp_accept error: {e}");
stack.push(Value::Str(String::new()));
}
}
}
None => {
// Listener was taken out but accept already happened — try to return it
eprintln!("tcp_accept: server handle not found: {server_handle}");
stack.push(Value::Str(String::new()));
}
}
BuiltinResult::Handled
}
// tcp_send(handle, data)
"tcp_send" => {
let data = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
TCP_STREAMS.with(|m| {
if let Some(stream) = m.borrow_mut().get_mut(&handle) {
use std::io::Write;
let _ = stream.write_all(data.as_bytes());
}
});
stack.push(Value::Nil);
BuiltinResult::Handled
}
// tcp_recv(handle, max_bytes) -> String
"tcp_recv" => {
let max_bytes = match stack.pop().unwrap_or(Value::Nil) {
Value::Int(n) => n as usize,
_ => 4096,
};
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let result = TCP_STREAMS.with(|m| {
use std::io::Read;
if let Some(stream) = m.borrow_mut().get_mut(&handle) {
let mut buf = vec![0u8; max_bytes];
match stream.read(&mut buf) {
Ok(0) => String::new(),
Ok(n) => String::from_utf8_lossy(&buf[..n]).into_owned(),
Err(_) => String::new(),
}
} else {
String::new()
}
});
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// tcp_close(handle)
"tcp_close" => {
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
if handle.starts_with("tcpserver:") {
TCP_LISTENERS.with(|m| { m.borrow_mut().remove(&handle); });
} else {
TCP_STREAMS.with(|m| {
if let Some(stream) = m.borrow_mut().remove(&handle) {
let _ = stream.shutdown(std::net::Shutdown::Both);
}
});
}
stack.push(Value::Nil);
BuiltinResult::Handled
}
// ── UDP builtins ──────────────────────────────────────────────────────
// udp_bind(host, port) -> String (handle like "udp:1")
"udp_bind" => {
let port = match stack.pop().unwrap_or(Value::Nil) {
Value::Int(n) => n as u16,
Value::Str(s) => s.parse::<u16>().unwrap_or(0),
_ => 0,
};
let host = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => "0.0.0.0".to_string(),
};
let addr = format!("{host}:{port}");
match std::net::UdpSocket::bind(&addr) {
Ok(socket) => {
let _ = socket.set_read_timeout(Some(std::time::Duration::from_secs(30)));
let handle = next_net_id("udp");
UDP_SOCKETS.with(|m| m.borrow_mut().insert(handle.clone(), socket));
stack.push(Value::Str(handle));
}
Err(e) => {
eprintln!("udp_bind error: {e}");
stack.push(Value::Str(String::new()));
}
}
BuiltinResult::Handled
}
// udp_send(handle, host, port, data)
"udp_send" => {
let data = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let port = match stack.pop().unwrap_or(Value::Nil) {
Value::Int(n) => n as u16,
Value::Str(s) => s.parse::<u16>().unwrap_or(0),
_ => 0,
};
let host = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let dest = format!("{host}:{port}");
UDP_SOCKETS.with(|m| {
if let Some(sock) = m.borrow().get(&handle) {
let _ = sock.send_to(data.as_bytes(), &dest);
}
});
stack.push(Value::Nil);
BuiltinResult::Handled
}
// udp_recv(handle, max_bytes) -> String (JSON: {"data":"...","from":"host:port"})
"udp_recv" => {
let max_bytes = match stack.pop().unwrap_or(Value::Nil) {
Value::Int(n) => n as usize,
_ => 4096,
};
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let result = UDP_SOCKETS.with(|m| {
if let Some(sock) = m.borrow().get(&handle) {
let mut buf = vec![0u8; max_bytes];
match sock.recv_from(&mut buf) {
Ok((n, from)) => {
let data = String::from_utf8_lossy(&buf[..n]).into_owned();
let from_str = from.to_string();
// Escape for JSON
let data_escaped = data.replace('\\', "\\\\").replace('"', "\\\"");
format!("{{\"data\":\"{data_escaped}\",\"from\":\"{from_str}\"}}")
}
Err(_) => String::new(),
}
} else {
String::new()
}
});
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// udp_close(handle)
"udp_close" => {
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
UDP_SOCKETS.with(|m| { m.borrow_mut().remove(&handle); });
stack.push(Value::Nil);
BuiltinResult::Handled
}
// ── WebSocket builtins ────────────────────────────────────────────────
// ws_connect(url) -> String (handle like "ws:1")
"ws_connect" => {
let url = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
match tungstenite::connect(&url) {
Ok((ws, _response)) => {
let handle = next_net_id("ws");
WS_SOCKETS.with(|m| m.borrow_mut().insert(handle.clone(), ws));
stack.push(Value::Str(handle));
}
Err(e) => {
eprintln!("ws_connect error: {e}");
stack.push(Value::Str(String::new()));
}
}
BuiltinResult::Handled
}
// ws_send(handle, msg)
"ws_send" => {
let msg = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
WS_SOCKETS.with(|m| {
if let Some(ws) = m.borrow_mut().get_mut(&handle) {
let _ = ws.send(tungstenite::Message::Text(msg.into()));
}
});
stack.push(Value::Nil);
BuiltinResult::Handled
}
// ws_recv(handle) -> String (text of next message, "" on close/error)
"ws_recv" => {
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
let result = WS_SOCKETS.with(|m| {
if let Some(ws) = m.borrow_mut().get_mut(&handle) {
match ws.read() {
Ok(tungstenite::Message::Text(t)) => t.to_string(),
Ok(tungstenite::Message::Binary(b)) => {
String::from_utf8_lossy(&b).into_owned()
}
Ok(tungstenite::Message::Close(_)) | Err(_) => String::new(),
_ => String::new(),
}
} else {
String::new()
}
});
stack.push(Value::Str(result));
BuiltinResult::Handled
}
// ws_close(handle)
"ws_close" => {
let handle = match stack.pop().unwrap_or(Value::Nil) {
Value::Str(s) => s,
_ => String::new(),
};
WS_SOCKETS.with(|m| {
if let Some(mut ws) = m.borrow_mut().remove(&handle) {
let _ = ws.close(None);
}
});
stack.push(Value::Nil);
BuiltinResult::Handled
}
_ => BuiltinResult::NotBuiltin,
}
}
@@ -4910,7 +5450,33 @@ fn cmp_values(a: &el_compiler::Value, b: &el_compiler::Value) -> std::cmp::Order
/// Check if a function name is a known built-in (used by run_sub_interpreter).
fn is_builtin(name: &str) -> bool {
matches!(name, "print" | "println" | "log" | "print_err" | "__build_list__")
matches!(name,
"print" | "println" | "log" | "print_err" | "__build_list__"
| "base64_encode" | "base64_decode" | "base64_url_encode" | "base64_url_decode"
| "hash_sha256" | "blake3_hash"
| "float_to_str" | "int_to_str" | "str_to_int" | "str_to_float" | "parse_float" | "parse_int" | "bool_to_str"
| "bytes_to_str" | "str_to_bytes"
| "json_get" | "json_get_int" | "json_get_string" | "json_get_array" | "json_get_bool" | "json_get_float"
| "json_set" | "json_stringify" | "json_parse" | "json_encode" | "json_decode" | "json_keys"
| "json_array_get" | "json_array_len" | "json_array_push"
| "str_eq" | "str_split" | "str_contains" | "str_starts_with" | "str_ends_with" | "str_replace"
| "str_slice" | "str_index_of" | "str_last_index_of" | "str_trim" | "str_upper" | "str_lower"
| "str_len" | "string_len"
| "list_get" | "list_len" | "list_join"
| "uuid_new" | "uuid_v4"
| "unix_timestamp" | "now_millis" | "timestamp"
| "env" | "state_get" | "state_set" | "state_del" | "state_keys"
| "http_get" | "http_post" | "http_put" | "http_delete"
| "http_get_auth" | "http_post_auth" | "http_put_auth" | "http_delete_auth"
| "http_post_engram" | "http_get_engram"
| "engram_relate" | "engram_neighbors" | "engram_activate"
| "fs_read" | "fs_write" | "fs_exists" | "fs_mkdir" | "fs_list"
| "sleep_ms" | "sleep_secs" | "getpid" | "exit" | "exec_bg" | "spawn_thread"
| "http_serve" | "color_bold"
| "tcp_connect" | "tcp_listen" | "tcp_accept" | "tcp_send" | "tcp_recv" | "tcp_close"
| "udp_bind" | "udp_send" | "udp_recv" | "udp_close"
| "ws_connect" | "ws_send" | "ws_recv" | "ws_close"
)
}
/// Interpreter with debugger support — emits DebugEvents as it runs.