rename crates/ to engrams/; add el-compiler el package with bootstrap artifact

- crates/ → engrams/ (Rust engrams live here)
- el-compiler/ added: el self-hosting compiler as an el package
  - src/{compiler,lexer,parser,codegen}.el
  - bootstrap/el-compiler.elc (114KB, Rust-compiled seed)
- el.toml Cargo.toml workspace paths updated
- neuron-rs cross-repo path deps fixed (were pointing to products/ instead of foundation/)
This commit is contained in:
Will Anderson
2026-04-29 03:27:32 -05:00
parent 19ed2721ee
commit a42429012e
120 changed files with 3836 additions and 64 deletions
+719
View File
@@ -0,0 +1,719 @@
// codegen.el el self-hosting bytecode code generator
//
// Input: list of AST statement maps (from parser.el)
// Output: JSON string encoding an array of bytecode instructions
//
// Bytecode JSON format matches the Rust serde output exactly.
// See the el-compiler/src/bytecode.rs for the canonical enum.
//
// Entry point: fn codegen(stmts: [Map<String, Any>], source: String) -> String
// JSON helpers
// Escape a string for JSON embedding.
fn json_escape(s: String) -> String {
let chars: [String] = native_string_chars(s)
let total: Int = native_list_len(chars)
let out = ""
let i = 0
while i < total {
let ch: String = native_list_get(chars, i)
if ch == "\"" {
let out = out + "\\\""
} else {
if ch == "\\" {
let out = out + "\\\\"
} else {
if ch == "\n" {
let out = out + "\\n"
} else {
if ch == "\r" {
let out = out + "\\r"
} else {
if ch == "\t" {
let out = out + "\\t"
} else {
let out = out + ch
}
}
}
}
}
let i = i + 1
}
out
}
fn json_str(s: String) -> String {
"\"" + json_escape(s) + "\""
}
fn json_int(n: Int) -> String {
native_int_to_str(n)
}
fn json_bool(b: Bool) -> String {
if b { return "true" }
"false"
}
// Codegen state
//
// el has no mutable globals. We thread a "ctx" map through all codegen
// functions. The context holds:
// "instrs" -> [String] JSON strings for each emitted instruction
// "patches" -> [Map<String,Any>] pending forward-jump patches
// each patch: { "idx": Int, "kind": String }
// (kind is "JumpIfNot" / "Jump" we store the instruction index and
// patch it at the end of the construct)
fn ctx_new() -> Map<String, Any> {
{ "instrs": native_list_empty(), "patches": native_list_empty() }
}
fn ctx_emit(ctx: Map<String, Any>, instr_json: String) -> Map<String, Any> {
let instrs = ctx["instrs"]
let instrs = native_list_append(instrs, instr_json)
{ "instrs": instrs, "patches": ctx["patches"] }
}
fn ctx_len(ctx: Map<String, Any>) -> Int {
let instrs = ctx["instrs"]
native_list_len(instrs)
}
// Patch a previously-emitted placeholder instruction at index idx.
// For Jump/JumpIf/JumpIfNot, we need to replace the entry in instrs.
// We rebuild the list with the patched value at position idx.
fn ctx_patch(ctx: Map<String, Any>, idx: Int, dest: Int) -> Map<String, Any> {
// offset = dest - (idx + 1)
let offset = dest - (idx + 1)
let instrs = ctx["instrs"]
let total = native_list_len(instrs)
let new_instrs: [String] = native_list_empty()
let i = 0
while i < total {
let instr: String = native_list_get(instrs, i)
if i == idx {
// Replace: determine kind from original instruction string
if native_string_contains(instr, "JumpIfNot") {
let new_instrs = native_list_append(new_instrs, "{\"JumpIfNot\":" + json_int(offset) + "}")
} else {
if native_string_contains(instr, "JumpIf") {
let new_instrs = native_list_append(new_instrs, "{\"JumpIf\":" + json_int(offset) + "}")
} else {
let new_instrs = native_list_append(new_instrs, "{\"Jump\":" + json_int(offset) + "}")
}
}
} else {
let new_instrs = native_list_append(new_instrs, instr)
}
let i = i + 1
}
{ "instrs": new_instrs, "patches": ctx["patches"] }
}
// Instruction emitters
fn emit_halt(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Halt\"")
}
fn emit_nop(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Nop\"")
}
fn emit_pop(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Pop\"")
}
fn emit_return(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Return\"")
}
fn emit_add(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Add\"")
}
fn emit_sub(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Sub\"")
}
fn emit_mul(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Mul\"")
}
fn emit_div(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Div\"")
}
fn emit_eq(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Eq\"")
}
fn emit_not_eq(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"NotEq\"")
}
fn emit_lt(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Lt\"")
}
fn emit_gt(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Gt\"")
}
fn emit_lt_eq(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"LtEq\"")
}
fn emit_gt_eq(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"GtEq\"")
}
fn emit_and(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"And\"")
}
fn emit_or(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Or\"")
}
fn emit_not(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"Not\"")
}
fn emit_get_index(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "\"GetIndex\"")
}
fn emit_push_nil(ctx: Map<String, Any>) -> Map<String, Any> {
ctx_emit(ctx, "{\"Push\":\"Nil\"}")
}
fn emit_push_int(ctx: Map<String, Any>, n: Int) -> Map<String, Any> {
ctx_emit(ctx, "{\"Push\":{\"Int\":" + json_int(n) + "}}")
}
fn emit_push_bool(ctx: Map<String, Any>, b: Bool) -> Map<String, Any> {
ctx_emit(ctx, "{\"Push\":{\"Bool\":" + json_bool(b) + "}}")
}
fn emit_push_str(ctx: Map<String, Any>, s: String) -> Map<String, Any> {
ctx_emit(ctx, "{\"Push\":{\"Str\":" + json_str(s) + "}}")
}
fn emit_load(ctx: Map<String, Any>, name: String) -> Map<String, Any> {
ctx_emit(ctx, "{\"LoadLocal\":" + json_str(name) + "}")
}
fn emit_store(ctx: Map<String, Any>, name: String) -> Map<String, Any> {
ctx_emit(ctx, "{\"StoreLocal\":" + json_str(name) + "}")
}
fn emit_call(ctx: Map<String, Any>, name: String, arity: Int) -> Map<String, Any> {
ctx_emit(ctx, "{\"Call\":{\"name\":" + json_str(name) + ",\"arity\":" + json_int(arity) + "}}")
}
fn emit_get_field(ctx: Map<String, Any>, field: String) -> Map<String, Any> {
ctx_emit(ctx, "{\"GetField\":" + json_str(field) + "}")
}
fn emit_build_map(ctx: Map<String, Any>, n: Int) -> Map<String, Any> {
ctx_emit(ctx, "{\"BuildMap\":" + json_int(n) + "}")
}
fn emit_build_list(ctx: Map<String, Any>, n: Int) -> Map<String, Any> {
ctx_emit(ctx, "{\"BuildList\":" + json_int(n) + "}")
}
// Emit a placeholder Jump and return its index (for later patching)
fn emit_jump_placeholder(ctx: Map<String, Any>) -> Map<String, Any> {
let idx = ctx_len(ctx)
let ctx = ctx_emit(ctx, "{\"Jump\":0}")
{ "ctx": ctx, "idx": idx }
}
fn emit_jump_if_not_placeholder(ctx: Map<String, Any>) -> Map<String, Any> {
let idx = ctx_len(ctx)
let ctx = ctx_emit(ctx, "{\"JumpIfNot\":0}")
{ "ctx": ctx, "idx": idx }
}
fn emit_jump_to(ctx: Map<String, Any>, dest: Int) -> Map<String, Any> {
let here = ctx_len(ctx)
let offset = dest - (here + 1)
ctx_emit(ctx, "{\"Jump\":" + json_int(offset) + "}")
}
// Expression codegen
fn cg_expr(ctx: Map<String, Any>, expr: Map<String, Any>) -> Map<String, Any> {
let kind: String = expr["expr"]
if kind == "Int" {
let v: String = expr["value"]
let n: Int = native_str_to_int(v)
return emit_push_int(ctx, n)
}
if kind == "Float" {
let v: String = expr["value"]
// Push as string for now VM handles float parsing from Str
return emit_push_str(ctx, v)
}
if kind == "Str" {
let v: String = expr["value"]
return emit_push_str(ctx, v)
}
if kind == "Bool" {
let v: String = expr["value"]
if v == "true" {
return emit_push_bool(ctx, true)
}
return emit_push_bool(ctx, false)
}
if kind == "Nil" {
return emit_push_nil(ctx)
}
if kind == "Ident" {
let name: String = expr["name"]
return emit_load(ctx, name)
}
if kind == "Not" {
let inner = expr["inner"]
let ctx = cg_expr(ctx, inner)
return emit_not(ctx)
}
if kind == "Neg" {
// unary minus: push 0, push inner, sub
let ctx = emit_push_int(ctx, 0)
let inner = expr["inner"]
let ctx = cg_expr(ctx, inner)
return emit_sub(ctx)
}
if kind == "BinOp" {
let op: String = expr["op"]
let left = expr["left"]
let right = expr["right"]
let ctx = cg_expr(ctx, left)
let ctx = cg_expr(ctx, right)
if op == "Plus" { return emit_add(ctx) }
if op == "Minus" { return emit_sub(ctx) }
if op == "Star" { return emit_mul(ctx) }
if op == "Slash" { return emit_div(ctx) }
if op == "EqEq" { return emit_eq(ctx) }
if op == "NotEq" { return emit_not_eq(ctx) }
if op == "Lt" { return emit_lt(ctx) }
if op == "Gt" { return emit_gt(ctx) }
if op == "LtEq" { return emit_lt_eq(ctx) }
if op == "GtEq" { return emit_gt_eq(ctx) }
if op == "And" { return emit_and(ctx) }
if op == "Or" { return emit_or(ctx) }
return ctx
}
if kind == "Call" {
let func = expr["func"]
let args = expr["args"]
let arity: Int = native_list_len(args)
// push args left-to-right
let i = 0
while i < arity {
let arg = native_list_get(args, i)
let ctx = cg_expr(ctx, arg)
let i = i + 1
}
// get function name from func expr
let func_kind: String = func["expr"]
if func_kind == "Ident" {
let fn_name: String = func["name"]
return emit_call(ctx, fn_name, arity)
}
if func_kind == "Field" {
let obj = func["object"]
let field: String = func["field"]
let ctx = cg_expr(ctx, obj)
return emit_call(ctx, field, arity + 1)
}
return emit_call(ctx, "__dynamic__", arity)
}
if kind == "Field" {
let obj = expr["object"]
let field: String = expr["field"]
let ctx = cg_expr(ctx, obj)
return emit_get_field(ctx, field)
}
if kind == "Index" {
let obj = expr["object"]
let idx = expr["index"]
let ctx = cg_expr(ctx, obj)
let ctx = cg_expr(ctx, idx)
return emit_get_index(ctx)
}
if kind == "Array" {
let elems = expr["elems"]
let n: Int = native_list_len(elems)
let i = 0
while i < n {
let elem = native_list_get(elems, i)
let ctx = cg_expr(ctx, elem)
let i = i + 1
}
return emit_build_list(ctx, n)
}
if kind == "Map" {
let pairs = expr["pairs"]
let n: Int = native_list_len(pairs)
let i = 0
while i < n {
let pair = native_list_get(pairs, i)
let key: String = pair["key"]
let val = pair["value"]
let ctx = emit_push_str(ctx, key)
let ctx = cg_expr(ctx, val)
let i = i + 1
}
return emit_build_map(ctx, n)
}
if kind == "If" {
let cond = expr["cond"]
let then_stmts = expr["then"]
let else_stmts = expr["else"]
let has_else: Bool = expr["has_else"]
// cond
let ctx = cg_expr(ctx, cond)
// JumpIfNot placeholder
let r = emit_jump_if_not_placeholder(ctx)
let ctx = r["ctx"]
let jump_false_idx: Int = r["idx"]
// then body
let ctx = cg_stmts(ctx, then_stmts)
if has_else {
// jump over else
let r2 = emit_jump_placeholder(ctx)
let ctx = r2["ctx"]
let jump_end_idx: Int = r2["idx"]
// patch jump_false to here
let else_start = ctx_len(ctx)
let ctx = ctx_patch(ctx, jump_false_idx, else_start)
// else body
let ctx = cg_stmts(ctx, else_stmts)
// patch jump_end to here
let after_else = ctx_len(ctx)
let ctx = ctx_patch(ctx, jump_end_idx, after_else)
return ctx
} else {
let after_then = ctx_len(ctx)
let ctx = ctx_patch(ctx, jump_false_idx, after_then)
return ctx
}
}
if kind == "Match" {
let subject = expr["subject"]
let arms = expr["arms"]
let n_arms: Int = native_list_len(arms)
let ctx = cg_expr(ctx, subject)
// store subject in temp var
let ctx = emit_store(ctx, "__match_subj__")
let end_jump_idxs: [Int] = native_list_empty()
let i = 0
while i < n_arms {
let arm = native_list_get(arms, i)
let pattern = arm["pattern"]
let body = arm["body"]
let pat_kind: String = pattern["pattern"]
if pat_kind == "Wildcard" {
// always matches just emit body
let ctx = cg_expr(ctx, body)
let r = emit_jump_placeholder(ctx)
let ctx = r["ctx"]
let jidx: Int = r["idx"]
let end_jump_idxs = native_list_append(end_jump_idxs, jidx)
} else {
if pat_kind == "Binding" {
let bind_name: String = pattern["name"]
let ctx = emit_load(ctx, "__match_subj__")
let ctx = emit_store(ctx, bind_name)
let ctx = cg_expr(ctx, body)
let r = emit_jump_placeholder(ctx)
let ctx = r["ctx"]
let jidx: Int = r["idx"]
let end_jump_idxs = native_list_append(end_jump_idxs, jidx)
} else {
// literal pattern: compare subject to literal
let ctx = emit_load(ctx, "__match_subj__")
if pat_kind == "LitInt" {
let v: String = pattern["value"]
let n: Int = native_str_to_int(v)
let ctx = emit_push_int(ctx, n)
} else {
if pat_kind == "LitStr" {
let v: String = pattern["value"]
let ctx = emit_push_str(ctx, v)
} else {
if pat_kind == "LitBool" {
let v: String = pattern["value"]
if v == "true" {
let ctx = emit_push_bool(ctx, true)
} else {
let ctx = emit_push_bool(ctx, false)
}
} else {
let ctx = emit_push_nil(ctx)
}
}
}
let ctx = emit_eq(ctx)
let r = emit_jump_if_not_placeholder(ctx)
let ctx = r["ctx"]
let no_match_idx: Int = r["idx"]
let ctx = cg_expr(ctx, body)
let r2 = emit_jump_placeholder(ctx)
let ctx = r2["ctx"]
let jidx: Int = r2["idx"]
let end_jump_idxs = native_list_append(end_jump_idxs, jidx)
let next_arm = ctx_len(ctx)
let ctx = ctx_patch(ctx, no_match_idx, next_arm)
}
}
let i = i + 1
}
// default: push nil
let ctx = emit_push_nil(ctx)
let end_pos = ctx_len(ctx)
// patch all end jumps
let n_end: Int = native_list_len(end_jump_idxs)
let j = 0
while j < n_end {
let jidx: Int = native_list_get(end_jump_idxs, j)
let ctx = ctx_patch(ctx, jidx, end_pos)
let j = j + 1
}
return ctx
}
if kind == "For" {
// for item in list { body }
// Implementation:
// __for_list__ = list
// __for_len__ = len(list)
// __for_i__ = 0
// loop_start:
// if __for_i__ >= __for_len__ goto done
// item = __for_list__[__for_i__]
// body
// __for_i__ = __for_i__ + 1
// goto loop_start
// done:
let item: String = expr["item"]
let list_expr = expr["list"]
let body = expr["body"]
// emit list, store it
let ctx = cg_expr(ctx, list_expr)
let ctx = emit_store(ctx, "__for_list__")
// compute length
let ctx = emit_load(ctx, "__for_list__")
let ctx = emit_call(ctx, "native_list_len", 1)
let ctx = emit_store(ctx, "__for_len__")
// init counter
let ctx = emit_push_int(ctx, 0)
let ctx = emit_store(ctx, "__for_i__")
// loop start
let loop_start = ctx_len(ctx)
// condition: __for_i__ < __for_len__
let ctx = emit_load(ctx, "__for_i__")
let ctx = emit_load(ctx, "__for_len__")
let ctx = emit_lt(ctx)
let r = emit_jump_if_not_placeholder(ctx)
let ctx = r["ctx"]
let to_done_idx: Int = r["idx"]
// get current element
let ctx = emit_load(ctx, "__for_list__")
let ctx = emit_load(ctx, "__for_i__")
let ctx = emit_get_index(ctx)
let ctx = emit_store(ctx, item)
// body
let ctx = cg_stmts(ctx, body)
// increment counter
let ctx = emit_load(ctx, "__for_i__")
let ctx = emit_push_int(ctx, 1)
let ctx = emit_add(ctx)
let ctx = emit_store(ctx, "__for_i__")
// jump back
let ctx = emit_jump_to(ctx, loop_start)
// patch done
let done_pos = ctx_len(ctx)
let ctx = ctx_patch(ctx, to_done_idx, done_pos)
return ctx
}
if kind == "Try" {
// Just emit the inner expression error propagation handled at runtime
let inner = expr["inner"]
return cg_expr(ctx, inner)
}
// Fallback
emit_push_nil(ctx)
}
// Statement codegen
fn cg_stmt(ctx: Map<String, Any>, stmt: Map<String, Any>) -> Map<String, Any> {
let kind: String = stmt["stmt"]
if kind == "Let" {
let name: String = stmt["name"]
let val = stmt["value"]
let ctx = cg_expr(ctx, val)
return emit_store(ctx, name)
}
if kind == "Return" {
let val = stmt["value"]
let ctx = cg_expr(ctx, val)
return emit_return(ctx)
}
if kind == "FnDef" {
let fn_name: String = stmt["name"]
let params = stmt["params"]
let body = stmt["body"]
let n_params: Int = native_list_len(params)
// emit a Jump to skip over the function body
let r = emit_jump_placeholder(ctx)
let ctx = r["ctx"]
let skip_jump_idx: Int = r["idx"]
// function body entry: store params in reverse order (caller pushes LR, so pop RL)
let pi = n_params - 1
while pi >= 0 {
let param = native_list_get(params, pi)
let pname: String = param["name"]
let ctx = emit_store(ctx, pname)
let pi = pi - 1
}
// emit body statements
let ctx = cg_stmts(ctx, body)
// implicit nil return
let ctx = emit_push_nil(ctx)
let ctx = emit_return(ctx)
// patch skip jump
let after_fn = ctx_len(ctx)
let ctx = ctx_patch(ctx, skip_jump_idx, after_fn)
// register function entry point
let entry_ip = skip_jump_idx + 1
let ctx = emit_push_int(ctx, entry_ip)
let ctx = emit_store(ctx, "__fn_" + fn_name)
return ctx
}
if kind == "While" {
let cond = stmt["cond"]
let body = stmt["body"]
let loop_start = ctx_len(ctx)
let ctx = cg_expr(ctx, cond)
let r = emit_jump_if_not_placeholder(ctx)
let ctx = r["ctx"]
let to_done_idx: Int = r["idx"]
let ctx = cg_stmts(ctx, body)
let ctx = emit_jump_to(ctx, loop_start)
let done_pos = ctx_len(ctx)
let ctx = ctx_patch(ctx, to_done_idx, done_pos)
return ctx
}
if kind == "For" {
// Desugar for-loop as expression codegen
let item: String = stmt["item"]
let list_expr = stmt["list"]
let body = stmt["body"]
let for_expr = { "expr": "For", "item": item, "list": list_expr, "body": body }
return cg_expr(ctx, for_expr)
}
if kind == "Expr" {
let val = stmt["value"]
let val_kind: String = val["expr"]
let ctx = cg_expr(ctx, val)
// Discard result unless it's a control-flow expression
if val_kind == "If" {
return ctx
}
if val_kind == "For" {
return ctx
}
if val_kind == "Match" {
return ctx
}
return emit_pop(ctx)
}
if kind == "TypeDef" {
// compile-time only; no runtime code
return ctx
}
if kind == "EnumDef" {
// compile-time only; no runtime code
return ctx
}
if kind == "Import" {
// handled at a higher level; skip
return ctx
}
ctx
}
fn cg_stmts(ctx: Map<String, Any>, stmts: [Map<String, Any>]) -> Map<String, Any> {
let n: Int = native_list_len(stmts)
let i = 0
while i < n {
let stmt = native_list_get(stmts, i)
let ctx = cg_stmt(ctx, stmt)
let i = i + 1
}
ctx
}
// JSON serialisation
fn instrs_to_json(instrs: [String]) -> String {
let n: Int = native_list_len(instrs)
let out = "["
let i = 0
while i < n {
let instr: String = native_list_get(instrs, i)
if i > 0 {
let out = out + ","
}
let out = out + instr
let i = i + 1
}
out + "]"
}
// Entry point
fn codegen(stmts: [Map<String, Any>], source: String) -> String {
let ctx = ctx_new()
let ctx = cg_stmts(ctx, stmts)
let ctx = emit_halt(ctx)
let instrs = ctx["instrs"]
instrs_to_json(instrs)
}