rename crates/ to engrams/; add el-compiler el package with bootstrap artifact

- crates/ → engrams/ (Rust engrams live here)
- el-compiler/ added: el self-hosting compiler as an el package
  - src/{compiler,lexer,parser,codegen}.el
  - bootstrap/el-compiler.elc (114KB, Rust-compiled seed)
- el.toml Cargo.toml workspace paths updated
- neuron-rs cross-repo path deps fixed (were pointing to products/ instead of foundation/)
This commit is contained in:
Will Anderson
2026-04-29 03:27:32 -05:00
parent 19ed2721ee
commit a42429012e
120 changed files with 3836 additions and 64 deletions
+759
View File
@@ -0,0 +1,759 @@
// parser.el el self-hosting recursive descent parser
//
// Consumes the token list produced by lexer.el and builds a list of AST
// statement maps. Each statement and expression is a Map<String, Any>.
//
// The cursor (integer position into the token list) is threaded through every
// parse function. Functions return { "node": <map>, "pos": <int> }.
//
// Entry point: fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>]
// Token access helpers
fn tok_at(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
native_list_get(tokens, pos)
}
fn tok_kind(tokens: [Map<String, Any>], pos: Int) -> String {
let t = tok_at(tokens, pos)
t["kind"]
}
fn tok_value(tokens: [Map<String, Any>], pos: Int) -> String {
let t = tok_at(tokens, pos)
t["value"]
}
fn expect(tokens: [Map<String, Any>], pos: Int, kind: String) -> Int {
let k = tok_kind(tokens, pos)
if k == kind {
return pos + 1
}
// On mismatch just advance; error recovery is best-effort
pos + 1
}
// Result helpers
fn make_result(node: Map<String, Any>, pos: Int) -> Map<String, Any> {
{ "node": node, "pos": pos }
}
// Type annotation parser
// Skips over a type annotation, returning the new position.
// Types can be: Ident, [Type], Map<K,V>, Type?, Type<Type,...>
fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int {
let k = tok_kind(tokens, pos)
// Array type: [Type]
if k == "LBracket" {
let p = pos + 1
let p = skip_type(tokens, p)
let p = expect(tokens, p, "RBracket")
return p
}
// Named type (possibly generic)
if k == "Ident" {
let p = pos + 1
let k2 = tok_kind(tokens, p)
if k2 == "Lt" {
// Generic params: skip until matching >
let p = p + 1
let depth = 1
let running = true
while running {
let kk = tok_kind(tokens, p)
if kk == "Eof" {
let running = false
} else {
if kk == "Lt" {
let depth = depth + 1
let p = p + 1
} else {
if kk == "Gt" {
let depth = depth - 1
let p = p + 1
if depth <= 0 {
let running = false
}
} else {
let p = p + 1
}
}
}
}
let k3 = tok_kind(tokens, p)
if k3 == "QuestionMark" {
let p = p + 1
}
return p
}
// Optional marker
if k2 == "QuestionMark" {
return p + 1
}
return p
}
pos + 1
}
// Parameter list
// Parses (name: Type, name: Type, ...) returns { "params": [...], "pos": ... }
fn parse_params(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LParen")
let params: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k = tok_kind(tokens, p)
if k == "RParen" {
let running = false
} else {
if k == "Eof" {
let running = false
} else {
// param name
let pname = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "Colon")
let p = skip_type(tokens, p)
let param = { "name": pname }
let params = native_list_append(params, param)
let k2 = tok_kind(tokens, p)
if k2 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RParen")
{ "params": params, "pos": p }
}
// Expression parsing
fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos)
let v = tok_value(tokens, pos)
// Integer literal
if k == "Int" {
return make_result({ "expr": "Int", "value": v }, pos + 1)
}
// Float literal
if k == "Float" {
return make_result({ "expr": "Float", "value": v }, pos + 1)
}
// String literal
if k == "Str" {
return make_result({ "expr": "Str", "value": v }, pos + 1)
}
// Bool literal
if k == "Bool" {
return make_result({ "expr": "Bool", "value": v }, pos + 1)
}
// Identifier
if k == "Ident" {
return make_result({ "expr": "Ident", "name": v }, pos + 1)
}
// Grouped expression
if k == "LParen" {
let r = parse_expr(tokens, pos + 1)
let node = r["node"]
let p = r["pos"]
let p = expect(tokens, p, "RParen")
return make_result(node, p)
}
// Array literal: [e1, e2, ...]
if k == "LBracket" {
let p = pos + 1
let elems: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k2 = tok_kind(tokens, p)
if k2 == "RBracket" {
let running = false
} else {
if k2 == "Eof" {
let running = false
} else {
let r = parse_expr(tokens, p)
let elem = r["node"]
let p = r["pos"]
let elems = native_list_append(elems, elem)
let k3 = tok_kind(tokens, p)
if k3 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBracket")
return make_result({ "expr": "Array", "elems": elems }, p)
}
// Map literal: { "key": val, ... }
if k == "LBrace" {
let p = pos + 1
let pairs: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k2 = tok_kind(tokens, p)
if k2 == "RBrace" {
let running = false
} else {
if k2 == "Eof" {
let running = false
} else {
// key: Str token
let key = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "Colon")
let r = parse_expr(tokens, p)
let val_node = r["node"]
let p = r["pos"]
let pair = { "key": key, "value": val_node }
let pairs = native_list_append(pairs, pair)
let k3 = tok_kind(tokens, p)
if k3 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBrace")
return make_result({ "expr": "Map", "pairs": pairs }, p)
}
// if expression
if k == "If" {
let r = parse_if(tokens, pos)
return r
}
// match expression
if k == "Match" {
let r = parse_match(tokens, pos)
return r
}
// for expression (used as statement)
if k == "For" {
let r = parse_for_expr(tokens, pos)
return r
}
// Unary not
if k == "Not" {
let r = parse_primary(tokens, pos + 1)
let inner = r["node"]
let p = r["pos"]
return make_result({ "expr": "Not", "inner": inner }, p)
}
// Unary minus
if k == "Minus" {
let r = parse_primary(tokens, pos + 1)
let inner = r["node"]
let p = r["pos"]
return make_result({ "expr": "Neg", "inner": inner }, p)
}
// Fallback: skip unknown token
make_result({ "expr": "Nil" }, pos + 1)
}
fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "If")
let r = parse_expr(tokens, p)
let cond = r["node"]
let p = r["pos"]
let r2 = parse_block(tokens, p)
let then_stmts = r2["stmts"]
let p = r2["pos"]
let has_else = false
let else_stmts: [Map<String, Any>] = native_list_empty()
let k2 = tok_kind(tokens, p)
if k2 == "Else" {
let p = p + 1
let k3 = tok_kind(tokens, p)
if k3 == "If" {
// else-if chain: parse as nested if
let r3 = parse_if(tokens, p)
let nested = r3["node"]
let p = r3["pos"]
let else_stmts = native_list_append(else_stmts, { "stmt": "Expr", "value": nested })
let has_else = true
} else {
let r3 = parse_block(tokens, p)
let else_stmts = r3["stmts"]
let p = r3["pos"]
let has_else = true
}
}
make_result({ "expr": "If", "cond": cond, "then": then_stmts, "else": else_stmts, "has_else": has_else }, p)
}
fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "Match")
let r = parse_expr(tokens, p)
let subject = r["node"]
let p = r["pos"]
let p = expect(tokens, p, "LBrace")
let arms: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k = tok_kind(tokens, p)
if k == "RBrace" {
let running = false
} else {
if k == "Eof" {
let running = false
} else {
// parse pattern => body
let r2 = parse_pattern(tokens, p)
let pattern = r2["node"]
let p = r2["pos"]
let p = expect(tokens, p, "FatArrow")
let r3 = parse_expr(tokens, p)
let body = r3["node"]
let p = r3["pos"]
let arm = { "pattern": pattern, "body": body }
let arms = native_list_append(arms, arm)
let k2 = tok_kind(tokens, p)
if k2 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBrace")
make_result({ "expr": "Match", "subject": subject, "arms": arms }, p)
}
fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos)
if k == "Ident" {
let v = tok_value(tokens, pos)
if v == "_" {
return make_result({ "pattern": "Wildcard" }, pos + 1)
}
return make_result({ "pattern": "Binding", "name": v }, pos + 1)
}
if k == "Int" {
return make_result({ "pattern": "LitInt", "value": tok_value(tokens, pos) }, pos + 1)
}
if k == "Str" {
return make_result({ "pattern": "LitStr", "value": tok_value(tokens, pos) }, pos + 1)
}
if k == "Bool" {
return make_result({ "pattern": "LitBool", "value": tok_value(tokens, pos) }, pos + 1)
}
// Wildcard _
make_result({ "pattern": "Wildcard" }, pos + 1)
}
fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "For")
let item_name = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "In")
let r = parse_expr(tokens, p)
let list_expr = r["node"]
let p = r["pos"]
let r2 = parse_block(tokens, p)
let body = r2["stmts"]
let p = r2["pos"]
make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p)
}
fn parse_block(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LBrace")
let stmts: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k = tok_kind(tokens, p)
if k == "RBrace" {
let running = false
} else {
if k == "Eof" {
let running = false
} else {
let r = parse_stmt(tokens, p)
let stmt = r["node"]
let p = r["pos"]
let stmts = native_list_append(stmts, stmt)
}
}
}
let p = expect(tokens, p, "RBrace")
{ "stmts": stmts, "pos": p }
}
// Postfix expressions (calls, field access, index)
fn parse_postfix(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let r = parse_primary(tokens, pos)
let node = r["node"]
let p = r["pos"]
let running = true
while running {
let k = tok_kind(tokens, p)
if k == "LParen" {
// function call
let p = p + 1
let args: [Map<String, Any>] = native_list_empty()
let run2 = true
while run2 {
let k2 = tok_kind(tokens, p)
if k2 == "RParen" {
let run2 = false
} else {
if k2 == "Eof" {
let run2 = false
} else {
let r2 = parse_expr(tokens, p)
let arg = r2["node"]
let p = r2["pos"]
let args = native_list_append(args, arg)
let k3 = tok_kind(tokens, p)
if k3 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RParen")
let node = { "expr": "Call", "func": node, "args": args }
} else {
if k == "Dot" {
let field = tok_value(tokens, p + 1)
let p = p + 2
let node = { "expr": "Field", "object": node, "field": field }
} else {
if k == "LBracket" {
let r2 = parse_expr(tokens, p + 1)
let idx = r2["node"]
let p = r2["pos"]
let p = expect(tokens, p, "RBracket")
let node = { "expr": "Index", "object": node, "index": idx }
} else {
if k == "QuestionMark" {
let p = p + 1
let node = { "expr": "Try", "inner": node }
} else {
let running = false
}
}
}
}
}
make_result(node, p)
}
// Binary expression precedence climbing
fn op_precedence(kind: String) -> Int {
if kind == "Or" { return 1 }
if kind == "And" { return 2 }
if kind == "EqEq" { return 3 }
if kind == "NotEq" { return 3 }
if kind == "Lt" { return 4 }
if kind == "Gt" { return 4 }
if kind == "LtEq" { return 4 }
if kind == "GtEq" { return 4 }
if kind == "Plus" { return 5 }
if kind == "Minus" { return 5 }
if kind == "Star" { return 6 }
if kind == "Slash" { return 6 }
0
}
fn is_binop(kind: String) -> Bool {
if kind == "Or" { return true }
if kind == "And" { return true }
if kind == "EqEq" { return true }
if kind == "NotEq" { return true }
if kind == "Lt" { return true }
if kind == "Gt" { return true }
if kind == "LtEq" { return true }
if kind == "GtEq" { return true }
if kind == "Plus" { return true }
if kind == "Minus" { return true }
if kind == "Star" { return true }
if kind == "Slash" { return true }
false
}
fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<String, Any> {
let r = parse_postfix(tokens, pos)
let left = r["node"]
let p = r["pos"]
let running = true
while running {
let k = tok_kind(tokens, p)
let prec = op_precedence(k)
if is_binop(k) {
if prec >= min_prec {
let op = k
let r2 = parse_binop(tokens, p + 1, prec + 1)
let right = r2["node"]
let p = r2["pos"]
let left = { "expr": "BinOp", "op": op, "left": left, "right": right }
} else {
let running = false
}
} else {
let running = false
}
}
make_result(left, p)
}
fn parse_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
parse_binop(tokens, pos, 1)
}
// Statement parsing
fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos)
// let binding
if k == "Let" {
let p = pos + 1
let name = tok_value(tokens, p)
let p = p + 1
let k2 = tok_kind(tokens, p)
// optional type annotation: name: Type
if k2 == "Colon" {
let p = p + 1
let p = skip_type(tokens, p)
}
let p = expect(tokens, p, "Eq")
let r = parse_expr(tokens, p)
let val = r["node"]
let p = r["pos"]
return make_result({ "stmt": "Let", "name": name, "value": val }, p)
}
// return statement
if k == "Return" {
let p = pos + 1
let k2 = tok_kind(tokens, p)
if k2 == "RBrace" {
return make_result({ "stmt": "Return", "value": { "expr": "Nil" } }, p)
}
if k2 == "Eof" {
return make_result({ "stmt": "Return", "value": { "expr": "Nil" } }, p)
}
let r = parse_expr(tokens, p)
let val = r["node"]
let p = r["pos"]
return make_result({ "stmt": "Return", "value": val }, p)
}
// fn definition
if k == "Fn" {
let p = pos + 1
let name = tok_value(tokens, p)
let p = p + 1
let r = parse_params(tokens, p)
let params = r["params"]
let p = r["pos"]
// return type annotation: -> Type
let k2 = tok_kind(tokens, p)
if k2 == "Arrow" {
let p = p + 1
let p = skip_type(tokens, p)
}
let r2 = parse_block(tokens, p)
let body = r2["stmts"]
let p = r2["pos"]
return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body }, p)
}
// type definition
if k == "Type" {
let p = pos + 1
let name = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "LBrace")
let fields: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k2 = tok_kind(tokens, p)
if k2 == "RBrace" {
let running = false
} else {
if k2 == "Eof" {
let running = false
} else {
let fname = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "Colon")
let p = skip_type(tokens, p)
let fields = native_list_append(fields, { "name": fname })
let k3 = tok_kind(tokens, p)
if k3 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBrace")
return make_result({ "stmt": "TypeDef", "name": name, "fields": fields }, p)
}
// enum definition
if k == "Enum" {
let p = pos + 1
let name = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "LBrace")
let variants: [Map<String, Any>] = native_list_empty()
let running = true
while running {
let k2 = tok_kind(tokens, p)
if k2 == "RBrace" {
let running = false
} else {
if k2 == "Eof" {
let running = false
} else {
let vname = tok_value(tokens, p)
let p = p + 1
let variants = native_list_append(variants, { "name": vname })
let k3 = tok_kind(tokens, p)
if k3 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBrace")
return make_result({ "stmt": "EnumDef", "name": name, "variants": variants }, p)
}
// import statement
if k == "Import" {
let p = pos + 1
let path = tok_value(tokens, p)
let p = p + 1
return make_result({ "stmt": "Import", "path": path }, p)
}
// from ... import { ... }
if k == "From" {
let p = pos + 1
let module_name = tok_value(tokens, p)
let p = p + 1
// skip "import" keyword
let k2 = tok_kind(tokens, p)
if k2 == "Import" {
let p = p + 1
}
// skip { Name, ... }
let k3 = tok_kind(tokens, p)
if k3 == "LBrace" {
let p = p + 1
let running = true
while running {
let k4 = tok_kind(tokens, p)
if k4 == "RBrace" {
let running = false
} else {
if k4 == "Eof" {
let running = false
} else {
let p = p + 1
let k5 = tok_kind(tokens, p)
if k5 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBrace")
}
return make_result({ "stmt": "Import", "path": module_name }, p)
}
// while loop
if k == "While" {
let p = pos + 1
let r = parse_expr(tokens, p)
let cond = r["node"]
let p = r["pos"]
let r2 = parse_block(tokens, p)
let body = r2["stmts"]
let p = r2["pos"]
return make_result({ "stmt": "While", "cond": cond, "body": body }, p)
}
// for loop
if k == "For" {
let p = pos + 1
let item_name = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "In")
let r = parse_expr(tokens, p)
let list_expr = r["node"]
let p = r["pos"]
let r2 = parse_block(tokens, p)
let body = r2["stmts"]
let p = r2["pos"]
return make_result({ "stmt": "For", "item": item_name, "list": list_expr, "body": body }, p)
}
// @decorator skip and parse next stmt
if k == "At" {
let p = pos + 1
// skip decorator name
let p = p + 1
return parse_stmt(tokens, p)
}
// bare expression or if/match statement
let r = parse_expr(tokens, pos)
let val = r["node"]
let p = r["pos"]
make_result({ "stmt": "Expr", "value": val }, p)
}
// Top-level parse
fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>] {
let total: Int = native_list_len(tokens)
let stmts: [Map<String, Any>] = native_list_empty()
let pos: Int = 0
let running = true
while running {
if pos >= total {
let running = false
} else {
let k = tok_kind(tokens, pos)
if k == "Eof" {
let running = false
} else {
let r = parse_stmt(tokens, pos)
let stmt = r["node"]
let new_pos: Int = r["pos"]
let stmts = native_list_append(stmts, stmt)
// Guard against infinite loops if pos didn't advance, force it
if new_pos <= pos {
let pos = pos + 1
} else {
let pos = new_pos
}
}
}
}
stmts
}