feat: port arbor, dharma, forge El source into monorepo

Brings the remaining foundation repos that were not included in the
original monorepo consolidation:

- arbor/vessels/ — 6 vessels (arbor-cli, arbor-core, arbor-diagram,
  arbor-layout, arbor-parse, arbor-render) with manifests + src/main.el
- dharma/ — CGI Provenance Registry package (flat layout, 14 .el files
  across registry/, sandbox/, training/, validation/, tests/)
- forge/ — consciousness channel tool (8 src .el files + new manifest.el)
- elp/src/ — 36 test fixture files not carried over in original merge
  (dedup_*, realizer_*, semantics_*, morph_*, ext_*, one_extern_* helpers)

el-ide, engram, elql are already complete in ide/, engram/, ql/.
This commit is contained in:
Will Anderson
2026-05-05 04:27:34 -05:00
parent bdd7b56703
commit 90ddbdbfc3
78 changed files with 18211 additions and 0 deletions
+19
View File
@@ -0,0 +1,19 @@
// arbor-parse hand-written recursive-descent parser for the .arbor source
// language. Produces an Arbor graph value consumable by arbor-layout and
// arbor-render.
vessel "arbor-parse" {
version "0.1.0"
description "Recursive-descent parser for the .arbor diagram language"
authors ["Neuron Technologies"]
edition "2026"
}
dependencies {
arbor-core "0.1"
}
build {
entry "src/main.el"
output "dist/"
}
+763
View File
@@ -0,0 +1,763 @@
// arbor-parse recursive-descent parser for the .arbor source language.
//
// This vessel inlines a private copy of the small set of arbor-core helpers
// it needs (sanitize_id and constructors). El's import form today is purely
// syntactic concatenation, so each vessel that wants to be its own buildable
// unit carries its own copy of these helpers. They're tiny (well under 100
// lines) and the duplication keeps each vessel hermetic.
//
// Public entry point: fn arbor_parse(source: String) -> Map<String, Any>
//
// Returns either a graph value or a parse-error map. Callers test for the
// "error" field:
// { "error": "..." , "line": Int, "text": "...source line..." } on failure
// { "title", "direction", "nodes", "edges", "groups" } on success
// Sanitisation (copy of arbor-core's sanitize_id)
fn is_alnum_underscore(ch: String) -> Bool {
let code: Int = str_char_code(ch, 0)
if code >= 48 {
if code <= 57 { return true }
}
if code >= 65 {
if code <= 90 { return true }
}
if code >= 97 {
if code <= 122 { return true }
}
if code == 95 { return true }
false
}
fn is_ascii_digit(ch: String) -> Bool {
let code: Int = str_char_code(ch, 0)
if code >= 48 {
if code <= 57 { return true }
}
false
}
fn sanitize_id(s: String) -> String {
let n: Int = str_len(s)
if n == 0 { return "node" }
let out = ""
let prev_underscore = false
let i = 0
while i < n {
let ch: String = str_char_at(s, i)
if is_alnum_underscore(ch) {
let out = out + ch
let prev_underscore = false
} else {
if !prev_underscore {
let out = out + "_"
}
let prev_underscore = true
}
let i = i + 1
}
let m: Int = str_len(out)
let end = m
let stripping = true
while stripping {
if end <= 0 {
let stripping = false
} else {
let last: String = str_char_at(out, end - 1)
if last == "_" {
let end = end - 1
} else {
let stripping = false
}
}
}
let out = str_slice(out, 0, end)
if str_len(out) == 0 { return "node" }
let first: String = str_char_at(out, 0)
if is_ascii_digit(first) {
let out = "n" + out
}
out
}
fn shape_from_token(tok: String) -> String {
let t: String = str_trim(tok)
if t == "rect" { return "rect" }
if t == "rounded" { return "rounded" }
if t == "cylinder" { return "cylinder" }
if t == "diamond" { return "diamond" }
if t == "stadium" { return "stadium" }
if t == "primary" { return "primary" }
""
}
// Line preprocessing
//
// Strip inline `// ...` comments, trim, drop empties. Returns a list of maps
// { "no": Int, "text": String }.
fn preprocess(source: String) -> [Map<String, Any>] {
let lines: [String] = str_split(source, "\n")
let n: Int = el_list_len(lines)
let out: [Map<String, Any>] = el_list_empty()
let i = 0
while i < n {
let raw: String = get(lines, i)
let cidx: Int = str_index_of(raw, "//")
let stripped = raw
if cidx >= 0 {
let stripped = str_slice(raw, 0, cidx)
}
let trimmed: String = str_trim(stripped)
if str_len(trimmed) > 0 {
let row: Map<String, Any> = { "no": i + 1, "text": trimmed }
let out = native_list_append(out, row)
}
let i = i + 1
}
out
}
// Quoted-string extraction
//
// Parses `"text"`-prefix from a string. Returns `{ "ok": Bool, "value": Str,
// "rest": Str }`. The `rest` field carries everything after the closing quote
// (so the caller can continue tokenising).
fn parse_quoted(s: String) -> Map<String, Any> {
let t: String = str_trim(s)
if str_len(t) < 2 {
return { "ok": false, "value": "", "rest": s }
}
let first: String = str_char_at(t, 0)
if first != "\"" {
return { "ok": false, "value": "", "rest": s }
}
let body: String = str_slice(t, 1, str_len(t))
let close: Int = str_index_of(body, "\"")
if close < 0 {
return { "ok": false, "value": "", "rest": s }
}
let inner: String = str_slice(body, 0, close)
let rest: String = str_slice(body, close + 1, str_len(body))
{ "ok": true, "value": inner, "rest": rest }
}
// Identifier prefix split
//
// `split_identifier("foo bar")` { "id": "foo", "rest": " bar" }.
// `split_identifier("a-b")` { "id": "a", "rest": "-b" }.
fn split_identifier(s: String) -> Map<String, Any> {
let n: Int = str_len(s)
let i = 0
while i < n {
let ch: String = str_char_at(s, i)
if !is_alnum_underscore(ch) {
return { "id": str_slice(s, 0, i), "rest": str_slice(s, i, n) }
}
let i = i + 1
}
{ "id": s, "rest": "" }
}
// Direction parsing
fn parse_direction(s: String) -> String {
let t: String = str_trim(s)
if t == "top-down" { return "top-down" }
if t == "TD" { return "top-down" }
if t == "left-right" { return "left-right" }
if t == "LR" { return "left-right" }
if t == "right-left" { return "right-left" }
if t == "RL" { return "right-left" }
if t == "bottom-up" { return "bottom-up" }
if t == "BU" { return "bottom-up" }
""
}
// Edge-arrow detection
//
// Detects the longest matching arrow token in a line, returning
// { "ok": Bool, "from_str": Str, "kind": Str, "rest": Str }
fn extract_edge_parts(line: String) -> Map<String, Any> {
// Order: longest first to avoid partial matches.
let f1: Int = str_index_of(line, "-/->")
if f1 >= 0 {
return { "ok": true,
"from_str": str_slice(line, 0, f1),
"kind": "forbidden",
"rest": str_slice(line, f1 + 4, str_len(line)) }
}
let f2: Int = str_index_of(line, "<->")
if f2 >= 0 {
return { "ok": true,
"from_str": str_slice(line, 0, f2),
"kind": "bidirectional",
"rest": str_slice(line, f2 + 3, str_len(line)) }
}
let f3: Int = str_index_of(line, "-->")
if f3 >= 0 {
return { "ok": true,
"from_str": str_slice(line, 0, f3),
"kind": "dashed",
"rest": str_slice(line, f3 + 3, str_len(line)) }
}
let f4: Int = str_index_of(line, "->")
if f4 >= 0 {
return { "ok": true,
"from_str": str_slice(line, 0, f4),
"kind": "solid",
"rest": str_slice(line, f4 + 2, str_len(line)) }
}
{ "ok": false, "from_str": "", "kind": "", "rest": "" }
}
fn is_edge_line(line: String) -> Bool {
if str_contains(line, "->") { return true }
if str_contains(line, "<->") { return true }
false
}
// Error helpers
fn make_error(line_no: Int, line_text: String, message: String) -> Map<String, Any> {
{ "error": message, "line": line_no, "text": line_text }
}
// Parse driver
//
// State is held in process-local k/v rather than threaded through every
// function. Specifically:
// "title", "direction" graph header
// "nodes_json", "edges_json", "groups_json" accumulators (string lists)
// "group_stack_depth" "0".."N" open groups
// "group_stack_<i>_id" / "_label" / "_line" frame data
// "group_stack_<i>_node_ids" JSON array of ids inside frame
// "error" non-empty if parse failed
// "error_line", "error_text" context
fn st_set_int(key: String, v: Int) -> Int { state_set(key, int_to_str(v)); 0 }
fn st_get_int(key: String) -> Int {
let s: String = state_get(key)
if str_eq(s, "") { return 0 }
str_to_int(s)
}
// Encode/decode small string lists via "" delimiter (unit separator).
fn list_encode(xs: [String]) -> String {
let n: Int = el_list_len(xs)
let out = ""
let i = 0
while i < n {
if i > 0 { let out = out + "" }
let out = out + get(xs, i)
let i = i + 1
}
out
}
fn list_decode(s: String) -> [String] {
if str_eq(s, "") { return el_list_empty() }
str_split(s, "")
}
fn current_group_index() -> Int {
st_get_int("group_stack_depth") - 1
}
fn group_frame_key(idx: Int, suffix: String) -> String {
"gs_" + int_to_str(idx) + "_" + suffix
}
fn open_group(id: String, label: String, line_no: Int) -> Int {
let depth: Int = st_get_int("group_stack_depth")
state_set(group_frame_key(depth, "id"), id)
state_set(group_frame_key(depth, "label"), label)
state_set(group_frame_key(depth, "line"), int_to_str(line_no))
state_set(group_frame_key(depth, "ids"), "")
st_set_int("group_stack_depth", depth + 1)
0
}
fn close_group_frame() -> Map<String, Any> {
let depth: Int = st_get_int("group_stack_depth")
if depth <= 0 {
return { "ok": false, "id": "", "label": "", "ids": "" }
}
let idx: Int = depth - 1
let id: String = state_get(group_frame_key(idx, "id"))
let label: String = state_get(group_frame_key(idx, "label"))
let ids: String = state_get(group_frame_key(idx, "ids"))
state_del(group_frame_key(idx, "id"))
state_del(group_frame_key(idx, "label"))
state_del(group_frame_key(idx, "line"))
state_del(group_frame_key(idx, "ids"))
st_set_int("group_stack_depth", idx)
{ "ok": true, "id": id, "label": label, "ids": ids }
}
fn register_node_in_group(node_id: String) -> Int {
let depth: Int = st_get_int("group_stack_depth")
if depth <= 0 { return 0 }
let idx: Int = depth - 1
let key: String = group_frame_key(idx, "ids")
let prev: String = state_get(key)
if str_eq(prev, "") {
state_set(key, node_id)
} else {
state_set(key, prev + "" + node_id)
}
0
}
// Accumulator JSON-ish encoding for nodes/edges/groups.
// We render each entry as a small string and stash in state under a counter.
fn store_node(id: String, label: String, shape: String) -> Int {
let n: Int = st_get_int("node_count")
state_set("node_id_" + int_to_str(n), id)
state_set("node_label_" + int_to_str(n), label)
state_set("node_shape_" + int_to_str(n), shape)
st_set_int("node_count", n + 1)
0
}
fn store_edge(src: String, dst: String, label: String, kind: String) -> Int {
let n: Int = st_get_int("edge_count")
state_set("edge_from_" + int_to_str(n), src)
state_set("edge_to_" + int_to_str(n), dst)
state_set("edge_label_" + int_to_str(n), label)
state_set("edge_kind_" + int_to_str(n), kind)
st_set_int("edge_count", n + 1)
0
}
fn store_group(id: String, label: String, ids: String) -> Int {
let n: Int = st_get_int("group_count")
state_set("group_id_" + int_to_str(n), id)
state_set("group_label_" + int_to_str(n), label)
state_set("group_ids_" + int_to_str(n), ids)
st_set_int("group_count", n + 1)
0
}
fn set_error(msg: String, line_no: Int, line_text: String) -> Int {
state_set("parse_error", msg)
st_set_int("parse_error_line", line_no)
state_set("parse_error_text", line_text)
0
}
fn has_error() -> Bool {
let m: String = state_get("parse_error")
if str_eq(m, "") { return false }
true
}
// Reset state at the start of each parse pass.
fn reset_state() -> Int {
state_set("graph_title", "")
state_set("graph_direction", "top-down")
st_set_int("node_count", 0)
st_set_int("edge_count", 0)
st_set_int("group_count", 0)
st_set_int("group_stack_depth", 0)
state_set("parse_error", "")
st_set_int("parse_error_line", 0)
state_set("parse_error_text", "")
0
}
// Statement-level parsing
fn parse_node_stmt(line_no: Int, line: String) -> Int {
let id_split: Map<String, Any> = split_identifier(line)
let raw_id: String = id_split["id"]
if str_eq(raw_id, "") {
set_error("expected node id, edge, or keyword", line_no, line)
return 0
}
let id: String = sanitize_id(raw_id)
let rest: String = str_trim(id_split["rest"])
// Optional shape: [token]
let shape = "rect"
let after_shape = rest
if str_len(rest) > 0 {
let lead: String = str_char_at(rest, 0)
if lead == "[" {
let close: Int = str_index_of(rest, "]")
if close < 0 {
set_error("unclosed `[` in shape token", line_no, line)
return 0
}
let token: String = str_slice(rest, 1, close)
let parsed_shape: String = shape_from_token(token)
if str_eq(parsed_shape, "") {
set_error("unknown shape `" + token + "`", line_no, line)
return 0
}
let shape = parsed_shape
let after_shape = str_trim(str_slice(rest, close + 1, str_len(rest)))
}
}
// Optional quoted label.
let quoted: Map<String, Any> = parse_quoted(after_shape)
let label = raw_id
let ok: Bool = quoted["ok"]
if ok {
let label = quoted["value"]
}
store_node(id, label, shape)
register_node_in_group(id)
1
}
fn parse_edge_stmt(line_no: Int, line: String) -> Int {
let parts: Map<String, Any> = extract_edge_parts(line)
let ok: Bool = parts["ok"]
if !ok {
set_error("malformed edge — expected `->` `-->` `<->` or `-/->`", line_no, line)
return 0
}
let from_str: String = parts["from_str"]
let rest_str: String = parts["rest"]
let kind: String = parts["kind"]
let src: String = sanitize_id(str_trim(from_str))
let rest_t: String = str_trim(rest_str)
let id_split: Map<String, Any> = split_identifier(rest_t)
let to_raw: String = id_split["id"]
if str_eq(to_raw, "") {
set_error("edge missing target node id", line_no, line)
return 0
}
let dst: String = sanitize_id(to_raw)
let label_rest: String = str_trim(id_split["rest"])
let quoted: Map<String, Any> = parse_quoted(label_rest)
let label = ""
let qok: Bool = quoted["ok"]
if qok {
let label = quoted["value"]
}
store_edge(src, dst, label, kind)
1
}
fn parse_group_open(line_no: Int, line: String, rest: String) -> Int {
// Strip trailing `{`.
let trimmed: String = str_trim(rest)
let n: Int = str_len(trimmed)
let body = trimmed
if n > 0 {
let last: String = str_char_at(trimmed, n - 1)
if last == "{" {
let body = str_trim(str_slice(trimmed, 0, n - 1))
}
}
let id_split: Map<String, Any> = split_identifier(body)
let raw_id: String = id_split["id"]
if str_eq(raw_id, "") {
set_error("group declaration missing id", line_no, line)
return 0
}
let label_rest: String = str_trim(id_split["rest"])
let quoted: Map<String, Any> = parse_quoted(label_rest)
let label = raw_id
let qok: Bool = quoted["ok"]
if qok {
let label = quoted["value"]
}
open_group(raw_id, label, line_no)
1
}
fn parse_close_brace(line_no: Int) -> Int {
let frame: Map<String, Any> = close_group_frame()
let frame_ok: Bool = frame["ok"]
if !frame_ok {
set_error("unexpected `}` — no open group", line_no, "}")
return 0
}
store_group(frame["id"], frame["label"], frame["ids"])
1
}
fn parse_line_dispatch(line_no: Int, line: String) -> Int {
if line == "}" { return parse_close_brace(line_no) }
if str_starts_with(line, "title:") {
let after: String = str_trim(str_slice(line, 6, str_len(line)))
let q: Map<String, Any> = parse_quoted(after)
let qok: Bool = q["ok"]
if !qok {
set_error("expected quoted string after `title:`", line_no, line)
return 0
}
state_set("graph_title", q["value"])
return 1
}
if str_starts_with(line, "direction:") {
let after: String = str_trim(str_slice(line, 10, str_len(line)))
let dir: String = parse_direction(after)
if str_eq(dir, "") {
set_error("unknown direction — expected top-down, left-right, right-left, or bottom-up",
line_no, line)
return 0
}
state_set("graph_direction", dir)
return 1
}
if str_starts_with(line, "group ") {
let after: String = str_slice(line, 6, str_len(line))
return parse_group_open(line_no, line, after)
}
if is_edge_line(line) {
return parse_edge_stmt(line_no, line)
}
parse_node_stmt(line_no, line)
}
// Materialise accumulators into the final graph map
fn build_graph_value() -> Map<String, Any> {
let n_nodes: Int = st_get_int("node_count")
let nodes: [Map<String, Any>] = el_list_empty()
let i = 0
while i < n_nodes {
let s: String = int_to_str(i)
let node: Map<String, Any> = {
"id": state_get("node_id_" + s),
"label": state_get("node_label_" + s),
"shape": state_get("node_shape_" + s)
}
let nodes = native_list_append(nodes, node)
let i = i + 1
}
let n_edges: Int = st_get_int("edge_count")
let edges: [Map<String, Any>] = el_list_empty()
let i = 0
while i < n_edges {
let s: String = int_to_str(i)
let edge: Map<String, Any> = {
"from": state_get("edge_from_" + s),
"to": state_get("edge_to_" + s),
"label": state_get("edge_label_" + s),
"kind": state_get("edge_kind_" + s)
}
let edges = native_list_append(edges, edge)
let i = i + 1
}
let n_groups: Int = st_get_int("group_count")
let groups: [Map<String, Any>] = el_list_empty()
let i = 0
while i < n_groups {
let s: String = int_to_str(i)
let raw_ids: String = state_get("group_ids_" + s)
let id_list: [String] = list_decode(raw_ids)
let group: Map<String, Any> = {
"id": state_get("group_id_" + s),
"label": state_get("group_label_" + s),
"node_ids": id_list,
"direction": ""
}
let groups = native_list_append(groups, group)
let i = i + 1
}
{
"title": state_get("graph_title"),
"direction": state_get("graph_direction"),
"nodes": nodes,
"edges": edges,
"groups": groups
}
}
// Public entry point
fn arbor_parse(source: String) -> Map<String, Any> {
reset_state()
let lines: [Map<String, Any>] = preprocess(source)
let n: Int = el_list_len(lines)
let i = 0
let abort = false
while i < n {
if abort {
// skip error already recorded
} else {
let row: Map<String, Any> = get(lines, i)
let line_no: Int = row["no"]
let text: String = row["text"]
parse_line_dispatch(line_no, text)
if has_error() {
let abort = true
}
}
let i = i + 1
}
if !has_error() {
let depth: Int = st_get_int("group_stack_depth")
if depth > 0 {
let idx: Int = depth - 1
let id: String = state_get(group_frame_key(idx, "id"))
let line_no: Int = st_get_int(group_frame_key(idx, "line"))
set_error("unclosed group '" + id + "' — missing closing `}`",
line_no, "group " + id)
}
}
if has_error() {
return {
"error": state_get("parse_error"),
"line": st_get_int("parse_error_line"),
"text": state_get("parse_error_text")
}
}
build_graph_value()
}
// Smoke test
fn fail_msg(label: String, got: String, want: String) -> Int {
println("FAIL " + label + " got=[" + got + "] want=[" + want + "]")
state_set("smoke_failures", "1")
0
}
fn check_eq(label: String, got: String, want: String) -> Int {
if got == want {
println("ok " + label)
return 1
}
fail_msg(label, got, want)
}
// Helper: a graph map is in the error state iff it has a non-empty "error".
fn parse_failed(g: Map<String, Any>) -> Bool {
let m: String = g["error"]
if str_eq(m, "") { return false }
// map_get returns NULL for missing keys; str_eq treats two NULLs as equal
// and NULL vs "" as not equal guard explicitly.
if str_len(m) == 0 { return false }
true
}
let src1 = "title: \"Test\"\ndirection: left-right\n\napi [rounded] \"REST API\"\ndb [cylinder] \"Postgres\"\n\napi -> db \"reads\""
let g1: Map<String, Any> = arbor_parse(src1)
if parse_failed(g1) {
println("FAIL parse 1: " + g1["error"])
state_set("smoke_failures", "1")
}
check_eq("title parsed", g1["title"], "Test")
check_eq("direction parsed", g1["direction"], "left-right")
let nodes1: [Map<String, Any>] = g1["nodes"]
let nn1: Int = el_list_len(nodes1)
check_eq("two nodes", int_to_str(nn1), "2")
let edges1: [Map<String, Any>] = g1["edges"]
let ne1: Int = el_list_len(edges1)
check_eq("one edge", int_to_str(ne1), "1")
let e0: Map<String, Any> = get(edges1, 0)
check_eq("edge from", e0["from"], "api")
check_eq("edge to", e0["to"], "db")
check_eq("edge label", e0["label"], "reads")
check_eq("edge kind", e0["kind"], "solid")
let n0: Map<String, Any> = get(nodes1, 0)
check_eq("node 0 shape", n0["shape"], "rounded")
check_eq("node 0 label", n0["label"], "REST API")
// Test edge varieties
let src2 = "a \"A\"\nb \"B\"\na -> b\na --> b\na -/-> b\na <-> b"
let g2: Map<String, Any> = arbor_parse(src2)
let edges2: [Map<String, Any>] = g2["edges"]
check_eq("4 edges parsed", int_to_str(el_list_len(edges2)), "4")
let kinds = ""
let i = 0
while i < el_list_len(edges2) {
let e: Map<String, Any> = get(edges2, i)
let k: String = e["kind"]
let kinds = kinds + k + ","
let i = i + 1
}
check_eq("edge kinds", kinds, "solid,dashed,forbidden,bidirectional,")
// Groups
let src3 = "group core \"Application Core\" {\n api [rounded] \"REST API\"\n svc \"Business Logic\"\n}\nstandalone \"Out\""
let g3: Map<String, Any> = arbor_parse(src3)
let groups3: [Map<String, Any>] = g3["groups"]
check_eq("one group", int_to_str(el_list_len(groups3)), "1")
let grp0: Map<String, Any> = get(groups3, 0)
check_eq("group label", grp0["label"], "Application Core")
let gnids: [String] = grp0["node_ids"]
check_eq("group has 2 members", int_to_str(el_list_len(gnids)), "2")
let nodes3: [Map<String, Any>] = g3["nodes"]
check_eq("3 total nodes (incl standalone)",
int_to_str(el_list_len(nodes3)), "3")
// Error: unknown shape
let src4 = "node [hexagon] \"X\""
let g4: Map<String, Any> = arbor_parse(src4)
let err4: String = g4["error"]
if str_eq(err4, "") {
println("FAIL expected error for unknown shape")
state_set("smoke_failures", "1")
} else {
if str_contains(err4, "hexagon") {
println("ok error mentions hexagon: " + err4)
} else {
println("FAIL error wording: " + err4)
state_set("smoke_failures", "1")
}
}
// Error: unclosed group
let src5 = "group g \"G\" {\n a \"A\"\n"
let g5: Map<String, Any> = arbor_parse(src5)
let err5: String = g5["error"]
if str_eq(err5, "") {
println("FAIL expected unclosed-group error")
state_set("smoke_failures", "1")
} else {
if str_contains(err5, "unclosed") {
println("ok unclosed group detected")
} else {
println("FAIL unclosed error wording: " + err5)
state_set("smoke_failures", "1")
}
}
// Comments and inline comments
let src6 = "// header\na \"A\" // trailing\nb \"B\""
let g6: Map<String, Any> = arbor_parse(src6)
check_eq("comments stripped", int_to_str(el_list_len(g6["nodes"])), "2")
// Empty input
let g7: Map<String, Any> = arbor_parse("")
check_eq("empty graph nodes", int_to_str(el_list_len(g7["nodes"])), "0")
check_eq("empty graph default direction", g7["direction"], "top-down")
println("")
let f: String = state_get("smoke_failures")
if str_eq(f, "1") {
println("arbor-parse: FAILED")
exit_program(1)
} else {
println("arbor-parse: ok")
}