Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 708722b7ff |
@@ -0,0 +1,269 @@
|
||||
// swarm.el — Native interruptibility for Neuron-dispatched agents.
|
||||
//
|
||||
// CANON: Neuron memory 1ca4d3e9 (native-interruptibility). A dispatched worker
|
||||
// must be re-steerable OR killable the INSTANT a correction / new signal arrives,
|
||||
// MID-TASK — like a human who stops the moment they're told "that's wrong", not
|
||||
// one who finishes the wrong workflow first. Claude's sub-agents cannot do this:
|
||||
// reset off a wrong path, they finish the current workflow before absorbing the
|
||||
// correction, wasting work on a KNOWN-WRONG thing. Neuron's must never.
|
||||
//
|
||||
// MECHANISM (Will's steer, 2026-08-15 — "not a hard problem, don't over-engineer"):
|
||||
// CancellationToken + always-on input.
|
||||
// (1) INPUT ALWAYS ON — every worker holds a CONTROL CHANNEL (the token). The
|
||||
// coordinator can push a signal at ANY time; the channel is NEVER gated by
|
||||
// whether the worker is busy. Like a person who keeps hearing while talking.
|
||||
// (2) COOPERATIVE CANCELLATION — the worker CHECKS the token at every step
|
||||
// boundary (a non-blocking poll, __channel_try_recv). On a signal it STOPS,
|
||||
// ABSORBS the correction (re-plans) or TERMINATES cleanly — with ZERO wasted
|
||||
// continuation of known-wrong work.
|
||||
//
|
||||
// The SAME mechanism serves the conversational speech loop: a mic barge-in simply
|
||||
// invokes the token on the running render (see peripheral loop 66155ba9).
|
||||
//
|
||||
// SAFETY (Rule-4 single-writer / bounded purview): a worker holds its task as a
|
||||
// PURVIEW (a meaning-plan). It writes ONLY to its own out-channel and its own
|
||||
// local purview — it holds NO global write-lock. So it can be interrupted or
|
||||
// killed mid-step with NO half-committed global state; the coordinator just
|
||||
// signals stop. Bounded purviews are what make live interruption clean.
|
||||
//
|
||||
// Built on the runtime concurrency primitives:
|
||||
// __channel_new / __channel_send / __channel_recv / __channel_try_recv (channels)
|
||||
// __thread_create / __thread_join (workers)
|
||||
//
|
||||
// Contrast: the pre-existing channel.el `_channel_worker` loop spawns+joins the
|
||||
// WHOLE task with no signal check — the exact broken pattern. This module checks
|
||||
// the token BETWEEN every bounded step, so the interrupt latency is one step, not
|
||||
// one whole workflow.
|
||||
|
||||
// ── low-level worker/thread wrappers (call seed prims directly; no collision) ──
|
||||
|
||||
fn _swarm_spawn(fn_name: String, arg: String) -> Int {
|
||||
return __thread_create(fn_name, arg)
|
||||
}
|
||||
|
||||
fn _swarm_join(tid: Int) -> String {
|
||||
return __thread_join(tid)
|
||||
}
|
||||
|
||||
// ── Cancellation token — the always-on control channel ─────────────────────────
|
||||
|
||||
// token_new — create a cancellation/pause/redirect token for one worker.
|
||||
// Unbounded so the coordinator NEVER blocks when it signals (always-on input).
|
||||
fn token_new() -> Int {
|
||||
return __channel_new(0)
|
||||
}
|
||||
|
||||
// ── Coordinator ops — INVOKE the token (interrupt / re-steer / kill) ───────────
|
||||
|
||||
// token_signal — send a raw signal JSON. Low-level; prefer the named helpers.
|
||||
fn token_signal(tok: Int, sig_json: String) {
|
||||
__channel_send(tok, sig_json)
|
||||
}
|
||||
|
||||
// token_interrupt — ask the worker to stop at the next step boundary and hold.
|
||||
fn token_interrupt(tok: Int) {
|
||||
__channel_send(tok, "{\"sig\":\"PAUSE\"}")
|
||||
}
|
||||
|
||||
// token_pause — alias for interrupt: stop stepping, hold state, wait for RESUME.
|
||||
fn token_pause(tok: Int) {
|
||||
__channel_send(tok, "{\"sig\":\"PAUSE\"}")
|
||||
}
|
||||
|
||||
// token_resume — resume a paused worker on its held plan.
|
||||
fn token_resume(tok: Int) {
|
||||
__channel_send(tok, "{\"sig\":\"RESUME\"}")
|
||||
}
|
||||
|
||||
// token_kill — terminate the worker cleanly at the next step boundary.
|
||||
fn token_kill(tok: Int) {
|
||||
__channel_send(tok, "{\"sig\":\"KILL\"}")
|
||||
}
|
||||
|
||||
// token_redirect — re-steer the worker onto a NEW plan MID-TASK. Progress already
|
||||
// made (completed steps, emitted outputs) is PRESERVED; only the remaining plan is
|
||||
// replaced. new_steps is a JSON array string, e.g. "[\"step-a\",\"step-b\"]".
|
||||
fn token_redirect(tok: Int, new_goal: String, new_steps: String) {
|
||||
let msg: String = "{\"sig\":\"REDIRECT\",\"goal\":\"" +
|
||||
json_escape_string(new_goal) + "\",\"steps\":" + new_steps + "}"
|
||||
__channel_send(tok, msg)
|
||||
}
|
||||
|
||||
// ── Worker ops — CHECK the token (cooperative cancellation point) ──────────────
|
||||
|
||||
// token_check — non-blocking poll of the token. Returns the pending signal JSON,
|
||||
// or "" when there is no signal. Called at every step boundary. This is the
|
||||
// always-on check: it never blocks the worker, so work proceeds at full speed
|
||||
// until — and only until — a signal actually arrives.
|
||||
fn token_check(tok: Int) -> String {
|
||||
return __channel_try_recv(tok)
|
||||
}
|
||||
|
||||
// token_wait — BLOCK until the next signal. Used only while PAUSED (the worker is
|
||||
// idle, so blocking is correct — it is not spinning).
|
||||
fn token_wait(tok: Int) -> String {
|
||||
return __channel_recv(tok)
|
||||
}
|
||||
|
||||
// ── Purview — the worker's meaning-plan (held task, resumable) ─────────────────
|
||||
|
||||
// purview_new — build a bounded meaning-plan the worker carries.
|
||||
// id — purview id
|
||||
// goal — what the worker is trying to achieve (steerable)
|
||||
// steps — JSON array string of step descriptors (the remaining plan)
|
||||
// A redirect UPDATES this plan; it is not lost.
|
||||
fn purview_new(id: String, goal: String, steps: String) -> String {
|
||||
return "{\"id\":\"" + json_escape_string(id) +
|
||||
"\",\"goal\":\"" + json_escape_string(goal) +
|
||||
"\",\"steps\":" + steps +
|
||||
",\"idx\":0,\"completed\":0,\"status\":\"ready\"}"
|
||||
}
|
||||
|
||||
// ── The interruptible worker loop ──────────────────────────────────────────────
|
||||
|
||||
// swarm_worker_run — the generic natively-interruptible worker.
|
||||
//
|
||||
// arg is a JSON object carrying:
|
||||
// "tok" — the cancellation token (control channel handle)
|
||||
// "out" — the worker's own output channel handle (its ONLY write surface)
|
||||
// "step_fn" — name of an El fn (String)->String that performs ONE bounded step
|
||||
// "purview" — the meaning-plan (from purview_new)
|
||||
//
|
||||
// The loop: at EVERY iteration it first polls the token (always-on check). Only
|
||||
// then does it execute ONE bounded step. So a KILL/REDIRECT/PAUSE is absorbed
|
||||
// within a single step — never after finishing the whole (possibly wrong) plan.
|
||||
//
|
||||
// step_fn is invoked as a child thread joined immediately: exactly ONE step of
|
||||
// work is in flight, so the interrupt latency is bounded by a single step.
|
||||
//
|
||||
// Returns the final purview JSON (status: complete | killed | redirected).
|
||||
fn swarm_worker_run(arg: String) -> String {
|
||||
let tok: Int = str_to_int(json_get(arg, "tok"))
|
||||
let out_ch: Int = str_to_int(json_get(arg, "out"))
|
||||
let step_fn: String = json_get(arg, "step_fn")
|
||||
let purview: String = json_get_raw(arg, "purview")
|
||||
|
||||
let pid: String = json_get(purview, "id")
|
||||
let goal: String = json_get(purview, "goal")
|
||||
let steps: String = json_get_raw(purview, "steps")
|
||||
let n: Int = json_array_len(steps)
|
||||
let idx: Int = 0
|
||||
let completed: Int = 0
|
||||
let status: String = "running"
|
||||
let stop: Int = 0
|
||||
|
||||
while stop == 0 {
|
||||
// ── ALWAYS-ON INTERRUPT CHECK (the cooperative cancellation point) ──
|
||||
let sig: String = token_check(tok)
|
||||
if str_eq(sig, "") {
|
||||
// no signal — proceed with one bounded step (or finish)
|
||||
if idx >= n {
|
||||
let status = "complete"
|
||||
let stop = 1
|
||||
} else {
|
||||
let step: String = json_array_get(steps, idx)
|
||||
let step_arg: String = "{\"goal\":\"" + json_escape_string(goal) +
|
||||
"\",\"idx\":" + int_to_str(idx) +
|
||||
",\"step\":" + step + "}"
|
||||
let tid: Int = _swarm_spawn(step_fn, step_arg)
|
||||
let result: String = _swarm_join(tid)
|
||||
__channel_send(out_ch, result)
|
||||
let idx = idx + 1
|
||||
let completed = completed + 1
|
||||
}
|
||||
} else {
|
||||
// a signal arrived — ABSORB it immediately, before any more work
|
||||
let kind: String = json_get(sig, "sig")
|
||||
if str_eq(kind, "KILL") {
|
||||
// terminate cleanly — commit nothing further. Bounded purview =
|
||||
// no half-committed global state to unwind.
|
||||
__channel_send(out_ch, "[KILL absorbed @step " + int_to_str(idx) +
|
||||
" — stopped, no wasted continuation]")
|
||||
let status = "killed"
|
||||
let stop = 1
|
||||
} else {
|
||||
if str_eq(kind, "REDIRECT") {
|
||||
// ABSORB the correction: re-plan onto the new goal/steps.
|
||||
// Completed steps + emitted outputs are PRESERVED; only the
|
||||
// remaining plan is replaced.
|
||||
let kept: Int = completed
|
||||
let goal = json_get(sig, "goal")
|
||||
let steps = json_get_raw(sig, "steps")
|
||||
let n = json_array_len(steps)
|
||||
let idx = 0
|
||||
let status = "redirected"
|
||||
__channel_send(out_ch, "[REDIRECT absorbed @step " +
|
||||
int_to_str(kept) + " — kept " +
|
||||
int_to_str(kept) +
|
||||
" done, re-planned to goal=" + goal + "]")
|
||||
} else {
|
||||
if str_eq(kind, "PAUSE") {
|
||||
// hold state; idle-wait for the next signal
|
||||
__channel_send(out_ch, "[PAUSE absorbed @step " +
|
||||
int_to_str(idx) + " — holding plan]")
|
||||
let status = "paused"
|
||||
let resumed: Int = 0
|
||||
while resumed == 0 {
|
||||
let s2: String = token_wait(tok)
|
||||
let k2: String = json_get(s2, "sig")
|
||||
if str_eq(k2, "RESUME") {
|
||||
__channel_send(out_ch, "[RESUME @step " +
|
||||
int_to_str(idx) + "]")
|
||||
let status = "running"
|
||||
let resumed = 1
|
||||
} else {
|
||||
if str_eq(k2, "KILL") {
|
||||
__channel_send(out_ch, "[KILL absorbed while paused]")
|
||||
let status = "killed"
|
||||
let stop = 1
|
||||
let resumed = 1
|
||||
} else {
|
||||
if str_eq(k2, "REDIRECT") {
|
||||
let goal = json_get(s2, "goal")
|
||||
let steps = json_get_raw(s2, "steps")
|
||||
let n = json_array_len(steps)
|
||||
let idx = 0
|
||||
__channel_send(out_ch, "[REDIRECT absorbed while paused — goal=" + goal + "]")
|
||||
let status = "running"
|
||||
let resumed = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// final purview snapshot
|
||||
let final: String = "{\"id\":\"" + pid +
|
||||
"\",\"goal\":\"" + json_escape_string(goal) +
|
||||
"\",\"idx\":" + int_to_str(idx) +
|
||||
",\"completed\":" + int_to_str(completed) +
|
||||
",\"planned\":" + int_to_str(n) +
|
||||
",\"status\":\"" + status + "\"}"
|
||||
__channel_send(out_ch, "[DONE status=" + status +
|
||||
" completed=" + int_to_str(completed) + "]")
|
||||
return final
|
||||
}
|
||||
|
||||
// ── Coordinator convenience — dispatch an interruptible worker ─────────────────
|
||||
|
||||
// swarm_dispatch — spawn a natively-interruptible worker on a purview.
|
||||
// step_fn — name of the per-step executor (String)->String
|
||||
// purview — the meaning-plan (from purview_new)
|
||||
// Returns a handle JSON: {"tok":T,"out":O,"tid":D} — the coordinator holds `tok`
|
||||
// to interrupt/redirect/kill live, and drains `out` for results.
|
||||
fn swarm_dispatch(step_fn: String, purview: String) -> String {
|
||||
let tok: Int = token_new()
|
||||
let out_ch: Int = __channel_new(0)
|
||||
let arg: String = "{\"tok\":" + int_to_str(tok) +
|
||||
",\"out\":" + int_to_str(out_ch) +
|
||||
",\"step_fn\":\"" + step_fn +
|
||||
"\",\"purview\":" + purview + "}"
|
||||
let tid: Int = _swarm_spawn("swarm_worker_run", arg)
|
||||
return "{\"tok\":" + int_to_str(tok) +
|
||||
",\"out\":" + int_to_str(out_ch) +
|
||||
",\"tid\":" + int_to_str(tid) + "}"
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// proof.el — Proof of native interruptibility for Neuron-dispatched agents.
|
||||
//
|
||||
// Concatenated after runtime/swarm.el (see run.sh). Demonstrates, with raw
|
||||
// before/after counts, that a dispatched worker interrupted MID-TASK stops
|
||||
// instantly (not after finishing the wrong workflow), absorbs a redirect
|
||||
// (re-plans, keeping progress), or terminates cleanly.
|
||||
//
|
||||
// Three scenarios on the SAME 12-step plan, SAME per-step cost, SAME signal
|
||||
// timing (sent ~50ms in ≈ after step 3):
|
||||
// A. BASELINE — the broken Claude-style worker: no mid-loop signal check.
|
||||
// A kill sent at step ~3 is ignored until the whole plan finishes → all 12
|
||||
// steps run = wasted work on a known-wrong task.
|
||||
// B. INTERRUPTIBLE KILL — swarm_worker_run: the kill is absorbed within one
|
||||
// step → stops at ~3, status=killed, ~9 steps of waste AVOIDED.
|
||||
// C. INTERRUPTIBLE REDIRECT — mirrors the live incident (build-python-synth →
|
||||
// native-fetch-render): the correction is absorbed mid-task; the 3 done
|
||||
// steps are kept; the worker re-plans onto the new goal and finishes it.
|
||||
|
||||
// ── per-step work — one bounded unit (~15ms) ──────────────────────────────────
|
||||
fn demo_step(arg: String) -> String {
|
||||
let goal: String = json_get(arg, "goal")
|
||||
let idx: String = json_get(arg, "idx")
|
||||
let step: String = json_get(arg, "step")
|
||||
sleep_ms(15)
|
||||
return "did[" + goal + "] step#" + idx + " (" + step + ")"
|
||||
}
|
||||
|
||||
// ── BASELINE: the broken, non-interruptible worker (Claude-style) ─────────────
|
||||
// Same shape as swarm_worker_run BUT it never polls the token during the loop.
|
||||
// It "finishes the workflow" and only notices the signal at the very end — the
|
||||
// exact pattern Will called out. Reports how many steps it wasted post-signal.
|
||||
fn broken_worker_run(arg: String) -> String {
|
||||
let tok: Int = str_to_int(json_get(arg, "tok"))
|
||||
let out_ch: Int = str_to_int(json_get(arg, "out"))
|
||||
let step_fn: String = json_get(arg, "step_fn")
|
||||
let purview: String = json_get_raw(arg, "purview")
|
||||
let goal: String = json_get(purview, "goal")
|
||||
let steps: String = json_get_raw(purview, "steps")
|
||||
let n: Int = json_array_len(steps)
|
||||
let idx: Int = 0
|
||||
// NO token check inside the loop — this is the bug.
|
||||
while idx < n {
|
||||
let step: String = json_array_get(steps, idx)
|
||||
let step_arg: String = "{\"goal\":\"" + goal + "\",\"idx\":" +
|
||||
int_to_str(idx) + ",\"step\":" + step + "}"
|
||||
let tid: Int = __thread_create(step_fn, step_arg)
|
||||
let result: String = __thread_join(tid)
|
||||
__channel_send(out_ch, result)
|
||||
let idx = idx + 1
|
||||
}
|
||||
// Only NOW does it look at the control signal — too late.
|
||||
let late: String = token_check(tok)
|
||||
if str_eq(late, "") {
|
||||
__channel_send(out_ch, "[no signal]")
|
||||
} else {
|
||||
__channel_send(out_ch, "[TOO LATE: absorbed " + json_get(late, "sig") +
|
||||
" only after running ALL " + int_to_str(n) +
|
||||
" steps — wasted work]")
|
||||
}
|
||||
return "{\"status\":\"ran-to-completion\",\"completed\":" + int_to_str(idx) + "}"
|
||||
}
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// drain_count — drain out_ch, print each line, return count of real step outputs
|
||||
// (lines beginning with "did").
|
||||
fn drain_and_report(out_ch: Int) -> Int {
|
||||
let done: Int = 0
|
||||
let did: Int = 0
|
||||
while done == 0 {
|
||||
let m: String = __channel_try_recv(out_ch)
|
||||
if str_eq(m, "") {
|
||||
let done = 1
|
||||
} else {
|
||||
println(" | " + m)
|
||||
if str_starts_with(m, "did") {
|
||||
let did = did + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return did
|
||||
}
|
||||
|
||||
fn twelve_steps() -> String {
|
||||
return "[\"s0\",\"s1\",\"s2\",\"s3\",\"s4\",\"s5\",\"s6\",\"s7\",\"s8\",\"s9\",\"s10\",\"s11\"]"
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println("=====================================================================")
|
||||
println(" NATIVE INTERRUPTIBILITY — PROOF (canon 1ca4d3e9)")
|
||||
println(" plan: 12 bounded steps @ ~15ms; signal sent ~50ms in (≈ after step 3)")
|
||||
println("=====================================================================")
|
||||
|
||||
// ── A. BASELINE — broken, non-interruptible ──
|
||||
println("")
|
||||
println("[A] BASELINE broken worker (no mid-loop signal check) — Claude-style")
|
||||
let tokA: Int = token_new()
|
||||
let outA: Int = __channel_new(0)
|
||||
let pvA: String = purview_new("A", "build-wrong-thing", twelve_steps())
|
||||
let argA: String = "{\"tok\":" + int_to_str(tokA) + ",\"out\":" + int_to_str(outA) +
|
||||
",\"step_fn\":\"demo_step\",\"purview\":" + pvA + "}"
|
||||
let t0A: Int = time_now()
|
||||
let tidA: Int = __thread_create("broken_worker_run", argA)
|
||||
sleep_ms(50)
|
||||
println(" -> coordinator sends KILL at +" + int_to_str(time_now() - t0A) + "ms (≈step 3)")
|
||||
token_kill(tokA)
|
||||
let finA: String = __thread_join(tidA)
|
||||
let elapA: Int = time_now() - t0A
|
||||
let didA: Int = drain_and_report(outA)
|
||||
println(" RESULT: executed " + int_to_str(didA) + "/12 steps, wall=" +
|
||||
int_to_str(elapA) + "ms, final=" + finA)
|
||||
println(" >> ignored the kill, RAN ALL 12 — ~9 steps of KNOWN-WRONG waste")
|
||||
|
||||
// ── B. INTERRUPTIBLE KILL ──
|
||||
println("")
|
||||
println("[B] INTERRUPTIBLE swarm_worker_run + token_kill")
|
||||
let pvB: String = purview_new("B", "build-wrong-thing", twelve_steps())
|
||||
let hB: String = swarm_dispatch("demo_step", pvB)
|
||||
let tokB: Int = str_to_int(json_get(hB, "tok"))
|
||||
let outB: Int = str_to_int(json_get(hB, "out"))
|
||||
let tidB: Int = str_to_int(json_get(hB, "tid"))
|
||||
let t0B: Int = time_now()
|
||||
sleep_ms(50)
|
||||
println(" -> coordinator sends KILL at +" + int_to_str(time_now() - t0B) + "ms (≈step 3)")
|
||||
token_kill(tokB)
|
||||
let finB: String = __thread_join(tidB)
|
||||
let elapB: Int = time_now() - t0B
|
||||
let didB: Int = drain_and_report(outB)
|
||||
println(" RESULT: executed " + int_to_str(didB) + "/12 steps, wall=" +
|
||||
int_to_str(elapB) + "ms, final=" + finB)
|
||||
println(" >> STOPPED within one step of the signal — no wasted continuation")
|
||||
|
||||
// ── C. INTERRUPTIBLE REDIRECT (the live incident) ──
|
||||
println("")
|
||||
println("[C] INTERRUPTIBLE redirect mid-task: build-python-synth -> native-fetch-render")
|
||||
let pvC: String = purview_new("C", "build-python-synth-renderer", twelve_steps())
|
||||
let hC: String = swarm_dispatch("demo_step", pvC)
|
||||
let tokC: Int = str_to_int(json_get(hC, "tok"))
|
||||
let outC: Int = str_to_int(json_get(hC, "out"))
|
||||
let tidC: Int = str_to_int(json_get(hC, "tid"))
|
||||
let t0C: Int = time_now()
|
||||
sleep_ms(50)
|
||||
println(" -> coordinator REDIRECTS at +" + int_to_str(time_now() - t0C) + "ms (≈step 3)")
|
||||
token_redirect(tokC, "native-fetch-render", "[\"fetch\",\"realize\",\"cohere\"]")
|
||||
let finC: String = __thread_join(tidC)
|
||||
let elapC: Int = time_now() - t0C
|
||||
let didC: Int = drain_and_report(outC)
|
||||
println(" RESULT: executed " + int_to_str(didC) + " steps total, wall=" +
|
||||
int_to_str(elapC) + "ms, final=" + finC)
|
||||
println(" >> absorbed the correction mid-task: kept early progress,")
|
||||
println(" re-planned onto native-fetch-render, finished the RIGHT plan")
|
||||
|
||||
// ── D. INTERRUPTIBLE PAUSE -> RESUME (hold state, then continue) ──
|
||||
println("")
|
||||
println("[D] INTERRUPTIBLE pause mid-task, hold state, then resume to finish")
|
||||
let pvD: String = purview_new("D", "long-render", twelve_steps())
|
||||
let hD: String = swarm_dispatch("demo_step", pvD)
|
||||
let tokD: Int = str_to_int(json_get(hD, "tok"))
|
||||
let outD: Int = str_to_int(json_get(hD, "out"))
|
||||
let tidD: Int = str_to_int(json_get(hD, "tid"))
|
||||
let t0D: Int = time_now()
|
||||
sleep_ms(50)
|
||||
println(" -> coordinator PAUSES at +" + int_to_str(time_now() - t0D) + "ms (≈step 3)")
|
||||
token_pause(tokD)
|
||||
sleep_ms(60)
|
||||
println(" -> worker held idle for ~60ms; coordinator RESUMES at +" +
|
||||
int_to_str(time_now() - t0D) + "ms")
|
||||
token_resume(tokD)
|
||||
let finD: String = __thread_join(tidD)
|
||||
let didD: Int = drain_and_report(outD)
|
||||
println(" RESULT: executed " + int_to_str(didD) + "/12 steps, final=" + finD)
|
||||
println(" >> paused on the spot, held its plan, resumed and finished it")
|
||||
|
||||
println("")
|
||||
println("=====================================================================")
|
||||
println(" A ran all 12 wrong steps. B stopped at ~3. C re-planned at ~3.")
|
||||
println(" Same plan, same timing, same signal — only the interruptible worker")
|
||||
println(" stops the instant it's told, like a human. QED.")
|
||||
println("=====================================================================")
|
||||
}
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# run.sh — build and run the native-interruptibility proof.
|
||||
#
|
||||
# Concatenates runtime/swarm.el + proof.el into one translation unit (the El
|
||||
# multi-file strategy — elc does not resolve cross-directory imports), compiles
|
||||
# via the canonical elc, links against el_runtime.c, and runs.
|
||||
#
|
||||
# A tiny forward-declaration prelude is prepended to the generated C for the
|
||||
# __channel_* seed primitives (they are defined in el_runtime.c but not declared
|
||||
# in el_runtime.h). This touches nothing shared — it is local to this build.
|
||||
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
EL_HOME="${EL_HOME:-$(cd ../.. && pwd)}"
|
||||
ELC="${EL_HOME}/dist/platform/elc"
|
||||
RT="${EL_HOME}/el-compiler/runtime"
|
||||
SWARM="${EL_HOME}/runtime/swarm.el"
|
||||
|
||||
OSSL="$(brew --prefix openssl@3 2>/dev/null || brew --prefix openssl 2>/dev/null || echo /usr/local)"
|
||||
LDF=(); [ -d "${OSSL}/lib" ] && LDF=(-L"${OSSL}/lib")
|
||||
|
||||
BUILD="$(mktemp -d -t swarmproof.XXXXXX)"
|
||||
trap 'rm -rf "${BUILD}"' EXIT
|
||||
|
||||
if [ ! -x "${ELC}" ]; then echo "elc not found at ${ELC}" >&2; exit 1; fi
|
||||
|
||||
# 1. Concatenate library + proof into one .el
|
||||
cat "${SWARM}" proof.el > "${BUILD}/combined.el"
|
||||
|
||||
# 2. elc emit -> C
|
||||
if ! "${ELC}" "${BUILD}/combined.el" > "${BUILD}/body.c" 2>"${BUILD}/elc.err"; then
|
||||
echo "elc FAILED:"; sed 's/^/ /' "${BUILD}/elc.err"; exit 1
|
||||
fi
|
||||
|
||||
# 3. Prepend forward-decl prelude for the __channel_* seed primitives
|
||||
cat > "${BUILD}/prog.c" <<'PRELUDE'
|
||||
#include <stdint.h>
|
||||
typedef int64_t el_val_t;
|
||||
el_val_t __channel_new(el_val_t);
|
||||
el_val_t __channel_send(el_val_t, el_val_t);
|
||||
el_val_t __channel_recv(el_val_t);
|
||||
el_val_t __channel_try_recv(el_val_t);
|
||||
el_val_t __channel_close(el_val_t);
|
||||
PRELUDE
|
||||
cat "${BUILD}/body.c" >> "${BUILD}/prog.c"
|
||||
|
||||
# 4. cc link
|
||||
if ! cc -O2 -Wno-implicit-function-declaration -I "${RT}" "${LDF[@]}" \
|
||||
"${BUILD}/prog.c" "${RT}/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o "${BUILD}/proof" 2>"${BUILD}/cc.err"; then
|
||||
echo "cc FAILED:"; tail -20 "${BUILD}/cc.err"; exit 1
|
||||
fi
|
||||
|
||||
# 5. run
|
||||
"${BUILD}/proof"
|
||||
@@ -1,8 +0,0 @@
|
||||
# Build + runtime artifacts — never committed.
|
||||
bin/
|
||||
# Captured media (camera frames, mic audio) and syntheses. Raw streams stay
|
||||
# LOCAL and never egress — including into git.
|
||||
out/
|
||||
# Runtime consent + resume state (local, per-machine).
|
||||
.consent.json
|
||||
.resume.json
|
||||
@@ -1,80 +0,0 @@
|
||||
# peripheral — Neuron's I/O organ (own-core, local, consent-gated)
|
||||
|
||||
The interface made physical. Two afferent senses in, one efferent voice out —
|
||||
all reached the way the agentic surface reaches any tool.
|
||||
|
||||
```
|
||||
MIC (hear) afferent device -> capture -> descriptor -> ingest -> geometry
|
||||
CAMERA (see) afferent device -> capture -> descriptor -> ingest -> scene-geometry
|
||||
SPEAKER(speak) efferent render WAV -> PLAY ALOUD out the speaker
|
||||
```
|
||||
|
||||
Closes the conversational loop: **hear (mic) -> understand (engram) -> speak (speaker)**.
|
||||
|
||||
## Rails
|
||||
- **Own-core.** macOS-native only: AVFoundation (camera/mic), CoreAudio voice-
|
||||
processing (AEC), afplay (speaker), ImageIO/CoreGraphics (frames), hand-rolled
|
||||
DSP (WAV, LPC, formant synthesis). No cloud, no heavy deps.
|
||||
- **Local-only.** Raw streams are written to `out/` and never egress. `.gitignore`
|
||||
keeps captured media out of git.
|
||||
- **Consent-gated (two locks).** A Neuron-level grant (`grant`/`revoke`) *and* the
|
||||
OS TCC permission. Sensitive senses (camera/mic) fail closed without both.
|
||||
- **Disclosed.** Every device touch prints a `[peripheral]` line on stderr.
|
||||
|
||||
## Build
|
||||
```
|
||||
swiftc -O -o bin/periph src/periph.swift \
|
||||
-framework AVFoundation -framework CoreMedia -framework Foundation \
|
||||
-framework CoreGraphics -framework ImageIO -framework CoreImage
|
||||
```
|
||||
|
||||
## Commands
|
||||
```
|
||||
periph grant|revoke <camera|mic> # Neuron-level consent
|
||||
periph status
|
||||
periph speak <file.wav> # SPEAK ALOUD (efferent)
|
||||
periph tone <out.wav> [hz] [sec] # own-core WAV synth
|
||||
periph listen <sec> <out.wav> # MIC capture (afferent), 16k mono
|
||||
periph see <out.jpg> # CAMERA one frame (afferent)
|
||||
periph feat-audio <wav> | feat-image <jpg> # capture -> compact descriptor
|
||||
periph ingest-audio|ingest-image <file> <engramURL> # descriptor -> engram node (geometry)
|
||||
periph voiceprint <voice.wav> # extract F0 + formants F1-F5
|
||||
periph imitate <voice.wav> <out.wav> # speak back in that voice (LPC resynthesis)
|
||||
periph hear-imitate <sec> <out.wav> # MIC -> signature -> imitate -> SPEAK ALOUD
|
||||
periph converse <manifest.json> [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] [--live-mic]
|
||||
```
|
||||
|
||||
## The afferent metabolism
|
||||
A capture is never shipped raw. It becomes a **compact descriptor** — the afferent
|
||||
twin of the music instrument-signature:
|
||||
- audio -> `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` (~2400-6000x smaller)
|
||||
- image -> `[w, h, meanRGB, brightness, 3x3 luminance grid]` (~400000x smaller)
|
||||
- voice -> `[F0, F1..F5, bandwidths]` (11 numbers)
|
||||
|
||||
That descriptor is what the ingest organ (engram `POST /api/nodes`) turns into an
|
||||
embedded node = geometry.
|
||||
|
||||
## Voice by imitation
|
||||
`voiceprint`/`imitate` are own-core LPC (autocorrelation + Levinson-Durbin, order
|
||||
16 @ 16 kHz), formant extraction from the LPC spectral envelope, and source-filter
|
||||
resynthesis (glottal impulse train at F0 through the all-pole formant filter). A
|
||||
voice is grabbed by ear as ~a dozen numbers and spoken back — **no training, no
|
||||
stolen voice.** Measured fidelity on real speech: resynthesized formants match the
|
||||
source within 2-3%. The full phoneme->formant path for *novel* sentences is the
|
||||
speech faculty's seam (`elp` audio surface profile); this engine provides the
|
||||
formant synthesis primitive it renders through.
|
||||
|
||||
## Interruptibility (native turn-taking)
|
||||
`converse` plays the utterance as an ordered, salience-tagged **meaning-plan**
|
||||
while the mic listens (full-duplex, AEC on so it never barges in on its own voice):
|
||||
- **barge-in**: user speech -> pause on the spot (sample-accurate), not "finish the buffer."
|
||||
- **yield-or-hold**: a decision grounded in the current segment's salience + progress
|
||||
+ the interrupter's authority — YIELD (stop) or HOLD ("hang on, let me finish").
|
||||
- **backchannel** ("mm-hm"): brief/low -> keep going, resume seamlessly.
|
||||
- **resumable**: on yield the remaining plan persists (`.resume.json`); `--resume`
|
||||
picks the thread back up ("as I was saying").
|
||||
|
||||
Live full-duplex uses `--live-mic` (OS AEC). Injected `--barge-at` drives the
|
||||
decision loop deterministically for testing.
|
||||
```
|
||||
```
|
||||
@@ -1,939 +0,0 @@
|
||||
// periph.swift — Neuron's PERIPHERAL I/O organ (own-core, LOCAL, CONSENT-GATED).
|
||||
//
|
||||
// The interface made physical:
|
||||
// MIC (hear) = afferent : device -> capture -> [ingest -> geometry]
|
||||
// CAMERA (see) = afferent : device -> capture -> [ingest -> scene-geometry]
|
||||
// SPEAKER(speak) = efferent : [render WAV] -> PLAY ALOUD out the speaker
|
||||
//
|
||||
// Rails: own-core (AVFoundation / CoreAudio / afplay — all ship with macOS),
|
||||
// no cloud, no heavy deps, raw streams stay LOCAL and never egress,
|
||||
// every device access is CONSENT-GATED and DISCLOSED.
|
||||
//
|
||||
// Full-duplex CONVERSE mode implements native interruptibility: while the
|
||||
// speaker plays the utterance (a persistent, segmented meaning-plan), the mic
|
||||
// listens; on user speech it interrupts instantly, then DECIDES yield-or-hold
|
||||
// grounded in the salience of what it is mid-saying, and can RESUME the thread.
|
||||
//
|
||||
// Build: swiftc -O -o peripheral/bin/periph peripheral/src/periph.swift \
|
||||
// -framework AVFoundation -framework CoreMedia -framework Foundation
|
||||
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import CoreMedia
|
||||
import CoreGraphics
|
||||
import ImageIO
|
||||
import CoreImage
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Disclosure — every peripheral touch is announced on stderr. Nothing is silent.
|
||||
// ----------------------------------------------------------------------------
|
||||
func disclose(_ msg: String) {
|
||||
FileHandle.standardError.write(" [peripheral] \(msg)\n".data(using: .utf8)!)
|
||||
}
|
||||
func emit(_ obj: [String: Any]) { // machine-readable event on stdout (JSON line)
|
||||
if let d = try? JSONSerialization.data(withJSONObject: obj),
|
||||
let s = String(data: d, encoding: .utf8) {
|
||||
print(s)
|
||||
}
|
||||
}
|
||||
func die(_ msg: String) -> Never {
|
||||
disclose("ERROR: \(msg)")
|
||||
emit(["ok": false, "error": msg])
|
||||
exit(1)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Consent store — Neuron's OWN gate, on top of the OS (TCC) gate. Two locks on
|
||||
// the sensitive senses. Persisted locally next to the binary's organ dir.
|
||||
// ----------------------------------------------------------------------------
|
||||
struct Consent {
|
||||
static let path: String = {
|
||||
let dir = ProcessInfo.processInfo.environment["PERIPH_HOME"]
|
||||
?? FileManager.default.currentDirectoryPath + "/peripheral"
|
||||
return dir + "/.consent.json"
|
||||
}()
|
||||
|
||||
static func load() -> [String: Bool] {
|
||||
guard let d = FileManager.default.contents(atPath: path),
|
||||
let o = try? JSONSerialization.jsonObject(with: d) as? [String: Bool]
|
||||
else { return ["camera": false, "mic": false] }
|
||||
return o
|
||||
}
|
||||
static func save(_ g: [String: Bool]) {
|
||||
let d = try! JSONSerialization.data(withJSONObject: g, options: [.prettyPrinted])
|
||||
try? d.write(to: URL(fileURLWithPath: path))
|
||||
}
|
||||
// Neuron-level gate. Sensitive senses (camera/mic) require an explicit grant.
|
||||
static func require(_ device: String) {
|
||||
let g = load()
|
||||
if g[device] != true {
|
||||
die("CONSENT DENIED for '\(device)'. The user has not granted this sense. " +
|
||||
"Run: periph grant \(device) (raw streams stay local, never egress).")
|
||||
}
|
||||
disclose("consent OK (Neuron-level) for '\(device)' — local only, never egresses.")
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// OS (TCC) permission — the second lock. AVFoundation prompts the user the first
|
||||
// time; if denied, we fail cleanly rather than hang.
|
||||
// ----------------------------------------------------------------------------
|
||||
func requireOSAccess(_ media: AVMediaType, _ label: String) {
|
||||
let status = AVCaptureDevice.authorizationStatus(for: media)
|
||||
switch status {
|
||||
case .authorized:
|
||||
disclose("consent OK (OS/TCC) for \(label).")
|
||||
return
|
||||
case .notDetermined:
|
||||
disclose("requesting OS permission for \(label) (first use) — user must grant...")
|
||||
let sem = DispatchSemaphore(value: 0)
|
||||
var ok = false
|
||||
AVCaptureDevice.requestAccess(for: media) { granted in ok = granted; sem.signal() }
|
||||
_ = sem.wait(timeout: .now() + 30)
|
||||
if !ok { die("OS permission for \(label) was not granted.") }
|
||||
disclose("consent OK (OS/TCC) for \(label).")
|
||||
case .denied, .restricted:
|
||||
die("OS permission for \(label) is DENIED in System Settings > Privacy. " +
|
||||
"Grant it to the controlling terminal/app, then retry.")
|
||||
@unknown default:
|
||||
die("unknown OS permission state for \(label).")
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Own-core WAV writer (16-bit PCM). No library — proves we own the medium.
|
||||
// ----------------------------------------------------------------------------
|
||||
func writeWav(_ url: URL, samples: [Int16], sampleRate: Int, channels: Int = 1) {
|
||||
var data = Data()
|
||||
func u32(_ v: UInt32) { var x = v.littleEndian; data.append(Data(bytes: &x, count: 4)) }
|
||||
func u16(_ v: UInt16) { var x = v.littleEndian; data.append(Data(bytes: &x, count: 2)) }
|
||||
let bytesPerSample = 2
|
||||
let dataBytes = samples.count * bytesPerSample
|
||||
let byteRate = sampleRate * channels * bytesPerSample
|
||||
data.append("RIFF".data(using: .ascii)!); u32(UInt32(36 + dataBytes))
|
||||
data.append("WAVE".data(using: .ascii)!)
|
||||
data.append("fmt ".data(using: .ascii)!); u32(16); u16(1); u16(UInt16(channels))
|
||||
u32(UInt32(sampleRate)); u32(UInt32(byteRate))
|
||||
u16(UInt16(channels * bytesPerSample)); u16(16)
|
||||
data.append("data".data(using: .ascii)!); u32(UInt32(dataBytes))
|
||||
for s in samples { var x = s.littleEndian; data.append(Data(bytes: &x, count: 2)) }
|
||||
try? data.write(to: url)
|
||||
}
|
||||
|
||||
// Read a WAV's basic geometry (own-core header parse). Walks chunks to find
|
||||
// 'fmt ' and 'data' — robust to JUNK/FLLR padding chunks (AVAudioRecorder emits them).
|
||||
func wavInfo(_ path: String) -> (sampleRate: Int, channels: Int, bits: Int, frames: Int)? {
|
||||
guard let d = FileManager.default.contents(atPath: path), d.count > 44 else { return nil }
|
||||
func rd16(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1]) << 8) }
|
||||
func rd32(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1])<<8) | (Int(d[o+2])<<16) | (Int(d[o+3])<<24) }
|
||||
var channels = 0, sampleRate = 0, bits = 0, dataSize = 0
|
||||
var o = 12
|
||||
while o + 8 <= d.count {
|
||||
let id = String(bytes: d[o..<o+4], encoding: .ascii) ?? ""
|
||||
let sz = rd32(o+4)
|
||||
if id == "fmt " && o + 24 <= d.count {
|
||||
channels = rd16(o+10); sampleRate = rd32(o+12); bits = rd16(o+22)
|
||||
} else if id == "data" {
|
||||
dataSize = min(sz, d.count - (o+8))
|
||||
}
|
||||
o += 8 + sz + (sz & 1)
|
||||
}
|
||||
let frames = (channels > 0 && bits > 0) ? dataSize / (channels * bits/8) : 0
|
||||
return (sampleRate, channels, bits, frames)
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// SPEAKER (efferent) — play a WAV ALOUD. Own-core: afplay ships with macOS.
|
||||
// ----------------------------------------------------------------------------
|
||||
func speak(_ wavPath: String) {
|
||||
guard FileManager.default.fileExists(atPath: wavPath) else { die("no such file: \(wavPath)") }
|
||||
disclose("SPEAKER: playing '\(wavPath)' ALOUD out the local speaker (efferent).")
|
||||
let p = Process()
|
||||
p.executableURL = URL(fileURLWithPath: "/usr/bin/afplay")
|
||||
p.arguments = [wavPath]
|
||||
try? p.run(); p.waitUntilExit()
|
||||
let ok = p.terminationStatus == 0
|
||||
disclose(ok ? "SPEAKER: done — Neuron spoke aloud." : "SPEAKER: afplay failed.")
|
||||
if let i = wavInfo(wavPath) {
|
||||
emit(["ok": ok, "op": "speak", "file": wavPath, "played_aloud": ok,
|
||||
"sample_rate": i.sampleRate, "channels": i.channels,
|
||||
"seconds": Double(i.frames)/Double(max(i.sampleRate,1))])
|
||||
} else {
|
||||
emit(["ok": ok, "op": "speak", "file": wavPath, "played_aloud": ok])
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// MIC (afferent) — capture N seconds -> 16k mono 16-bit WAV (formant-ready).
|
||||
// ----------------------------------------------------------------------------
|
||||
func listen(seconds: Double, out: String) {
|
||||
Consent.require("mic")
|
||||
requireOSAccess(.audio, "microphone")
|
||||
disclose("MIC: capturing \(seconds)s -> '\(out)' (16 kHz mono, LOCAL, never egresses).")
|
||||
|
||||
let url = URL(fileURLWithPath: out)
|
||||
let settings: [String: Any] = [
|
||||
AVFormatIDKey: kAudioFormatLinearPCM,
|
||||
AVSampleRateKey: 16000.0,
|
||||
AVNumberOfChannelsKey: 1,
|
||||
AVLinearPCMBitDepthKey: 16,
|
||||
AVLinearPCMIsFloatKey: false,
|
||||
AVLinearPCMIsBigEndianKey: false,
|
||||
]
|
||||
guard let rec = try? AVAudioRecorder(url: url, settings: settings) else {
|
||||
die("could not open the microphone recorder.")
|
||||
}
|
||||
rec.record()
|
||||
Thread.sleep(forTimeInterval: seconds)
|
||||
rec.stop()
|
||||
// let the file flush
|
||||
Thread.sleep(forTimeInterval: 0.1)
|
||||
if let i = wavInfo(out) {
|
||||
disclose("MIC: captured \(i.frames) frames @ \(i.sampleRate)Hz — ready to hand to the ingest organ.")
|
||||
emit(["ok": true, "op": "listen", "file": out, "sample_rate": i.sampleRate,
|
||||
"channels": i.channels, "frames": i.frames,
|
||||
"seconds": Double(i.frames)/Double(max(i.sampleRate,1)),
|
||||
"next": "ingest -> phonetic/voice geometry"])
|
||||
} else {
|
||||
die("mic capture produced no readable WAV.")
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// CAMERA (afferent) — capture ONE frame -> JPEG on disk.
|
||||
// ----------------------------------------------------------------------------
|
||||
// Grab one video frame via AVCaptureVideoDataOutput (CLI-safe; no KVO/photo classes).
|
||||
final class FrameGrabber: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
||||
let sem = DispatchSemaphore(value: 0)
|
||||
var cgImage: CGImage?
|
||||
var seen = 0
|
||||
let cictx = CIContext(options: nil)
|
||||
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
|
||||
from connection: AVCaptureConnection) {
|
||||
seen += 1
|
||||
if cgImage != nil || seen < 5 { return } // let exposure settle a few frames
|
||||
guard let pb = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||
let ci = CIImage(cvPixelBuffer: pb)
|
||||
cgImage = cictx.createCGImage(ci, from: ci.extent)
|
||||
sem.signal()
|
||||
}
|
||||
}
|
||||
func see(out: String) {
|
||||
Consent.require("camera")
|
||||
requireOSAccess(.video, "camera")
|
||||
disclose("CAMERA: capturing one frame -> '\(out)' (LOCAL, never egresses).")
|
||||
|
||||
let session = AVCaptureSession()
|
||||
session.sessionPreset = .photo
|
||||
guard let device = AVCaptureDevice.default(for: .video),
|
||||
let input = try? AVCaptureDeviceInput(device: device),
|
||||
session.canAddInput(input) else { die("no camera device available.") }
|
||||
session.addInput(input)
|
||||
let output = AVCaptureVideoDataOutput()
|
||||
output.alwaysDiscardsLateVideoFrames = true
|
||||
let grabber = FrameGrabber()
|
||||
output.setSampleBufferDelegate(grabber, queue: DispatchQueue(label: "periph.cam"))
|
||||
guard session.canAddOutput(output) else { die("cannot add video output.") }
|
||||
session.addOutput(output)
|
||||
session.startRunning()
|
||||
|
||||
if grabber.sem.wait(timeout: .now() + 10) == .timedOut { session.stopRunning(); die("camera capture timed out.") }
|
||||
session.stopRunning()
|
||||
|
||||
guard let cg = grabber.cgImage,
|
||||
let dst = CGImageDestinationCreateWithURL(URL(fileURLWithPath: out) as CFURL,
|
||||
"public.jpeg" as CFString, 1, nil)
|
||||
else { die("camera returned no frame.") }
|
||||
CGImageDestinationAddImage(dst, cg, nil)
|
||||
guard CGImageDestinationFinalize(dst) else { die("could not write JPEG.") }
|
||||
let bytes = ((try? FileManager.default.attributesOfItem(atPath: out))?[.size] as? Int) ?? 0
|
||||
disclose("CAMERA: wrote \(cg.width)x\(cg.height) frame (\(bytes) bytes) — ready for scene-geometry ingest.")
|
||||
emit(["ok": true, "op": "see", "file": out, "width": cg.width, "height": cg.height,
|
||||
"bytes": bytes, "next": "ingest -> scene-geometry"])
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// FEAT — the afferent METABOLISM: a raw capture becomes a COMPACT descriptor
|
||||
// (a few dozen numbers), the mirror of the efferent signature. This is what
|
||||
// gets handed to the ingest organ as geometry — NOT the raw stream. Own-core.
|
||||
// ============================================================================
|
||||
|
||||
// Read all 16-bit PCM samples from a WAV (own-core).
|
||||
func readWavSamples(_ path: String) -> (samples: [Double], sr: Int, ch: Int)? {
|
||||
guard let d = FileManager.default.contents(atPath: path), d.count > 44 else { return nil }
|
||||
func rd16(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1]) << 8) }
|
||||
func rd32(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1])<<8) | (Int(d[o+2])<<16) | (Int(d[o+3])<<24) }
|
||||
var ch = 0, sr = 0, bits = 0
|
||||
var o = 12
|
||||
while o + 8 <= d.count {
|
||||
let id = String(bytes: d[o..<o+4], encoding: .ascii) ?? ""
|
||||
let sz = rd32(o+4)
|
||||
if id == "fmt " && o + 24 <= d.count { ch = rd16(o+10); sr = rd32(o+12); bits = rd16(o+22) }
|
||||
if id == "data" {
|
||||
guard bits == 16, ch > 0 else { return nil }
|
||||
var samples = [Double](); let start = o + 8
|
||||
let end = min(start + sz, d.count - 1)
|
||||
var i = start
|
||||
while i + 1 < end {
|
||||
var v = Int(rd16(i)); if v >= 32768 { v -= 65536 }
|
||||
samples.append(Double(v) / 32768.0)
|
||||
i += 2 * ch // take channel 0 if stereo
|
||||
}
|
||||
return (samples, sr, ch)
|
||||
}
|
||||
o += 8 + sz + (sz & 1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Audio descriptor = compact sound/voice signature (energy, ZCR, centroid, F0).
|
||||
// The seed for phonetic geometry + the hear->imitate voice-signature.
|
||||
func computeAudio(_ path: String) -> (content: String, vector: [Double], extra: [String: Any]) {
|
||||
guard let (s, sr, ch) = readWavSamples(path), !s.isEmpty else { die("cannot read PCM from \(path)") }
|
||||
let n = s.count
|
||||
let seconds = Double(n) / Double(sr)
|
||||
var sumsq = 0.0, peak = 0.0, zc = 0.0
|
||||
for i in 0..<n {
|
||||
sumsq += s[i]*s[i]; peak = max(peak, abs(s[i]))
|
||||
if i > 0 && (s[i-1] < 0) != (s[i] < 0) { zc += 1 }
|
||||
}
|
||||
let rms = (sumsq / Double(n)).squareRoot()
|
||||
let zcr = zc / Double(n) * Double(sr) // ~2*dominant freq for tonal
|
||||
// Spectral centroid via a coarse DFT on a mid window (own-core).
|
||||
let W = min(2048, n); let off = max(0, (n - W)/2)
|
||||
var num = 0.0, den = 0.0
|
||||
let bins = 64
|
||||
for k in 1..<bins {
|
||||
let f = Double(k) * Double(sr) / Double(2*bins)
|
||||
var re = 0.0, im = 0.0
|
||||
for j in 0..<W {
|
||||
let ang = -2*Double.pi*Double(k)*Double(j)/Double(2*bins)
|
||||
re += s[off+j]*cos(ang); im += s[off+j]*sin(ang)
|
||||
}
|
||||
let mag = (re*re+im*im).squareRoot()
|
||||
num += f*mag; den += mag
|
||||
}
|
||||
let centroid = den > 0 ? num/den : 0
|
||||
// F0 via autocorrelation (voice pitch) over plausible speech range 70-400 Hz.
|
||||
var bestLag = 0; var bestCorr = 0.0
|
||||
let lagMin = sr/400, lagMax = min(sr/70, n-1)
|
||||
if lagMax > lagMin {
|
||||
for lag in lagMin...lagMax {
|
||||
var c = 0.0
|
||||
var i = 0; while i + lag < min(n, off+W) { c += s[off+i]*s[off+i+lag]; i += 1 }
|
||||
if c > bestCorr { bestCorr = c; bestLag = lag }
|
||||
}
|
||||
}
|
||||
let f0 = bestLag > 0 ? Double(sr)/Double(bestLag) : 0
|
||||
let vector: [Double] = [seconds, Double(sr), Double(ch), rms, peak, zcr, centroid, f0]
|
||||
let content = String(format:
|
||||
"Heard sound (afferent, mic): %.2fs at %dHz. RMS energy %.3f, peak %.3f, " +
|
||||
"zero-crossing rate %.0fHz, spectral centroid %.0fHz, estimated voice pitch F0 %.0fHz. " +
|
||||
"Compact voice/sound signature (%d numbers) — phonetic geometry + hear-to-imitate seed.",
|
||||
seconds, sr, rms, peak, zcr, centroid, f0, vector.count)
|
||||
disclose("FEAT(audio): \(vector.count)-number signature vs \(n) raw samples (~\(n/max(vector.count,1))x compression).")
|
||||
return (content, vector, ["f0_hz": f0, "centroid_hz": centroid, "zcr_hz": zcr,
|
||||
"rms": rms, "seconds": seconds, "raw_samples": n])
|
||||
}
|
||||
func featAudio(_ path: String) {
|
||||
let r = computeAudio(path)
|
||||
var out: [String: Any] = ["ok": true, "op": "feat-audio", "file": path,
|
||||
"vector": r.vector, "content": r.content,
|
||||
"ingest": ["node_type": "Observation", "tier": "Episodic", "content": r.content]]
|
||||
r.extra.forEach { out[$0] = $1 }
|
||||
emit(out)
|
||||
}
|
||||
|
||||
// Image descriptor = compact scene-geometry (dims, brightness, region grid).
|
||||
func computeImage(_ path: String) -> (content: String, vector: [Double], extra: [String: Any]) {
|
||||
guard let src = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil),
|
||||
let img = CGImageSourceCreateImageAtIndex(src, 0, nil) else { die("cannot decode image \(path)") }
|
||||
let w = img.width, h = img.height
|
||||
let cs = CGColorSpaceCreateDeviceRGB()
|
||||
let bpr = w * 4
|
||||
var buf = [UInt8](repeating: 0, count: h * bpr)
|
||||
guard let ctx = CGContext(data: &buf, width: w, height: h, bitsPerComponent: 8,
|
||||
bytesPerRow: bpr, space: cs,
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {
|
||||
die("cannot rasterize image")
|
||||
}
|
||||
ctx.draw(img, in: CGRect(x: 0, y: 0, width: w, height: h))
|
||||
// 3x3 region average luminance + overall average color.
|
||||
var rAvg = 0.0, gAvg = 0.0, bAvg = 0.0
|
||||
var grid = [Double](repeating: 0, count: 9); var gridN = [Int](repeating: 0, count: 9)
|
||||
let step = max(1, (w*h)/40000) // subsample for speed
|
||||
var count = 0; var idx = 0
|
||||
while idx < w*h {
|
||||
let x = idx % w, y = idx / w
|
||||
let p = y*bpr + x*4
|
||||
let r = Double(buf[p]), g = Double(buf[p+1]), b = Double(buf[p+2])
|
||||
rAvg += r; gAvg += g; bAvg += b; count += 1
|
||||
let cell = (min(2, y*3/h))*3 + min(2, x*3/w)
|
||||
grid[cell] += 0.299*r + 0.587*g + 0.114*b; gridN[cell] += 1
|
||||
idx += step
|
||||
}
|
||||
if count == 0 { die("no pixels sampled") }
|
||||
rAvg /= Double(count); gAvg /= Double(count); bAvg /= Double(count)
|
||||
for i in 0..<9 { grid[i] = gridN[i] > 0 ? grid[i]/Double(gridN[i]) : 0 }
|
||||
let bright = (0.299*rAvg + 0.587*gAvg + 0.114*bAvg)/255.0
|
||||
let vector = [Double(w), Double(h), rAvg/255, gAvg/255, bAvg/255, bright] + grid.map { $0/255 }
|
||||
let content = String(format:
|
||||
"Saw scene (afferent, camera): %dx%d frame. Mean color rgb(%.0f,%.0f,%.0f), " +
|
||||
"brightness %.2f. 3x3 luminance grid [%.0f %.0f %.0f / %.0f %.0f %.0f / %.0f %.0f %.0f]. " +
|
||||
"Compact scene-geometry (%d numbers) vs %d pixel-channels.",
|
||||
w, h, rAvg, gAvg, bAvg, bright,
|
||||
grid[0],grid[1],grid[2],grid[3],grid[4],grid[5],grid[6],grid[7],grid[8],
|
||||
vector.count, w*h*3)
|
||||
disclose("FEAT(image): \(vector.count)-number scene-geometry vs \(w*h*3) pixel-channels (~\(w*h*3/max(vector.count,1))x).")
|
||||
return (content, vector, ["width": w, "height": h, "brightness": bright])
|
||||
}
|
||||
func featImage(_ path: String) {
|
||||
let r = computeImage(path)
|
||||
var out: [String: Any] = ["ok": true, "op": "feat-image", "file": path,
|
||||
"vector": r.vector, "content": r.content,
|
||||
"ingest": ["node_type": "Observation", "tier": "Episodic", "content": r.content]]
|
||||
r.extra.forEach { out[$0] = $1 }
|
||||
emit(out)
|
||||
}
|
||||
|
||||
// The afferent WIRE — hand a capture's descriptor to the ingest organ (engram),
|
||||
// where it becomes an embedded node = GEOMETRY. Own-core URLSession POST.
|
||||
// LOCAL only: point at a local engram; raw stream never leaves the machine.
|
||||
func postNode(engramURL: String, content: String, label: String, tags: [String]) -> String? {
|
||||
guard let url = URL(string: engramURL + "/api/nodes") else { return nil }
|
||||
let body: [String: Any] = ["content": content, "node_type": "Observation",
|
||||
"label": label, "tier": "Episodic",
|
||||
"salience": 0.7, "importance": 0.6, "confidence": 0.9,
|
||||
"tags": tags]
|
||||
var req = URLRequest(url: url); req.httpMethod = "POST"
|
||||
req.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
req.httpBody = try? JSONSerialization.data(withJSONObject: body)
|
||||
let sem = DispatchSemaphore(value: 0); var out: String?
|
||||
URLSession.shared.dataTask(with: req) { data, _, _ in
|
||||
if let d = data { out = String(data: d, encoding: .utf8) }
|
||||
sem.signal()
|
||||
}.resume()
|
||||
_ = sem.wait(timeout: .now() + 15)
|
||||
return out
|
||||
}
|
||||
func ingest(_ path: String, kind: String, engramURL: String) {
|
||||
let r = kind == "audio" ? computeAudio(path) : computeImage(path)
|
||||
let label = kind == "audio" ? "heard:mic" : "saw:camera"
|
||||
disclose("INGEST: handing \(kind) descriptor to the ingest organ at \(engramURL) (LOCAL) -> geometry.")
|
||||
guard let resp = postNode(engramURL: engramURL, content: r.content, label: label,
|
||||
tags: ["peripheral", kind == "audio" ? "afferent-mic" : "afferent-camera"]) else {
|
||||
die("ingest POST failed (no local engram at \(engramURL)?)")
|
||||
}
|
||||
// pull the node id out of the response (own-core, tolerant)
|
||||
var nodeId = ""
|
||||
if let d = resp.data(using: .utf8),
|
||||
let o = try? JSONSerialization.jsonObject(with: d) as? [String: Any] {
|
||||
nodeId = (o["id"] as? String) ?? (o["node_id"] as? String) ?? ""
|
||||
}
|
||||
disclose("INGEST: landed as node \(nodeId.isEmpty ? "(see response)" : nodeId) — the capture is now geometry in the engram.")
|
||||
emit(["ok": !nodeId.isEmpty, "op": "ingest-\(kind)", "file": path,
|
||||
"node_id": nodeId, "engram_response": resp, "content": r.content,
|
||||
"vector": r.vector])
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// VOICE BY IMITATION — hear a voice, grab its compact SIGNATURE (pitch +
|
||||
// formants F1-F5 via LPC), and speak back in that voice by source-filter
|
||||
// resynthesis. Own-core DSP (physics), no training, no stolen voice. The
|
||||
// afferent twin of the music instrument-signature: a voice = a few dozen
|
||||
// numbers, not a corpus.
|
||||
// ============================================================================
|
||||
|
||||
func hamming(_ x: [Double]) -> [Double] {
|
||||
let n = x.count; if n < 2 { return x }
|
||||
return (0..<n).map { x[$0] * (0.54 - 0.46*cos(2*Double.pi*Double($0)/Double(n-1))) }
|
||||
}
|
||||
func autocorr(_ x: [Double], _ p: Int) -> [Double] {
|
||||
var r = [Double](repeating: 0, count: p+1)
|
||||
for lag in 0...p { var s = 0.0; var i = lag; while i < x.count { s += x[i]*x[i-lag]; i += 1 }; r[lag] = s }
|
||||
return r
|
||||
}
|
||||
// Levinson-Durbin -> LPC coeffs a[0..p] (A(z)=1+sum a[k]z^-k) and residual energy.
|
||||
func levinson(_ r: [Double], _ p: Int) -> (a: [Double], err: Double) {
|
||||
var a = [Double](repeating: 0, count: p+1); a[0] = 1
|
||||
var err = r[0]
|
||||
if err <= 0 { return (a, 0) }
|
||||
for i in 1...p {
|
||||
var acc = r[i]
|
||||
if i > 1 { for j in 1..<i { acc += a[j]*r[i-j] } }
|
||||
let k = -acc/err
|
||||
var na = a; na[i] = k
|
||||
if i > 1 { for j in 1..<i { na[j] = a[j] + k*a[i-j] } }
|
||||
a = na; err *= (1 - k*k)
|
||||
if err <= 0 { break }
|
||||
}
|
||||
return (a, err)
|
||||
}
|
||||
// Formant peaks from the LPC all-pole spectral envelope.
|
||||
func formants(_ a: [Double], sr: Int) -> [(f: Double, bw: Double)] {
|
||||
let p = a.count - 1
|
||||
let steps = 512
|
||||
var mag = [Double](repeating: 0, count: steps)
|
||||
for s in 0..<steps {
|
||||
let w = Double.pi * Double(s) / Double(steps) // 0..pi -> 0..sr/2
|
||||
var re = 0.0, im = 0.0
|
||||
for k in 0...p { re += a[k]*cos(w*Double(k)); im -= a[k]*sin(w*Double(k)) }
|
||||
mag[s] = 1.0 / max((re*re+im*im).squareRoot(), 1e-9)
|
||||
}
|
||||
var peaks: [(f: Double, bw: Double)] = []
|
||||
for s in 1..<(steps-1) where mag[s] > mag[s-1] && mag[s] >= mag[s+1] {
|
||||
let f = Double(s) * Double(sr) / 2 / Double(steps)
|
||||
if f > 150 && f < 5200 {
|
||||
// crude bandwidth: width where magnitude falls to peak/sqrt(2)
|
||||
let thr = mag[s]/1.4142
|
||||
var lo = s; while lo > 0 && mag[lo] > thr { lo -= 1 }
|
||||
var hi = s; while hi < steps-1 && mag[hi] > thr { hi += 1 }
|
||||
let bw = Double(hi-lo) * Double(sr) / 2 / Double(steps)
|
||||
peaks.append((f, bw))
|
||||
}
|
||||
}
|
||||
return Array(peaks.prefix(5))
|
||||
}
|
||||
func pitchOf(_ frame: [Double], sr: Int) -> Double {
|
||||
let n = frame.count
|
||||
let lagMin = sr/400, lagMax = min(sr/70, n-1)
|
||||
if lagMax <= lagMin { return 0 }
|
||||
var r0 = 0.0; for v in frame { r0 += v*v }
|
||||
if r0 < 1e-5 { return 0 }
|
||||
var bestLag = 0; var best = 0.0
|
||||
for lag in lagMin...lagMax { var c = 0.0; var i = lag; while i < n { c += frame[i]*frame[i-lag]; i += 1 }; if c > best { best = c; bestLag = lag } }
|
||||
return (best / r0 > 0.30 && bestLag > 0) ? Double(sr)/Double(bestLag) : 0 // voiced?
|
||||
}
|
||||
|
||||
let LPC_ORDER = 16
|
||||
let FRAME = 400 // 25ms @16k
|
||||
let HOP = 160 // 10ms
|
||||
|
||||
// Extract Will's voice-signature: averaged F0 + formants over voiced frames.
|
||||
func voiceprint(_ path: String) -> (f0: Double, f0lo: Double, f0hi: Double, formants: [(Double,Double)], content: String) {
|
||||
guard let (x, sr, _) = readWavSamples(path), x.count > FRAME else { die("cannot read speech from \(path)") }
|
||||
var f0s: [Double] = []
|
||||
var fbank: [[Double]] = [[],[],[],[],[]]
|
||||
var bbank: [[Double]] = [[],[],[],[],[]]
|
||||
var pos = 0
|
||||
while pos + FRAME <= x.count {
|
||||
let raw = Array(x[pos..<pos+FRAME])
|
||||
let f0 = pitchOf(raw, sr: sr)
|
||||
if f0 > 0 { // voiced frame only
|
||||
f0s.append(f0)
|
||||
let r = autocorr(hamming(raw), LPC_ORDER)
|
||||
if r[0] > 1e-6 {
|
||||
let (a, _) = levinson(r, LPC_ORDER)
|
||||
let fs = formants(a, sr: sr)
|
||||
for (i, fm) in fs.enumerated() where i < 5 { fbank[i].append(fm.f); bbank[i].append(fm.bw) }
|
||||
}
|
||||
}
|
||||
pos += HOP
|
||||
}
|
||||
func med(_ v: [Double]) -> Double { v.isEmpty ? 0 : v.sorted()[v.count/2] }
|
||||
let f0med = med(f0s)
|
||||
let f0lo = f0s.isEmpty ? 0 : f0s.sorted().first!
|
||||
let f0hi = f0s.isEmpty ? 0 : f0s.sorted().last!
|
||||
var forms: [(Double,Double)] = []
|
||||
for i in 0..<5 where !fbank[i].isEmpty { forms.append((med(fbank[i]), med(bbank[i]))) }
|
||||
let fstr = forms.map { String(format:"%.0f", $0.0) }.joined(separator: "/")
|
||||
let content = String(format:
|
||||
"Voice-signature (afferent, heard a voice): pitch F0 %.0fHz (range %.0f-%.0fHz), " +
|
||||
"formants F1-F5 = %@ Hz. Compact voiceprint (%d numbers) — grabbed by ear for imitation, not trained.",
|
||||
f0med, f0lo, f0hi, fstr, 1 + forms.count*2)
|
||||
return (f0med, f0lo, f0hi, forms, content)
|
||||
}
|
||||
|
||||
// IMITATE: LPC analysis-resynthesis. Reconstruct the heard voice from its
|
||||
// per-frame filter model + pitch — the voice rebuilt from its signature.
|
||||
func imitate(inPath: String, outPath: String) {
|
||||
guard let (x, sr, _) = readWavSamples(inPath), x.count > FRAME else { die("cannot read speech from \(inPath)") }
|
||||
var out = [Double](repeating: 0, count: x.count)
|
||||
var state = [Double](repeating: 0, count: LPC_ORDER) // past outputs
|
||||
var phase = 0.0
|
||||
var lastF0 = 0.0
|
||||
var pos = 0
|
||||
while pos + FRAME <= x.count {
|
||||
let raw = Array(x[pos..<pos+FRAME])
|
||||
let r = autocorr(hamming(raw), LPC_ORDER)
|
||||
let f0 = pitchOf(raw, sr: sr)
|
||||
if r[0] < 1e-7 { pos += HOP; continue }
|
||||
let (a, err) = levinson(r, LPC_ORDER)
|
||||
let gain = max(err, 0).squareRoot()
|
||||
let useF0 = f0 > 0 ? f0 : (lastF0 > 0 ? lastF0 : 0)
|
||||
lastF0 = f0
|
||||
for i in 0..<HOP {
|
||||
let idx = pos + i; if idx >= x.count { break }
|
||||
var e = 0.0
|
||||
if useF0 > 0 { // voiced: glottal impulse train
|
||||
phase += useF0/Double(sr)
|
||||
if phase >= 1.0 { phase -= 1.0; e = sqrt(Double(sr)/useF0) } // energy-normalized impulse
|
||||
} else { // unvoiced: noise
|
||||
e = Double.random(in: -1...1)
|
||||
}
|
||||
var y = gain * e
|
||||
for k in 1...LPC_ORDER { y -= a[k]*state[k-1] }
|
||||
for k in stride(from: LPC_ORDER-1, through: 1, by: -1) { state[k] = state[k-1] }
|
||||
state[0] = y
|
||||
out[idx] = y
|
||||
}
|
||||
pos += HOP
|
||||
}
|
||||
// normalize to peak 0.9
|
||||
let peak = out.map { abs($0) }.max() ?? 1
|
||||
let scale = peak > 1e-9 ? 0.9/peak : 1
|
||||
let samples = out.map { Int16(max(-32767, min(32767, $0*scale*32767))) }
|
||||
writeWav(URL(fileURLWithPath: outPath), samples: samples, sampleRate: sr)
|
||||
let vp = voiceprint(inPath)
|
||||
disclose(String(format: "IMITATE: rebuilt the voice from its signature (F0 %.0fHz, formants %@) -> %@",
|
||||
vp.f0, vp.formants.map{String(format:"%.0f",$0.0)}.joined(separator:"/"), outPath))
|
||||
emit(["ok": true, "op": "imitate", "in": inPath, "out": outPath,
|
||||
"f0_hz": vp.f0, "f0_range": [vp.f0lo, vp.f0hi],
|
||||
"formants_hz": vp.formants.map { $0.0 },
|
||||
"method": "LPC analysis-resynthesis (own-core, no training, no stolen voice)"])
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CONVERSE (full-duplex) — the interruptible conversational loop.
|
||||
// The utterance is a persistent, ordered meaning-plan of SEGMENTS, each with
|
||||
// a salience. The speaker plays them; the mic listens concurrently. On user
|
||||
// speech: pause INSTANTLY, classify (backchannel vs barge-in), then DECIDE
|
||||
// yield-or-hold from the salience of the current segment + the social read.
|
||||
// Yielded utterances persist their remaining plan so Neuron can RESUME.
|
||||
// ============================================================================
|
||||
struct Segment { let file: String; let salience: Double; let text: String }
|
||||
|
||||
enum Decision { case backchannelContinue, hold, yield }
|
||||
|
||||
// The yield-or-hold DECISION — grounded, contextual. Not a fixed rule.
|
||||
func decide(currentSalience: Double, progress: Double,
|
||||
interrupterAuthority: Double, isBackchannel: Bool) -> Decision {
|
||||
if isBackchannel { return .backchannelContinue } // "mm-hm" => keep going
|
||||
// Holding the floor is justified when what I'm saying matters AND I'm nearly
|
||||
// done (cheap to finish) AND the interrupter isn't high-priority.
|
||||
let holdScore = currentSalience * 0.6 + progress * 0.4
|
||||
if holdScore >= 0.6 && interrupterAuthority < 0.8 { return .hold }
|
||||
return .yield // default: be polite, let them in
|
||||
}
|
||||
|
||||
final class Conversation {
|
||||
let engine = AVAudioEngine()
|
||||
let player = AVAudioPlayerNode()
|
||||
var micLive = false
|
||||
// VAD state (shared with the audio tap thread)
|
||||
let lock = NSLock()
|
||||
var micRMS: Float = 0
|
||||
var speechFrames = 0 // consecutive above-threshold frames
|
||||
var onsetHandled = false
|
||||
|
||||
let resumePath: String
|
||||
init(resumePath: String) { self.resumePath = resumePath }
|
||||
|
||||
// Try to bring the mic up as a live VAD. Returns false if unavailable/denied.
|
||||
func startMic() -> Bool {
|
||||
let status = AVCaptureDevice.authorizationStatus(for: .audio)
|
||||
if Consent.load()["mic"] != true || status != .authorized {
|
||||
disclose("CONVERSE: live mic not available (consent/OS) — using injected barge events for the proof.")
|
||||
return false
|
||||
}
|
||||
let input = engine.inputNode
|
||||
// Acoustic echo cancellation: the OS voice-processing unit subtracts our
|
||||
// own speaker output from the mic so Neuron does NOT hear itself and
|
||||
// barge in on its own voice. This is what makes real-room barge-in work.
|
||||
do { try input.setVoiceProcessingEnabled(true); disclose("CONVERSE: AEC on (echo-cancelled mic — won't self-interrupt).") }
|
||||
catch { disclose("CONVERSE: AEC unavailable (\(error)); raising VAD floor instead.") }
|
||||
let fmt = input.inputFormat(forBus: 0)
|
||||
if fmt.sampleRate == 0 { return false }
|
||||
input.installTap(onBus: 0, bufferSize: 1024, format: fmt) { [weak self] buf, _ in
|
||||
guard let self = self, let ch = buf.floatChannelData?[0] else { return }
|
||||
let n = Int(buf.frameLength)
|
||||
var sum: Float = 0
|
||||
for i in 0..<n { let v = ch[i]; sum += v*v }
|
||||
let rms = n > 0 ? (sum / Float(n)).squareRoot() : 0
|
||||
self.lock.lock(); self.micRMS = rms; self.lock.unlock()
|
||||
}
|
||||
micLive = true
|
||||
disclose("CONVERSE: full-duplex — mic listening WHILE speaking (barge-in armed).")
|
||||
return true
|
||||
}
|
||||
|
||||
func run(_ segs: [Segment], interrupterAuthority: Double,
|
||||
injectBargeAt: Double?, injectKind: String, startIndex: Int, liveMic: Bool) {
|
||||
engine.attach(player)
|
||||
let firstFmt = (try? AVAudioFile(forReading: URL(fileURLWithPath: segs[startIndex].file)))?.processingFormat
|
||||
?? AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1)!
|
||||
engine.connect(player, to: engine.mainMixerNode, format: firstFmt)
|
||||
if liveMic { _ = startMic() }
|
||||
else { disclose("CONVERSE: deterministic mode (live mic off) — barge events \(injectBargeAt != nil ? "injected" : "none").") }
|
||||
do { try engine.start() } catch { die("audio engine failed to start: \(error)") }
|
||||
player.play()
|
||||
|
||||
let injectDeadline = injectBargeAt.map { Date().addingTimeInterval($0) }
|
||||
var injectedFired = false
|
||||
var idx = startIndex
|
||||
|
||||
segmentLoop: while idx < segs.count {
|
||||
let seg = segs[idx]
|
||||
guard let f = try? AVAudioFile(forReading: URL(fileURLWithPath: seg.file)) else {
|
||||
disclose("CONVERSE: missing segment '\(seg.file)', skipping."); idx += 1; continue
|
||||
}
|
||||
let dur = Double(f.length) / f.processingFormat.sampleRate
|
||||
disclose(String(format: "CONVERSE: speaking segment %d/%d (salience %.2f) — \"%@\"",
|
||||
idx+1, segs.count, seg.salience, seg.text))
|
||||
emit(["op": "converse", "event": "speaking", "segment": idx,
|
||||
"salience": seg.salience, "text": seg.text])
|
||||
|
||||
let done = DispatchSemaphore(value: 0)
|
||||
// .dataPlayedBack: completion fires only after the audio has actually
|
||||
// played OUT the DAC (not merely been consumed) — so the tail is never
|
||||
// clipped and playback always runs the FULL file length.
|
||||
player.scheduleFile(f, at: nil, completionCallbackType: .dataPlayedBack) { _ in done.signal() }
|
||||
player.play()
|
||||
|
||||
// Monitor this segment: poll VAD / injected event until it finishes.
|
||||
let segStart = Date()
|
||||
while done.wait(timeout: .now() + 0.02) == .timedOut {
|
||||
let elapsed = Date().timeIntervalSince(segStart)
|
||||
let progress = min(elapsed / max(dur, 0.001), 1.0)
|
||||
|
||||
// --- detect an onset (live mic OR injected) ---
|
||||
var onset = false
|
||||
if micLive {
|
||||
lock.lock(); let rms = micRMS; lock.unlock()
|
||||
if rms > 0.02 { speechFrames += 1 } else { speechFrames = 0 }
|
||||
if speechFrames >= 3 && !onsetHandled { onset = true } // ~60ms of voice
|
||||
}
|
||||
if let dl = injectDeadline, !injectedFired, Date() >= dl, !onsetHandled { onset = true; injectedFired = true }
|
||||
|
||||
if onset {
|
||||
onsetHandled = true
|
||||
// (1) BARGE-IN: pause INSTANTLY, on the spot.
|
||||
player.pause()
|
||||
let tBarge = Date().timeIntervalSince(segStart)
|
||||
disclose(String(format: "CONVERSE: << user speech at %.2fs into segment %d — PAUSED instantly >>", tBarge, idx+1))
|
||||
emit(["op": "converse", "event": "barge_in", "segment": idx,
|
||||
"at_seconds": tBarge, "progress": progress])
|
||||
|
||||
// (2) classify backchannel vs real barge-in
|
||||
let isBackchannel = classifyBackchannel(injected: injectDeadline != nil,
|
||||
kind: injectKind)
|
||||
let d = decide(currentSalience: seg.salience, progress: progress,
|
||||
interrupterAuthority: interrupterAuthority,
|
||||
isBackchannel: isBackchannel)
|
||||
switch d {
|
||||
case .backchannelContinue:
|
||||
disclose("CONVERSE: read as BACKCHANNEL (\"mm-hm\") — keep going, resume seamlessly.")
|
||||
emit(["op": "converse", "event": "backchannel_continue", "segment": idx])
|
||||
onsetHandled = false; speechFrames = 0
|
||||
player.play() // seamless resume
|
||||
case .hold:
|
||||
disclose("CONVERSE: HOLD the floor — \"hang on, let me finish this thought.\" (high salience, nearly done)")
|
||||
emit(["op": "converse", "event": "hold_floor", "segment": idx,
|
||||
"salience": seg.salience, "progress": progress])
|
||||
onsetHandled = false; speechFrames = 0
|
||||
player.play() // finish the segment, THEN yield
|
||||
// after this segment completes we yield the remainder
|
||||
_ = done.wait(timeout: .now() + dur + 1.0)
|
||||
persistResume(segs: segs, from: idx + 1, reason: "held-then-yield")
|
||||
finish(); return
|
||||
case .yield:
|
||||
disclose("CONVERSE: YIELD — stop, let them in. Remembering where I was (resumable).")
|
||||
player.stop()
|
||||
persistResume(segs: segs, from: idx, reason: "yield")
|
||||
emit(["op": "converse", "event": "yield", "interrupted_segment": idx,
|
||||
"resume_from": idx])
|
||||
finish(); return
|
||||
}
|
||||
}
|
||||
}
|
||||
emit(["op": "converse", "event": "segment_done", "segment": idx])
|
||||
idx += 1
|
||||
}
|
||||
// whole utterance completed uninterrupted
|
||||
clearResume()
|
||||
disclose("CONVERSE: utterance complete (uninterrupted).")
|
||||
emit(["ok": true, "op": "converse", "event": "complete", "segments": segs.count])
|
||||
finish()
|
||||
}
|
||||
|
||||
// A backchannel is brief/low. Injected kind lets us prove both paths headlessly;
|
||||
// the live path would measure post-onset duration & energy.
|
||||
func classifyBackchannel(injected: Bool, kind: String) -> Bool {
|
||||
if injected { return kind == "backchannel" }
|
||||
// live: sample ~250ms after onset; if speech already died away, it was a backchannel
|
||||
Thread.sleep(forTimeInterval: 0.25)
|
||||
lock.lock(); let rms = micRMS; lock.unlock()
|
||||
return rms < 0.015
|
||||
}
|
||||
|
||||
func persistResume(segs: [Segment], from: Int, reason: String) {
|
||||
let remaining = segs[from...].map { ["file": $0.file, "salience": $0.salience, "text": $0.text] as [String: Any] }
|
||||
let state: [String: Any] = ["resume_from": from, "reason": reason,
|
||||
"remaining": remaining, "ts": Date().timeIntervalSince1970]
|
||||
if let d = try? JSONSerialization.data(withJSONObject: state, options: [.prettyPrinted]) {
|
||||
try? d.write(to: URL(fileURLWithPath: resumePath))
|
||||
}
|
||||
disclose("CONVERSE: meaning-plan persisted (\(remaining.count) segments remain) — Neuron can resume the thread.")
|
||||
}
|
||||
func clearResume() { try? FileManager.default.removeItem(atPath: resumePath) }
|
||||
func finish() { player.stop(); if micLive { engine.inputNode.removeTap(onBus: 0) }; engine.stop() }
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// CLI
|
||||
// ----------------------------------------------------------------------------
|
||||
func loadManifest(_ path: String) -> (segs: [Segment], utterance: String) {
|
||||
guard let d = FileManager.default.contents(atPath: path),
|
||||
let o = try? JSONSerialization.jsonObject(with: d) as? [String: Any],
|
||||
let arr = o["segments"] as? [[String: Any]] else { die("bad manifest: \(path)") }
|
||||
let segs = arr.map { Segment(file: $0["file"] as? String ?? "",
|
||||
salience: ($0["salience"] as? NSNumber)?.doubleValue ?? 0.5,
|
||||
text: $0["text"] as? String ?? "") }
|
||||
return (segs, o["utterance"] as? String ?? "")
|
||||
}
|
||||
|
||||
let args = CommandLine.arguments
|
||||
guard args.count >= 2 else {
|
||||
print("""
|
||||
periph — Neuron peripheral I/O (own-core, local, consent-gated)
|
||||
grant <camera|mic> grant a sensitive sense (Neuron-level consent)
|
||||
revoke <camera|mic> revoke it
|
||||
status show consent state
|
||||
speak <file.wav> SPEAK ALOUD (efferent) via the speaker
|
||||
tone <out.wav> [hz] [sec] own-core synth a test WAV (no deps)
|
||||
listen <sec> <out.wav> MIC capture (afferent) 16k mono
|
||||
see <out.jpg> CAMERA one frame (afferent)
|
||||
feat-audio <file.wav> extract compact voice/sound signature (for ingest)
|
||||
feat-image <file.jpg> extract compact scene-geometry (for ingest)
|
||||
ingest-audio <file.wav> <engramURL> capture -> descriptor -> engram node (geometry)
|
||||
ingest-image <file.jpg> <engramURL> capture -> descriptor -> engram node (geometry)
|
||||
voiceprint <voice.wav> extract voice-signature (F0 + formants F1-F5)
|
||||
imitate <voice.wav> <out.wav> speak back in that voice (LPC analysis-resynthesis)
|
||||
hear-imitate <sec> <out.wav> MIC -> extract signature -> imitate -> SPEAK ALOUD
|
||||
wav-info <file.wav> print WAV geometry
|
||||
converse <manifest.json> [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume]
|
||||
full-duplex interruptible utterance
|
||||
""")
|
||||
exit(0)
|
||||
}
|
||||
|
||||
switch args[1] {
|
||||
case "grant":
|
||||
guard args.count >= 3 else { die("grant needs a device") }
|
||||
var g = Consent.load(); g[args[2]] = true; Consent.save(g)
|
||||
disclose("granted '\(args[2])' — the user consents; raw stream stays local, never egresses.")
|
||||
emit(["ok": true, "op": "grant", "device": args[2], "consent": g])
|
||||
case "revoke":
|
||||
guard args.count >= 3 else { die("revoke needs a device") }
|
||||
var g = Consent.load(); g[args[2]] = false; Consent.save(g)
|
||||
emit(["ok": true, "op": "revoke", "device": args[2], "consent": g])
|
||||
case "status":
|
||||
emit(["ok": true, "op": "status", "consent": Consent.load()])
|
||||
case "speak":
|
||||
guard args.count >= 3 else { die("speak needs a wav") }
|
||||
speak(args[2])
|
||||
case "tone":
|
||||
guard args.count >= 3 else { die("tone needs an out path") }
|
||||
let hz = args.count >= 4 ? Double(args[3]) ?? 220 : 220
|
||||
let sec = args.count >= 5 ? Double(args[4]) ?? 1.0 : 1.0
|
||||
let sr = 16000
|
||||
var s = [Int16](); s.reserveCapacity(Int(Double(sr)*sec))
|
||||
for i in 0..<Int(Double(sr)*sec) {
|
||||
let t = Double(i)/Double(sr)
|
||||
let env = min(1.0, min(t*20, (sec - t)*20)) // gentle attack/release
|
||||
s.append(Int16(env * 0.3 * 32767 * sin(2*Double.pi*hz*t)))
|
||||
}
|
||||
writeWav(URL(fileURLWithPath: args[2]), samples: s, sampleRate: sr)
|
||||
disclose("tone: wrote own-core \(sec)s @ \(hz)Hz WAV to \(args[2]).")
|
||||
emit(["ok": true, "op": "tone", "file": args[2], "hz": hz, "seconds": sec])
|
||||
case "listen":
|
||||
guard args.count >= 4 else { die("listen needs <sec> <out.wav>") }
|
||||
listen(seconds: Double(args[2]) ?? 3.0, out: args[3])
|
||||
case "see":
|
||||
guard args.count >= 3 else { die("see needs an out path") }
|
||||
see(out: args[2])
|
||||
case "feat-audio":
|
||||
guard args.count >= 3 else { die("feat-audio needs a wav") }
|
||||
featAudio(args[2])
|
||||
case "feat-image":
|
||||
guard args.count >= 3 else { die("feat-image needs an image") }
|
||||
featImage(args[2])
|
||||
case "ingest-audio":
|
||||
guard args.count >= 4 else { die("ingest-audio needs <wav> <engramURL>") }
|
||||
ingest(args[2], kind: "audio", engramURL: args[3])
|
||||
case "ingest-image":
|
||||
guard args.count >= 4 else { die("ingest-image needs <image> <engramURL>") }
|
||||
ingest(args[2], kind: "image", engramURL: args[3])
|
||||
case "voiceprint":
|
||||
guard args.count >= 3 else { die("voiceprint needs a wav") }
|
||||
let vp = voiceprint(args[2])
|
||||
disclose("VOICEPRINT: \(vp.content)")
|
||||
emit(["ok": true, "op": "voiceprint", "file": args[2], "f0_hz": vp.f0,
|
||||
"f0_range": [vp.f0lo, vp.f0hi], "formants_hz": vp.formants.map { $0.0 },
|
||||
"bandwidths_hz": vp.formants.map { $0.1 }, "content": vp.content,
|
||||
"ingest": ["node_type": "Observation", "tier": "Episodic", "content": vp.content]])
|
||||
case "imitate":
|
||||
guard args.count >= 4 else { die("imitate needs <voice.wav> <out.wav>") }
|
||||
imitate(inPath: args[2], outPath: args[3])
|
||||
case "hear-imitate":
|
||||
guard args.count >= 4 else { die("hear-imitate needs <sec> <out.wav>") }
|
||||
let secs = Double(args[2]) ?? 4.0
|
||||
let outp = args[3]
|
||||
let capp = outp.replacingOccurrences(of: ".wav", with: "") + ".heard.wav"
|
||||
disclose("HEAR-IMITATE: open the ear, listen \(secs)s, grab the voice, speak it back.")
|
||||
listen(seconds: secs, out: capp) // afferent: hear the voice
|
||||
imitate(inPath: capp, outPath: outp) // extract signature + resynthesize
|
||||
speak(outp) // efferent: speak back ALOUD in that voice
|
||||
case "wav-info":
|
||||
guard args.count >= 3, let i = wavInfo(args[2]) else { die("wav-info needs a readable wav") }
|
||||
disclose("WAV \(args[2]): \(i.sampleRate)Hz \(i.channels)ch \(i.bits)bit \(i.frames) frames")
|
||||
emit(["ok": true, "op": "wav-info", "sample_rate": i.sampleRate, "channels": i.channels,
|
||||
"bits": i.bits, "frames": i.frames,
|
||||
"seconds": Double(i.frames)/Double(max(i.sampleRate,1))])
|
||||
case "converse":
|
||||
guard args.count >= 3 else { die("converse needs a manifest") }
|
||||
let (segs, utter) = loadManifest(args[2])
|
||||
var authority = 0.5
|
||||
var bargeAt: Double? = nil
|
||||
var bargeKind = "bargein"
|
||||
var resume = false
|
||||
var liveMic = false
|
||||
var i = 3
|
||||
while i < args.count {
|
||||
switch args[i] {
|
||||
case "--authority": if i+1 < args.count { authority = Double(args[i+1]) ?? 0.5; i += 1 }
|
||||
case "--barge-at":
|
||||
if i+1 < args.count {
|
||||
let parts = args[i+1].split(separator: ":")
|
||||
bargeAt = Double(parts[0]) ?? nil
|
||||
if parts.count > 1 { bargeKind = String(parts[1]) }
|
||||
i += 1
|
||||
}
|
||||
case "--resume": resume = true
|
||||
case "--live-mic": liveMic = true
|
||||
default: break
|
||||
}
|
||||
i += 1
|
||||
}
|
||||
let resumePath = (ProcessInfo.processInfo.environment["PERIPH_HOME"]
|
||||
?? FileManager.default.currentDirectoryPath + "/peripheral") + "/.resume.json"
|
||||
var startIndex = 0
|
||||
var runSegs = segs
|
||||
if resume, let d = FileManager.default.contents(atPath: resumePath),
|
||||
let o = try? JSONSerialization.jsonObject(with: d) as? [String: Any],
|
||||
let rem = o["remaining"] as? [[String: Any]] {
|
||||
runSegs = rem.map { Segment(file: $0["file"] as? String ?? "",
|
||||
salience: ($0["salience"] as? NSNumber)?.doubleValue ?? 0.5,
|
||||
text: $0["text"] as? String ?? "") }
|
||||
startIndex = 0
|
||||
disclose("CONVERSE: resuming — \"as I was saying...\" (\(runSegs.count) segments left).")
|
||||
emit(["op": "converse", "event": "resume", "remaining": runSegs.count])
|
||||
}
|
||||
if runSegs.isEmpty { die("no segments to speak") }
|
||||
disclose("CONVERSE: utterance = \"\(utter)\" (\(runSegs.count) segments).")
|
||||
let convo = Conversation(resumePath: resumePath)
|
||||
convo.run(runSegs, interrupterAuthority: authority,
|
||||
injectBargeAt: bargeAt, injectKind: bargeKind, startIndex: startIndex, liveMic: liveMic)
|
||||
default:
|
||||
die("unknown command: \(args[1])")
|
||||
}
|
||||
Reference in New Issue
Block a user