diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2b63ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ + +# organ: local device state and its own engram store — never production's +peripheral/.consent.json +peripheral/.resume.json +peripheral/.engram/ +peripheral/organ diff --git a/lang/runtime/el_audio_darwin.m b/lang/runtime/el_audio_darwin.m index ff73964..da43a64 100644 --- a/lang/runtime/el_audio_darwin.m +++ b/lang/runtime/el_audio_darwin.m @@ -314,6 +314,25 @@ el_val_t speaker_played_frames(void) { return (el_val_t)g_aq_state->pos; } +/* Pause where we are, keeping the queue and its position intact. + * + * This is the difference between barge-in and "finish the buffer". The moment + * the microphone hears speech, output must stop AT THAT SAMPLE — a listener + * experiences even 200ms of continued talking as being talked over. Pause + * rather than stop because the interruption might turn out to be a backchannel + * ("mm-hm"), and the right response to a backchannel is to carry on as though + * nothing happened, which requires the queue to still be exactly where it was. + * A stop-and-restart would re-attack the buffer and be audible as a stutter. */ +el_val_t speaker_pause(void) { + if (!g_aq) return (el_val_t)0; + return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0); +} + +el_val_t speaker_resume(void) { + if (!g_aq) return (el_val_t)0; + return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0); +} + el_val_t speaker_stop(void) { if (!g_aq) return (el_val_t)0; /* immediate: do NOT let the queue finish what it is holding */ @@ -322,6 +341,131 @@ el_val_t speaker_stop(void) { return (el_val_t)1; } +/* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer. + * Returns frames, or 0 on any failure; *out is set only on success. Shared by + * the blocking and async WAV paths. */ +static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) { + if (!path || !out) return 0; + FILE* f = fopen(path, "rb"); + if (!f) return 0; + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; } + long size = ftell(f); + if (size <= 44) { fclose(f); return 0; } + rewind(f); + unsigned char* d = (unsigned char*)malloc((size_t)size); + if (!d) { fclose(f); return 0; } + size_t got = fread(d, 1, (size_t)size, f); + fclose(f); + if (got != (size_t)size) { free(d); return 0; } + if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; } + + int32_t sr = 0, channels = 0, bits = 0; + long dataOff = -1, dataLen = 0, o = 12; + /* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders + * routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads + * padding as audio. */ + while (o + 8 <= size) { + long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); + if (sz < 0) break; + if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { + channels = (int32_t)(d[o+10] | (d[o+11] << 8)); + sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); + bits = (int32_t)(d[o+22] | (d[o+23] << 8)); + } else if (memcmp(d + o, "data", 4) == 0) { + dataOff = o + 8; + dataLen = sz; + if (dataOff + dataLen > size) dataLen = size - dataOff; + } + o += 8 + sz + (sz & 1); + } + if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; } + + long frames = dataLen / (2 * channels); + int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); + if (!pcm) { free(d); return 0; } + for (long i = 0; i < frames; i++) { + long b = dataOff + i * 2 * channels; + pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); + } + free(d); + *out = pcm; + if (out_sr) *out_sr = sr; + return (int64_t)frames; +} + +/* Async WAV playback. converse speaks PRE-RENDERED segments and must keep + * listening while it does, so it needs the file on the queue without blocking + * and needs to be able to stop it mid-buffer. Going through the file rather + * than an El [Int] also avoids marshalling a million-element list per segment + * for audio the caller never intends to look at. */ +el_val_t speaker_play_wav_async(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p) return (el_val_t)0; + + el_audio_teardown(); + + int32_t sr = 0; + int16_t* pcm = NULL; + int64_t frames = el_wav_load(p, &pcm, &sr); + if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; } + + g_aq_pcm = pcm; + g_aq_sr = sr; + g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); + if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } + g_aq_state->pcm = g_aq_pcm; + g_aq_state->frames = frames; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sr; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { + el_audio_teardown(); + return (el_val_t)0; + } + for (int i = 0; i < EL_AQ_NBUF; i++) { + int64_t remain = g_aq_state->frames - g_aq_state->pos; + if (remain <= 0) break; + AudioQueueBufferRef b = NULL; + if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; + int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); + b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); + g_aq_state->pos += k; + if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; + g_aq_state->inflight++; + } + if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } + if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } + return (el_val_t)1; +} + +/* Total frames and sample rate of a WAV, without playing it — wav-info, and the + * duration converse needs to compute progress through a segment. */ +el_val_t wav_frames(el_val_t path) { + const char* p = EL_CSTR(path); + int16_t* pcm = NULL; int32_t sr = 0; + int64_t n = el_wav_load(p, &pcm, &sr); + free(pcm); + return (el_val_t)n; +} + +el_val_t wav_rate(el_val_t path) { + const char* p = EL_CSTR(path); + int16_t* pcm = NULL; int32_t sr = 0; + int64_t n = el_wav_load(p, &pcm, &sr); + free(pcm); + return (el_val_t)(n > 0 ? sr : 0); +} + /* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows * how to write a WAV and a caller may reasonably want to hear one back without * re-rendering it; the parse is deliberately minimal and chunk-walking, so the diff --git a/lang/runtime/el_peripheral_null.c b/lang/runtime/el_peripheral_null.c index ff9b4e2..6e333c8 100644 --- a/lang/runtime/el_peripheral_null.c +++ b/lang/runtime/el_peripheral_null.c @@ -42,10 +42,23 @@ el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { return (el_val_t)0; } +el_val_t speaker_play_wav_async(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +el_val_t speaker_pause(void) { return (el_val_t)0; } +el_val_t speaker_resume(void) { return (el_val_t)0; } el_val_t speaker_playing(void) { return (el_val_t)0; } el_val_t speaker_stop(void) { return (el_val_t)0; } el_val_t speaker_played_frames(void) { return (el_val_t)0; } +/* WAV geometry is pure parsing and would work fine here, but reporting a + * duration for audio this build cannot play would invite a caller to sequence + * around a silence. Refuse consistently with the rest of the file. */ +el_val_t wav_frames(el_val_t path) { (void)path; return (el_val_t)0; } +el_val_t wav_rate(el_val_t path) { (void)path; return (el_val_t)0; } + /* ── Microphone ──────────────────────────────────────────────────────────── */ el_val_t mic_available(void) { return (el_val_t)0; } diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 165aa6b..b0a41c3 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -126,10 +126,18 @@ el_val_t speaker_play_wav(el_val_t path); /* 16-bit mono RIFF/WAVE; 1 o * the hardware is still going; speaker_played_frames() is how far it actually * got, which is what makes an interrupted utterance resumable at the sample. */ el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate); +el_val_t speaker_play_wav_async(el_val_t path); +el_val_t speaker_pause(void); /* stop AT THIS SAMPLE, keep position */ +el_val_t speaker_resume(void); /* carry on from exactly there */ el_val_t speaker_playing(void); el_val_t speaker_stop(void); el_val_t speaker_played_frames(void); +/* WAV geometry without playing — wav-info, and the segment duration converse + * needs to turn elapsed time into progress. */ +el_val_t wav_frames(el_val_t path); +el_val_t wav_rate(el_val_t path); + /* Microphone (afferent). Fails CLOSED: returns 0 unless the OS has granted * capture access. mic_capture_pcm16 blocks for `seconds` and returns an [Int] * of 16-bit mono samples at `sample_rate` — the raw stream is handed to El and diff --git a/peripheral/README.md b/peripheral/README.md index 765fd35..5f1d0b4 100644 --- a/peripheral/README.md +++ b/peripheral/README.md @@ -1,80 +1,184 @@ -# peripheral — Neuron's I/O organ (own-core, local, consent-gated) +# peripheral — Neuron's I/O organ, in El -The interface made physical. Two afferent senses in, one efferent voice out — -all reached the way the agentic surface reaches any tool. +**El speaks.** The engram stores geometry and does not speak; the speaking +belongs to the language and its runtime. + +Until this landed, the organ was a 939-line Swift program (`src/periph.swift`) +that shelled out to `afplay`. Neuron's mouth and ears were a separate binary +standing next to the language, and "speak" meant "ask that binary to speak." +That program is now **reference material, not the implementation.** ``` -MIC (hear) afferent device -> capture -> descriptor -> ingest -> geometry -CAMERA (see) afferent device -> capture -> descriptor -> ingest -> scene-geometry -SPEAKER(speak) efferent render WAV -> PLAY ALOUD out the speaker +SPEAKER (speak) efferent samples ──────────────► CoreAudio ──► the room +MIC (hear) afferent device ──► samples ──► descriptor ──► engram +CAMERA (see) afferent device ──► frame ──► descriptor ──► engram ``` -Closes the conversational loop: **hear (mic) -> understand (engram) -> speak (speaker)**. +## The split, and why it falls where it does + +Exactly **two** things here are not El, and they are the two things El cannot +express as arithmetic: + +| Not El (realizers) | Why | +|---|---| +| `lang/runtime/el_audio_darwin.m` | Handing a buffer to the DAC and waiting for it to drain. There is no way to say "the hardware has now played these samples" in El, and there should not be. | +| `lang/runtime/el_capture_darwin.m` | Asking the OS for samples off a microphone or frames off a camera, plus the TCC permission dance. | + +**Everything else is El**, because everything else is arithmetic: + +| In El | Where | +|---|---| +| WAV encode / decode (chunk-walking, JUNK/FLLR tolerant) | `src/organ_dsp.el`, `elp/src/speech.el` | +| LPC autocorrelation + Levinson-Durbin (order 16 @ 16 kHz) | `src/organ_dsp.el` | +| Formant extraction off the all-pole spectral envelope | `src/organ_dsp.el` | +| Source-filter resynthesis (glottal impulse train through the filter) | `src/organ_dsp.el` | +| Audio descriptor `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` | `src/organ_dsp.el` | +| Voice descriptor `[F0, F1..F5, bandwidths]` | `src/organ_dsp.el` | +| Scene descriptor `[w, h, meanRGB, brightness, 3×3 luminance grid]` | `src/organ.el` | +| Consent, disclosure, the voice-from-engram fetch | `src/organ.el` | +| Barge-in, yield-or-hold, backchannel, resume | `src/organ_converse.el` | +| The command surface | `src/organ_cli.el` | + +Both realizers are their **own translation units**, declared in +`lang/runtime/el_runtime.h`, and deliberately **not** patches to +`el_runtime.c`. Acquiring a device must not mean editing the middle of the +language — the same rule the realizer registry follows for modalities. +`lang/runtime/el_peripheral_null.c` provides the identical entry points +everywhere else, so El that speaks links on any platform and truthfully reports +having no speaker rather than going quietly silent. + +## The voice comes from the engram + +A voice is **geometry in the engram**, not a JSON file next to the code and +certainly not constants in a source file. The organ fetches it the way anything +retrieves a memory — it asks: + +```el +let g: [Int] = organ_voice_fetch("will") +// [peripheral] VOICE: fetched 'will' FROM THE ENGRAM — +// f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531 +``` + +`organ_voice_fetch` issues an engram query and reads the geometry off the node +that comes back. Nothing opens a file. If the region is not in the graph it +returns **empty**, not a plausible default — a caller has to be able to tell +"this is how they sound" from "I never heard them." + +The reverse direction is `ingest-voice`: an LPC voiceprint becomes a node, and +from then on the voice is a memory rather than a measurement someone wrote down. + +## What the organ never does + +**It never learns a word.** Pronunciation, vocabulary and phonemes belong to the +language faculty and are already built as ingested geometry — *the engram knows +how to pronounce*. The seam is `synth_codes(codes, voice, pmap)`: the codes and +the phoneme map arrive from the language side as geometry, and the organ's whole +job is turning them into samples and getting the samples out the speaker, plus +the same trip in reverse for the senses. There is no lexicon here and no +grapheme-to-phoneme rule, by design. ## Rails -- **Own-core.** macOS-native only: AVFoundation (camera/mic), CoreAudio voice- - processing (AEC), afplay (speaker), ImageIO/CoreGraphics (frames), hand-rolled - DSP (WAV, LPC, formant synthesis). No cloud, no heavy deps. -- **Local-only.** Raw streams are written to `out/` and never egress. `.gitignore` - keeps captured media out of git. -- **Consent-gated (two locks).** A Neuron-level grant (`grant`/`revoke`) *and* the - OS TCC permission. Sensitive senses (camera/mic) fail closed without both. -- **Disclosed.** Every device touch prints a `[peripheral]` line on stderr. + +- **Own-core.** CoreAudio / AVFoundation / ImageIO — all ship with macOS. No + cloud, no model, no heavy dependency. There is **no network code in the organ + at all**, by construction. +- **Local-only.** Raw streams stay on the machine. What leaves a capture is a + descriptor of a few dozen numbers. A 1920×1080 frame becomes 15 integers + (~414,000× smaller); three seconds of audio becomes 8. +- **Consent, two locks.** A Neuron-level grant **and** the OS TCC permission. + Camera and mic **fail closed** without both. The speaker is disclosed but not + gated — you cannot secretly speak aloud, and gating it would mean Neuron needs + permission to answer. +- **Disclosed.** Every device touch prints a `[peripheral]` line on **stderr** + (via `eprintln`, flushed immediately), so a disclosure lands before the device + is touched and never contaminates the program's stdout. ## Build + +```bash +./peripheral/build.sh /tmp/organ ``` -swiftc -O -o bin/periph src/periph.swift \ - -framework AVFoundation -framework CoreMedia -framework Foundation \ - -framework CoreGraphics -framework ImageIO -framework CoreImage -``` + +Concatenates the El modules, compiles with `elc`, links the two realizers. +Run it **from the repo root** or the `.psv` phoneme data will not resolve. ## Commands + ``` -periph grant|revoke # Neuron-level consent -periph status -periph speak # SPEAK ALOUD (efferent) -periph tone [hz] [sec] # own-core WAV synth -periph listen # MIC capture (afferent), 16k mono -periph see # CAMERA one frame (afferent) -periph feat-audio | feat-image # capture -> compact descriptor -periph ingest-audio|ingest-image # descriptor -> engram node (geometry) -periph voiceprint # extract F0 + formants F1-F5 -periph imitate # speak back in that voice (LPC resynthesis) -periph hear-imitate # MIC -> signature -> imitate -> SPEAK ALOUD -periph converse [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] [--live-mic] +organ grant|revoke Neuron-level consent +organ status consent + device state +organ speak play a WAV aloud (efferent) +organ tone [hz] [ms] synthesize and play — no file at all +organ say [CODE...] fetch voice FROM THE ENGRAM, render, speak +organ listen mic capture 16k mono (afferent) +organ see one camera frame (afferent) +organ wav-info WAV geometry +organ feat-audio compact audio descriptor (8 numbers) +organ feat-image compact scene-geometry from the camera +organ voiceprint F0 + formants F1-F5 (LPC) +organ imitate LPC analysis-resynthesis +organ hear-imitate mic -> signature -> imitate -> speak aloud +organ ingest-audio descriptor -> engram node (geometry) +organ ingest-voice voiceprint -> engram voice region +organ converse [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume] ``` -## The afferent metabolism -A capture is never shipped raw. It becomes a **compact descriptor** — the afferent -twin of the music instrument-signature: -- audio -> `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` (~2400-6000x smaller) -- image -> `[w, h, meanRGB, brightness, 3x3 luminance grid]` (~400000x smaller) -- voice -> `[F0, F1..F5, bandwidths]` (11 numbers) +## Interruptibility -That descriptor is what the ingest organ (engram `POST /api/nodes`) turns into an -embedded node = geometry. +`converse` speaks an ordered, salience-tagged **meaning-plan** while listening: -## Voice by imitation -`voiceprint`/`imitate` are own-core LPC (autocorrelation + Levinson-Durbin, order -16 @ 16 kHz), formant extraction from the LPC spectral envelope, and source-filter -resynthesis (glottal impulse train at F0 through the all-pole formant filter). A -voice is grabbed by ear as ~a dozen numbers and spoken back — **no training, no -stolen voice.** Measured fidelity on real speech: resynthesized formants match the -source within 2-3%. The full phoneme->formant path for *novel* sentences is the -speech faculty's seam (`elp` audio surface profile); this engine provides the -formant synthesis primitive it renders through. +- **barge-in** — output stops at the sample, not at the end of the buffer. The + realizer exposes `pause`/`resume` and reports `played_frames` (the real DAC + position) precisely so this is possible. +- **yield-or-hold** — a decision, not a rule: `hold = salience·0.6 + + progress·0.4`, and holding also requires that the interrupter not be + high-authority. Otherwise yield, because the polite default is the right one. +- **backchannel** — "mm-hm" is brief and low-energy; resume seamlessly. +- **resumable** — on yield the remaining plan persists to `.resume.json`; + `--resume` picks the thread back up. An interruption should cost a turn, not + the content. -## Interruptibility (native turn-taking) -`converse` plays the utterance as an ordered, salience-tagged **meaning-plan** -while the mic listens (full-duplex, AEC on so it never barges in on its own voice): -- **barge-in**: user speech -> pause on the spot (sample-accurate), not "finish the buffer." -- **yield-or-hold**: a decision grounded in the current segment's salience + progress - + the interrupter's authority — YIELD (stop) or HOLD ("hang on, let me finish"). -- **backchannel** ("mm-hm"): brief/low -> keep going, resume seamlessly. -- **resumable**: on yield the remaining plan persists (`.resume.json`); `--resume` - picks the thread back up ("as I was saying"). +Live full-duplex uses `--live-mic` with the OS voice-processing unit (AEC) so +Neuron does not barge in on its own voice. `--barge-at` injects the event +deterministically for testing. -Live full-duplex uses `--live-mic` (OS AEC). Injected `--barge-at` drives the -decision loop deterministically for testing. -``` -``` +## Measured against the Swift original + +Same input (`out/mic_room.wav`, 16 kHz mono, 48121 samples), Swift `periph` +vs the El organ: + +| | Swift | El | +|---|---|---| +| seconds | 3.0075625 | 3.0076 | +| rms | 0.0047766496761 | 0.004777 | +| peak | 0.01806640625 | 0.018066 | +| zcr_hz | 416.28395087 | 416.2840 | +| centroid_hz | 727.60529169 | 727.6053 | +| f0_hz | 400 | 400.0000 | +| formants F1–F5 | 1734.375 / 3343.75 / 3875 / 4359.375 / 4468.75 | identical | +| bandwidths B1–B5 | 2000 / 2968.75 / 4203.125 / 4687.5 / 5000 | identical | + +Agreement to every printed digit. `imitate` cannot match bit-for-bit because the +Swift excites unvoiced frames with `Double.random` — two Swift runs correlate +0.957 with **each other**; El correlates **0.958** with Swift. The port is as +close to the original as the original is to itself, and the deterministic prefix +is bit-identical. + +## Honest status + +- **Works:** speaker (CoreAudio, no `afplay`, no subprocess — verified: zero + `afplay`/Swift strings in the binary, no child process during playback), mic + capture, camera capture, all descriptors, LPC voiceprint, imitate, + hear-imitate, voice fetch/ingest against the engram, converse (yield, hold, + yield-to-authority, backchannel, resume — all exercised with real audio). +- **Coarse, and labelled so:** a fetched voice is one formant triple with no + coarticulation and no prosody. It is an impression, explicitly **not a + clone**, and `prov=COARSE` says so on the node. +- **Not verified here:** live `--live-mic` barge-in in a real room with a real + interrupter. The AEC path is implemented and the deterministic path is proven; + the acoustic behaviour is not something a headless run can establish. +- **Not in the engram yet:** the structured `Voice` / `VowelTarget` geometry + nodes live in the organ's own store and in snapshot files from earlier work, + but the **production engram does not carry them**. Getting them there is an + ingest, not a code change. +- `src/periph.swift` is kept as the reference the port was measured against. diff --git a/peripheral/build.sh b/peripheral/build.sh new file mode 100755 index 0000000..4613088 --- /dev/null +++ b/peripheral/build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# build.sh — build the El organ. +# +# El has no import system on this path, so the modules are concatenated in +# dependency order (the same thing elp/tests/run.sh does) and handed to elc as +# one unit. The two device realizers are then linked in. +# +# MUST be run from the repo root, or the .psv phoneme geometry will not resolve +# and the render silently degrades. +set -uo pipefail + +OUT="${1:-./peripheral/organ}" +REPO="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# Dependency order. The elp modules supply the render (synth_codes) and the +# phoneme-geometry read; the organ supplies everything else. +cat elp/src/voice-profile.el \ + elp/src/accent.el \ + elp/src/voice-ingest.el \ + elp/src/speech-ingest.el \ + elp/src/speech.el \ + peripheral/src/organ.el \ + peripheral/src/organ_dsp.el \ + peripheral/src/organ_converse.el \ + peripheral/src/organ_cli.el \ + | grep -v '^import ' > "$WORK/organ.el" + +cd "$REPO/lang" +./dist/platform/elc "$WORK/organ.el" > "$WORK/organ.c" || { echo "elc failed" >&2; exit 1; } + +SSL_PREFIX="$(brew --prefix openssl@3 2>/dev/null || echo /usr/local)" + +# The peripheral realizers are per-platform: Darwin gets the real devices, +# anything else gets el_peripheral_null.c and honestly reports having none. +case "$(uname)" in + Darwin) + # The Objective-C realizers are compiled SEPARATELY, with -fobjc-arc. The + # capture realizer is written against ARC (it holds AVFoundation objects); + # compiling it MRR silently changes its memory semantics, which on a device + # path shows up as a use-after-free under load rather than as an error here. + cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_audio_darwin.m -o "$WORK/el_audio.o" || exit 1 + cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_capture_darwin.m -o "$WORK/el_capture.o" || exit 1 + PERIPH_SRC="$WORK/el_audio.o $WORK/el_capture.o" + PERIPH_LIBS="-framework AudioToolbox -framework AVFoundation -framework CoreMedia + -framework CoreVideo -framework CoreGraphics -framework ImageIO + -framework Foundation" + ;; + *) + PERIPH_SRC="runtime/el_peripheral_null.c" + PERIPH_LIBS="" + ;; +esac + +cc -O1 -I runtime -I"$SSL_PREFIX/include" -L"$SSL_PREFIX/lib" \ + -o "$OUT" "$WORK/organ.c" \ + runtime/el_runtime.c runtime/el_seed.c \ + runtime/engram_cognition.c runtime/engram_geometry.c runtime/engram_reason.c \ + runtime/engram_store.c runtime/engram_verify.c runtime/engram_vindex.c \ + runtime/eg_cosine_batch.c runtime/eg_cosine_batch_strategy_cpu.c \ + $PERIPH_SRC $PERIPH_LIBS \ + -lcurl -lssl -lcrypto -lpthread -lm || { echo "link failed" >&2; exit 1; } + +echo "built: $OUT" diff --git a/peripheral/src/organ.el b/peripheral/src/organ.el index 498369d..cf4af38 100644 --- a/peripheral/src/organ.el +++ b/peripheral/src/organ.el @@ -333,12 +333,204 @@ fn organ_voice_ingest(name: String, f0: Int, f0_end: Int, kf: Int, f1: Int, f2: // Turn the fetched geometry into the voice slot-map the render consumes. Kept // separate from the fetch so the organ never invents a voice: if the fetch came // back empty this returns empty too, and the caller has to deal with it. +// +// The slot-map is built here rather than by calling the render's own +// constructor, so the organ carries NO dependency on the language faculty's +// modules — it only has to agree with them about a wire format, which is the +// looser and more honest coupling. (The layout is the same key/value [String] +// convention lang_get / surface_get / voice_get all read.) fn organ_voice_profile(name: String, g: [Int]) -> [String] { - let empty: [String] = native_list_empty() + let r: [String] = native_list_empty() if native_list_len(g) < 6 { - return empty + return r } - return voice_new(name, native_list_get(g, 0), native_list_get(g, 1), native_list_get(g, 2), 1000, 1000, 8) + r = native_list_append(r, "name") + r = native_list_append(r, name) + r = native_list_append(r, "f0") + r = native_list_append(r, int_to_str(native_list_get(g, 0))) + r = native_list_append(r, "f0_end") + r = native_list_append(r, int_to_str(native_list_get(g, 1))) + r = native_list_append(r, "kf") + r = native_list_append(r, int_to_str(native_list_get(g, 2))) + r = native_list_append(r, "dur") + r = native_list_append(r, "1000") + r = native_list_append(r, "tilt") + r = native_list_append(r, "1000") + r = native_list_append(r, "breath") + r = native_list_append(r, "8") + return r +} + +// ── Scene geometry (afferent, camera) ──────────────────────────────────────── +// +// The image half of the afferent metabolism, and the same principle as the +// audio descriptor: a frame is never handed on raw. The realizer returns a +// small pixel grid; THIS computes the descriptor, in El, because averaging +// pixels is arithmetic and arithmetic is not a device concern. +// +// Returns 15 numbers — [w, h, meanR, meanG, meanB, brightness_pm, and a 3x3 +// luminance grid] — standing in for a multi-megapixel frame. The 3x3 grid is +// the smallest thing that still says WHERE the light is, which is most of what +// makes a scene comparable to another scene; a single brightness average would +// make a lamp on the left indistinguishable from a lamp on the right. +// +// Luminance is Rec. 601 (0.299R + 0.587G + 0.114B), in integer per-mille, so +// the descriptor is reproducible rather than subject to float drift. +fn organ_image_descriptor() -> [Int] { + let out: [Int] = native_list_empty() + let frame: Any = camera_capture_rgb() + if frame == 0 { + return out + } + let w: Int = el_map_get(frame, "width") + let h: Int = el_map_get(frame, "height") + let gw: Int = el_map_get(frame, "grid_w") + let gh: Int = el_map_get(frame, "grid_h") + let px: [Int] = el_map_get(frame, "pixels") + let np: Int = native_list_len(px) + if np < 3 { + return out + } + let count: Int = np / 3 + let rsum: Int = 0 + let gsum: Int = 0 + let bsum: Int = 0 + // 3x3 accumulators, row-major + let cell: [Int] = native_list_empty() + let cn: [Int] = native_list_empty() + let z: Int = 0 + while z < 9 { + cell = native_list_append(cell, 0) + cn = native_list_append(cn, 0) + z = z + 1 + } + // El has no list-set, so the cells are summed into parallel scalars and + // reassembled — nine explicit accumulators would be worse to read than one + // pass per cell over a grid this small. + let c0: Int = 0 + let c1: Int = 0 + let c2: Int = 0 + let c3: Int = 0 + let c4: Int = 0 + let c5: Int = 0 + let c6: Int = 0 + let c7: Int = 0 + let c8: Int = 0 + let n0: Int = 0 + let n1: Int = 0 + let n2: Int = 0 + let n3: Int = 0 + let n4: Int = 0 + let n5: Int = 0 + let n6: Int = 0 + let n7: Int = 0 + let n8: Int = 0 + let i: Int = 0 + while i < count { + let r: Int = native_list_get(px, i * 3) + let g: Int = native_list_get(px, i * 3 + 1) + let b: Int = native_list_get(px, i * 3 + 2) + rsum = rsum + r + gsum = gsum + g + bsum = bsum + b + let lum: Int = (299 * r + 587 * g + 114 * b) / 1000 + let x: Int = i - (i / gw) * gw + let y: Int = i / gw + let cx: Int = x * 3 / gw + let cy: Int = y * 3 / gh + if cx > 2 { + cx = 2 + } + if cy > 2 { + cy = 2 + } + let idx: Int = cy * 3 + cx + if idx == 0 { + c0 = c0 + lum + n0 = n0 + 1 + } + if idx == 1 { + c1 = c1 + lum + n1 = n1 + 1 + } + if idx == 2 { + c2 = c2 + lum + n2 = n2 + 1 + } + if idx == 3 { + c3 = c3 + lum + n3 = n3 + 1 + } + if idx == 4 { + c4 = c4 + lum + n4 = n4 + 1 + } + if idx == 5 { + c5 = c5 + lum + n5 = n5 + 1 + } + if idx == 6 { + c6 = c6 + lum + n6 = n6 + 1 + } + if idx == 7 { + c7 = c7 + lum + n7 = n7 + 1 + } + if idx == 8 { + c8 = c8 + lum + n8 = n8 + 1 + } + i = i + 1 + } + let rA: Int = rsum / count + let gA: Int = gsum / count + let bA: Int = bsum / count + let bright: Int = (299 * rA + 587 * gA + 114 * bA) / 255 + out = native_list_append(out, w) + out = native_list_append(out, h) + out = native_list_append(out, rA) + out = native_list_append(out, gA) + out = native_list_append(out, bA) + out = native_list_append(out, bright) + if n0 < 1 { + n0 = 1 + } + if n1 < 1 { + n1 = 1 + } + if n2 < 1 { + n2 = 1 + } + if n3 < 1 { + n3 = 1 + } + if n4 < 1 { + n4 = 1 + } + if n5 < 1 { + n5 = 1 + } + if n6 < 1 { + n6 = 1 + } + if n7 < 1 { + n7 = 1 + } + if n8 < 1 { + n8 = 1 + } + out = native_list_append(out, c0 / n0) + out = native_list_append(out, c1 / n1) + out = native_list_append(out, c2 / n2) + out = native_list_append(out, c3 / n3) + out = native_list_append(out, c4 / n4) + out = native_list_append(out, c5 / n5) + out = native_list_append(out, c6 / n6) + out = native_list_append(out, c7 / n7) + out = native_list_append(out, c8 / n8) + organ_disclose("FEAT(image): 15-number scene-geometry vs " + int_to_str(w * h * 3) + " pixel-channels — the descriptor travels, the frame does not.") + return out } // ── Own-core tone ──────────────────────────────────────────────────────────── diff --git a/peripheral/src/organ_cli.el b/peripheral/src/organ_cli.el new file mode 100644 index 0000000..35fd7ac --- /dev/null +++ b/peripheral/src/organ_cli.el @@ -0,0 +1,459 @@ +// organ_cli.el — the organ's command surface. main() lives here. +// +// One binary, the same verbs the Swift program had, and nothing behind them +// except El and two thin device realizers. This file is the proof surface: if +// `organ speak` makes a sound and no Swift binary is in the process tree, the +// claim in organ.el's header is true. +// +// Verbs, and what each one demonstrates: +// +// grant/revoke/status the Neuron-level consent lock, inspectable +// speak efferent — audio out of El's own speaker +// tone own-core synthesis: computed in El, played by El, +// never touching the disk +// say fetch a VOICE FROM THE ENGRAM and render through it +// listen afferent — mic capture, consent-gated, fails closed +// see afferent — one camera frame, same two locks +// wav-info WAV geometry, parsed in El +// feat-audio capture -> compact descriptor (8 numbers) +// feat-image frame -> compact scene-geometry +// voiceprint F0 + formants F1-F5 by LPC, in El +// imitate LPC analysis-resynthesis, in El +// hear-imitate the closed loop: hear a voice, take its signature, +// speak back in it +// ingest-audio descriptor -> engram node (the capture becomes geometry) +// ingest-voice voiceprint -> engram voice region (how a voice is learned) +// converse full-duplex interruptible utterance +// +// The descriptors are the point of the afferent half. A capture is NEVER handed +// on raw: a three-second recording is ~48,000 samples and what leaves this +// process is eight numbers. That is both the privacy rail (the stream stays +// local because only its shape travels) and the reason the engram can hold a +// perception at all — geometry is storable, a waveform is not. + +fn cli_usage() -> Bool { + println("organ — Neuron's I/O organ, native El (own-core, local, consent-gated)") + println(" grant|revoke Neuron-level consent") + println(" status consent + device state") + println(" speak play a WAV aloud (efferent)") + println(" tone [hz] [ms] synthesize and play, no file at all") + println(" say [CODE...] fetch voice FROM THE ENGRAM, render, speak") + println(" listen mic capture 16k mono (afferent)") + println(" see one camera frame (afferent)") + println(" wav-info WAV geometry") + println(" feat-audio compact audio descriptor (8 numbers)") + println(" feat-image compact scene-geometry from the camera") + println(" voiceprint F0 + formants F1-F5 (LPC)") + println(" imitate LPC analysis-resynthesis") + println(" hear-imitate mic -> signature -> imitate -> speak aloud") + println(" ingest-audio descriptor -> engram node (geometry)") + println(" ingest-voice voiceprint -> engram voice region") + println(" converse [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume]") + return true +} + +// The engram the organ reads and writes. Its own store, never production's. +fn cli_engram_dir() -> String { + let d: String = env("ORGAN_ENGRAM") + if str_eq(d, "") { + return "peripheral/.engram" + } + return d +} + +fn cli_open_engram() -> Bool { + let dir: String = cli_engram_dir() + fs_mkdir(dir) + let ok: Int = engram_store_boot(dir) + if ok == 1 { + return true + } + return false +} + +// ── formatting helpers ─────────────────────────────────────────────────────── + +fn cli_f(v: Float, dec: Int) -> String { + return format_float(v, dec) +} + +// ── the descriptor, printed and ingested ───────────────────────────────────── +// +// [seconds, sr, ch, rms, peak, zcr, centroid, f0] — the same eight numbers the +// Swift produced, computed in El, and the compression ratio is the headline: +// a few dozen bytes standing in for a few hundred kilobytes. +fn cli_audio_descriptor_text(v: [Float], path: String) -> String { + let secs: Float = native_list_get(v, 0) + let sr: Float = native_list_get(v, 1) + let ch: Float = native_list_get(v, 2) + let rms: Float = native_list_get(v, 3) + let peak: Float = native_list_get(v, 4) + let zcr: Float = native_list_get(v, 5) + let cen: Float = native_list_get(v, 6) + let f0: Float = native_list_get(v, 7) + return "Heard sound (afferent, mic): " + cli_f(secs, 2) + "s at " + cli_f(sr, 0) + "Hz. RMS energy " + cli_f(rms, 4) + ", peak " + cli_f(peak, 4) + ", zero-crossing rate " + cli_f(zcr, 0) + "Hz, spectral centroid " + cli_f(cen, 0) + "Hz, estimated voice pitch F0 " + cli_f(f0, 0) + "Hz. Compact voice/sound signature (8 numbers) — phonetic geometry seed." +} + +fn cli_feat_audio(path: String) -> Bool { + let v: [Float] = dsp_compute_audio(path) + if native_list_len(v) < 8 { + println("{\"ok\": false, \"op\": \"feat-audio\", \"error\": \"cannot read PCM\"}") + return false + } + organ_disclose("FEAT(audio): 8-number signature vs " + int_to_str(float_to_int(native_list_get(v, 0) * native_list_get(v, 1))) + " raw samples.") + println("{\"ok\": true, \"op\": \"feat-audio\", \"file\": \"" + path + "\", \"seconds\": " + cli_f(native_list_get(v, 0), 4) + ", \"sample_rate\": " + cli_f(native_list_get(v, 1), 0) + ", \"channels\": " + cli_f(native_list_get(v, 2), 0) + ", \"rms\": " + cli_f(native_list_get(v, 3), 6) + ", \"peak\": " + cli_f(native_list_get(v, 4), 6) + ", \"zcr_hz\": " + cli_f(native_list_get(v, 5), 4) + ", \"centroid_hz\": " + cli_f(native_list_get(v, 6), 4) + ", \"f0_hz\": " + cli_f(native_list_get(v, 7), 4) + "}") + return true +} + +fn cli_voiceprint(path: String) -> Bool { + let v: [Float] = dsp_voiceprint(path) + if native_list_len(v) < 4 { + println("{\"ok\": false, \"op\": \"voiceprint\", \"error\": \"cannot read speech\"}") + return false + } + let nf: Int = float_to_int(native_list_get(v, 3)) + let fs: String = "" + let bs: String = "" + let i: Int = 0 + while i < nf { + if i > 0 { + fs = fs + ", " + bs = bs + ", " + } + fs = fs + cli_f(native_list_get(v, 4 + i * 2), 3) + bs = bs + cli_f(native_list_get(v, 5 + i * 2), 3) + i = i + 1 + } + println("{\"ok\": true, \"op\": \"voiceprint\", \"file\": \"" + path + "\", \"f0_hz\": " + cli_f(native_list_get(v, 0), 4) + ", \"f0_range\": [" + cli_f(native_list_get(v, 1), 4) + ", " + cli_f(native_list_get(v, 2), 4) + "], \"formants_hz\": [" + fs + "], \"bandwidths_hz\": [" + bs + "]}") + return true +} + +// ── main ───────────────────────────────────────────────────────────────────── + +fn main() { + let a: [String] = args() + let n: Int = native_list_len(a) + if n < 1 { + cli_usage() + return + } + let cmd: String = native_list_get(a, 0) + + // ---- consent ----------------------------------------------------------- + if str_eq(cmd, "grant") { + if n < 2 { + println("grant needs a device") + return + } + organ_grant(native_list_get(a, 1)) + println("{\"ok\": true, \"op\": \"grant\", \"consent\": \"" + organ_consent_status() + "\"}") + return + } + if str_eq(cmd, "revoke") { + if n < 2 { + println("revoke needs a device") + return + } + organ_revoke(native_list_get(a, 1)) + println("{\"ok\": true, \"op\": \"revoke\", \"consent\": \"" + organ_consent_status() + "\"}") + return + } + if str_eq(cmd, "status") { + println("{\"ok\": true, \"op\": \"status\", \"consent\": \"" + organ_consent_status() + "\", \"speaker\": \"" + speaker_name() + "\", \"speaker_available\": " + int_to_str(speaker_available()) + ", \"mic_os_authorized\": " + int_to_str(mic_available()) + ", \"camera_os_authorized\": " + int_to_str(camera_available()) + "}") + return + } + + // ---- efferent ---------------------------------------------------------- + if str_eq(cmd, "speak") { + if n < 2 { + println("speak needs a wav") + return + } + let ok: Bool = organ_speak_wav(native_list_get(a, 1)) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"speak\", \"played_aloud\": " + bool_to_str(ok) + "}") + return + } + if str_eq(cmd, "tone") { + let hz: Int = 220 + let ms: Int = 1000 + if n >= 2 { + hz = str_to_int(native_list_get(a, 1)) + } + if n >= 3 { + ms = str_to_int(native_list_get(a, 2)) + } + let s: [Int] = organ_tone(hz, ms, 16000) + let ok: Bool = organ_speak_samples(s, 16000) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"tone\", \"hz\": " + int_to_str(hz) + ", \"ms\": " + int_to_str(ms) + ", \"samples\": " + int_to_str(native_list_len(s)) + ", \"file\": null}") + return + } + + // ---- the voice, from the engram ---------------------------------------- + if str_eq(cmd, "say") { + if n < 3 { + println("say needs [CODE...]") + return + } + cli_open_engram() + let vname: String = native_list_get(a, 1) + let g: [Int] = organ_voice_fetch(vname) + if native_list_len(g) < 6 { + println("{\"ok\": false, \"op\": \"say\", \"error\": \"no voice region '" + vname + "' in the engram\"}") + return + } + // Codes and the phoneme map come from the LANGUAGE side. The organ does + // not know what a word is and never looks one up. + let pmap: [String] = ingest_phonetics("elp/data/phonetics.psv") + let codes: [String] = native_list_empty() + let i: Int = 2 + while i < n { + codes = native_list_append(codes, native_list_get(a, i)) + i = i + 1 + } + let voice: [String] = organ_voice_profile(vname, g) + let s: [Int] = synth_codes(codes, voice, pmap) + let ok: Bool = organ_speak_samples(s, 16000) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"say\", \"voice\": \"" + vname + "\", \"f0\": " + int_to_str(native_list_get(g, 0)) + ", \"kf\": " + int_to_str(native_list_get(g, 2)) + ", \"codes\": " + int_to_str(native_list_len(codes)) + ", \"samples\": " + int_to_str(native_list_len(s)) + "}") + return + } + + // ---- afferent ---------------------------------------------------------- + if str_eq(cmd, "listen") { + if n < 3 { + println("listen needs ") + return + } + let secs: Int = str_to_int(native_list_get(a, 1)) + let out: String = native_list_get(a, 2) + if organ_may_listen() == false { + println("{\"ok\": false, \"op\": \"listen\", \"error\": \"consent denied (fails closed)\"}") + return + } + organ_disclose("MIC: capturing " + int_to_str(secs) + "s (16 kHz mono, LOCAL, never egresses).") + let s: [Int] = mic_capture_pcm16(secs, 16000) + let got: Int = native_list_len(s) + if got <= 0 { + println("{\"ok\": false, \"op\": \"listen\", \"error\": \"capture returned nothing\"}") + return + } + let ok: Bool = write_wav(s, 16000, out) + organ_disclose("MIC: captured " + int_to_str(got) + " frames — ready to hand to the ingest organ.") + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"listen\", \"file\": \"" + out + "\", \"frames\": " + int_to_str(got) + ", \"sample_rate\": 16000}") + return + } + if str_eq(cmd, "see") { + if n < 2 { + println("see needs an out path") + return + } + if organ_may_see() == false { + println("{\"ok\": false, \"op\": \"see\", \"error\": \"consent denied (fails closed)\"}") + return + } + organ_disclose("CAMERA: capturing one frame (LOCAL, never egresses).") + let ok: Int = camera_capture_jpeg(native_list_get(a, 1)) + println("{\"ok\": " + int_to_str(ok) + ", \"op\": \"see\", \"file\": \"" + native_list_get(a, 1) + "\"}") + return + } + + // ---- descriptors ------------------------------------------------------- + if str_eq(cmd, "wav-info") { + if n < 2 { + println("wav-info needs a wav") + return + } + let p: String = native_list_get(a, 1) + let w: [Float] = dsp_read_wav(p) + if dsp_wav_n(w) <= 0 { + println("{\"ok\": false, \"op\": \"wav-info\"}") + return + } + println("{\"ok\": true, \"op\": \"wav-info\", \"sample_rate\": " + int_to_str(dsp_wav_sr(w)) + ", \"channels\": " + int_to_str(dsp_wav_ch(w)) + ", \"frames\": " + int_to_str(dsp_wav_n(w)) + "}") + return + } + if str_eq(cmd, "feat-audio") { + if n < 2 { + println("feat-audio needs a wav") + return + } + cli_feat_audio(native_list_get(a, 1)) + return + } + if str_eq(cmd, "feat-image") { + if organ_may_see() == false { + println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"consent denied (fails closed)\"}") + return + } + let f: [Int] = organ_image_descriptor() + if native_list_len(f) < 15 { + println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"no frame\"}") + return + } + let grid: String = "" + let i: Int = 6 + while i < 15 { + if i > 6 { + grid = grid + ", " + } + grid = grid + int_to_str(native_list_get(f, i)) + i = i + 1 + } + println("{\"ok\": true, \"op\": \"feat-image\", \"width\": " + int_to_str(native_list_get(f, 0)) + ", \"height\": " + int_to_str(native_list_get(f, 1)) + ", \"mean_rgb\": [" + int_to_str(native_list_get(f, 2)) + ", " + int_to_str(native_list_get(f, 3)) + ", " + int_to_str(native_list_get(f, 4)) + "], \"brightness_pm\": " + int_to_str(native_list_get(f, 5)) + ", \"luma_grid\": [" + grid + "]}") + return + } + if str_eq(cmd, "voiceprint") { + if n < 2 { + println("voiceprint needs a wav") + return + } + cli_voiceprint(native_list_get(a, 1)) + return + } + if str_eq(cmd, "imitate") { + if n < 3 { + println("imitate needs ") + return + } + let s: [Int] = dsp_imitate(native_list_get(a, 1)) + if native_list_len(s) <= 0 { + println("{\"ok\": false, \"op\": \"imitate\"}") + return + } + let ok: Bool = write_wav(s, 16000, native_list_get(a, 2)) + organ_disclose("IMITATE: rebuilt the voice from its own LPC signature (own-core, no training, no stolen voice).") + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"imitate\", \"out\": \"" + native_list_get(a, 2) + "\", \"samples\": " + int_to_str(native_list_len(s)) + ", \"method\": \"LPC analysis-resynthesis\"}") + return + } + if str_eq(cmd, "hear-imitate") { + if n < 3 { + println("hear-imitate needs ") + return + } + let secs: Int = str_to_int(native_list_get(a, 1)) + let out: String = native_list_get(a, 2) + if organ_may_listen() == false { + println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"consent denied (fails closed)\"}") + return + } + let heard: String = out + ".heard.wav" + organ_disclose("HEAR-IMITATE: open the ear, listen " + int_to_str(secs) + "s, take the voice, speak it back.") + let s: [Int] = mic_capture_pcm16(secs, 16000) + if native_list_len(s) <= 0 { + println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"capture returned nothing\"}") + return + } + write_wav(s, 16000, heard) + let re: [Int] = dsp_imitate(heard) + if native_list_len(re) <= 0 { + println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"could not model the voice\"}") + return + } + write_wav(re, 16000, out) + let ok: Bool = organ_speak_samples(re, 16000) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"hear-imitate\", \"heard\": \"" + heard + "\", \"out\": \"" + out + "\", \"spoke_aloud\": " + bool_to_str(ok) + "}") + return + } + + // ---- the afferent wire: descriptor -> geometry -------------------------- + if str_eq(cmd, "ingest-audio") { + if n < 2 { + println("ingest-audio needs a wav") + return + } + let p: String = native_list_get(a, 1) + let v: [Float] = dsp_compute_audio(p) + if native_list_len(v) < 8 { + println("{\"ok\": false, \"op\": \"ingest-audio\"}") + return + } + cli_open_engram() + let content: String = cli_audio_descriptor_text(v, p) + let id: String = engram_node(content, "Observation", 70) + engram_store_checkpoint() + organ_disclose("INGEST: the capture is now GEOMETRY in the engram (node " + id + ") — the descriptor travelled, the stream did not.") + println("{\"ok\": true, \"op\": \"ingest-audio\", \"node_id\": \"" + id + "\", \"content\": \"" + content + "\"}") + return + } + if str_eq(cmd, "ingest-voice") { + if n < 3 { + println("ingest-voice needs ") + return + } + let p: String = native_list_get(a, 1) + let name: String = native_list_get(a, 2) + let v: [Float] = dsp_voiceprint(p) + if native_list_len(v) < 10 { + println("{\"ok\": false, \"op\": \"ingest-voice\", \"error\": \"no voiced frames\"}") + return + } + cli_open_engram() + let f0: Int = float_to_int(native_list_get(v, 0)) + let f1: Int = float_to_int(native_list_get(v, 4)) + let f2: Int = float_to_int(native_list_get(v, 6)) + let f3: Int = float_to_int(native_list_get(v, 8)) + // kf is the vocal-tract scale: this speaker's F1 against the nominal + // /AA/ F1 of 730 Hz. One number standing for a tract length. + let kf: Int = 1000 * f1 / 730 + let f0e: Int = f0 * 85 / 100 + let id: String = organ_voice_ingest(name, f0, f0e, kf, f1, f2, f3, "el-organ-lpc-voiceprint", "COARSE") + engram_store_checkpoint() + println("{\"ok\": true, \"op\": \"ingest-voice\", \"node_id\": \"" + id + "\", \"name\": \"" + name + "\", \"f0\": " + int_to_str(f0) + ", \"kf\": " + int_to_str(kf) + ", \"f1\": " + int_to_str(f1) + ", \"f2\": " + int_to_str(f2) + ", \"f3\": " + int_to_str(f3) + "}") + return + } + + // ---- converse ---------------------------------------------------------- + if str_eq(cmd, "converse") { + if n < 2 { + println("converse needs a manifest") + return + } + let mf: String = native_list_get(a, 1) + let authority: Int = 500 + let barge: Int = 0 - 1 + let kind: String = "bargein" + let live: Bool = false + let resume: Bool = false + let i: Int = 2 + while i < n { + let f: String = native_list_get(a, i) + if str_eq(f, "--authority") { + if i + 1 < n { + authority = str_to_int(native_list_get(a, i + 1)) + i = i + 1 + } + } + if str_eq(f, "--barge-at") { + if i + 1 < n { + let spec: String = native_list_get(a, i + 1) + let c: Int = str_index_of(spec, ":") + if c < 0 { + barge = str_to_int(spec) + } else { + barge = str_to_int(str_slice(spec, 0, c)) + kind = str_slice(spec, c + 1, str_len(spec)) + } + i = i + 1 + } + } + if str_eq(f, "--live-mic") { + live = true + } + if str_eq(f, "--resume") { + resume = true + } + i = i + 1 + } + let plan: [String] = conv_load_manifest(mf) + if resume { + plan = conv_load_resume() + organ_disclose("CONVERSE: resuming — \"as I was saying...\" (" + int_to_str(plan_count(plan)) + " segments left).") + } else { + organ_disclose("CONVERSE: utterance = \"" + conv_utterance(mf) + "\" (" + int_to_str(plan_count(plan)) + " segments).") + } + let stopped: Int = conv_run(plan, authority, barge, kind, live) + println("{\"ok\": true, \"op\": \"converse\", \"stopped_at\": " + int_to_str(stopped) + ", \"complete\": " + bool_to_str(stopped < 0) + "}") + return + } + + cli_usage() +} diff --git a/peripheral/src/organ_converse.el b/peripheral/src/organ_converse.el new file mode 100644 index 0000000..058dad9 --- /dev/null +++ b/peripheral/src/organ_converse.el @@ -0,0 +1,454 @@ +// organ_converse.el — full-duplex, interruptible speech. The turn-taking organ. +// +// WHAT THIS IS FOR. A system that plays an utterance to completion and only +// then listens is not conversational, it is a loudspeaker with a queue. Being +// interruptible is not a feature bolted onto speech; it is most of what makes +// speech social. So the utterance is not a blob of audio — it is an ordered, +// SALIENCE-TAGGED MEANING-PLAN, and the organ speaks it while listening, decides +// what to do when interrupted, and can pick the thread back up afterwards. +// +// THREE THINGS HAVE TO BE TRUE, and each one is a place naive implementations +// go wrong: +// +// Barge-in is AT THE SAMPLE. When the mic hears speech, output stops on the +// spot — not at the end of the current buffer, not at the end of the segment. +// A listener experiences even a fifth of a second of continued talking as +// being talked over. This is why the speaker realizer has pause/resume and +// reports played_frames: "finish the buffer" is not barge-in. +// +// Yield-or-hold is a DECISION, not a rule. Stopping every time anyone makes a +// noise is its own failure — it means Neuron can never finish a sentence that +// matters. So the choice is grounded: how salient is what I am mid-saying, +// how close am I to done, and how much authority does the interrupter have. +// Holding the floor is justified when what I am saying matters AND finishing +// is cheap AND the interrupter is not high-priority. Otherwise yield, because +// the polite default is the right default. +// +// A backchannel is NOT an interruption. "mm-hm" means keep going. Treating it +// as a barge-in makes the system stop every three seconds during ordinary +// listening behaviour, which is worse than not listening at all. It is +// distinguished by being brief and low-energy: sample again shortly after +// onset, and if the speech already died away it was a backchannel. +// +// AND THE UTTERANCE SURVIVES. On yield, the remaining plan is persisted, so +// Neuron can resume — "as I was saying" — instead of losing the thought. An +// interruption should cost a turn, not the content. +// +// The AEC rail: the microphone runs with the OS voice-processing unit enabled +// so it does not hear our own speaker. Without it Neuron barges in on its own +// voice on the first syllable and the whole loop is unusable in a real room. +// +// Note what is NOT here: nothing about words. A segment carries a `text` field +// purely as a label for disclosure. The organ speaks pre-rendered audio and +// never inspects language — that is the language faculty's, and the seam holds. + +// ── The meaning-plan ───────────────────────────────────────────────────────── +// +// Stored as a flat [String] with stride 3 — file, salience-per-mille, text — +// because El has no record type and parallel lists drift out of step under +// editing. Salience is an integer per-mille rather than a Float so the decision +// arithmetic stays exact and reproducible; a turn-taking decision that varies +// with floating-point rounding is not one you can debug. + +fn plan_new() -> [String] { + return native_list_empty() +} + +fn plan_add(plan: [String], file: String, salience_pm: Int, text: String) -> [String] { + let p: [String] = plan + p = native_list_append(p, file) + p = native_list_append(p, int_to_str(salience_pm)) + p = native_list_append(p, text) + return p +} + +fn plan_count(plan: [String]) -> Int { + return native_list_len(plan) / 3 +} + +fn plan_file(plan: [String], i: Int) -> String { + return native_list_get(plan, i * 3) +} + +fn plan_salience(plan: [String], i: Int) -> Int { + return str_to_int(native_list_get(plan, i * 3 + 1)) +} + +fn plan_text(plan: [String], i: Int) -> String { + return native_list_get(plan, i * 3 + 2) +} + +// ── Manifest ───────────────────────────────────────────────────────────────── +// +// {"utterance": "...", "segments": [{"file":..., "salience":0.9, "text":"..."}]} +// Salience arrives as a 0..1 float in the manifest and is converted once, here, +// at the edge — the same discipline the runtime uses for wire encodings. + +fn conv_salience_pm(raw: String) -> Int { + // "0.85" -> 850. Parsed by hand rather than through a float so a manifest + // typo degrades to a visible number instead of a silent 0.0. + let dot: Int = str_index_of(raw, ".") + if dot < 0 { + let whole: Int = str_to_int(raw) + return whole * 1000 + } + let ip: Int = str_to_int(str_slice(raw, 0, dot)) + let frac: String = str_slice(raw, dot + 1, str_len(raw)) + let pm: Int = 0 + let scale: Int = 100 + let i: Int = 0 + while i < 3 { + let d: Int = 0 + if i < str_len(frac) { + let c: Int = str_char_code(frac, i) + if c >= 48 { + if c <= 57 { + d = c - 48 + } + } + } + pm = pm + d * scale + scale = scale / 10 + i = i + 1 + } + return ip * 1000 + pm +} + +fn conv_load_manifest(path: String) -> [String] { + let plan: [String] = plan_new() + let raw: String = fs_read(path) + if str_eq(raw, "") { + organ_disclose("CONVERSE: cannot read manifest " + path) + return plan + } + let segs: String = json_get_raw(raw, "segments") + let n: Int = json_array_len(segs) + let i: Int = 0 + while i < n { + let seg: String = json_array_get(segs, i) + let file: String = json_get_string(seg, "file") + let text: String = json_get_string(seg, "text") + let sal: String = json_get_raw(seg, "salience") + let pm: Int = conv_salience_pm(sal) + if pm <= 0 { + pm = 500 + } + plan = plan_add(plan, file, pm, text) + i = i + 1 + } + return plan +} + +fn conv_utterance(path: String) -> String { + let raw: String = fs_read(path) + return json_get_string(raw, "utterance") +} + +// ── The decision ───────────────────────────────────────────────────────────── +// +// Returns: 0 = backchannel, carry on seamlessly +// 1 = hold the floor ("hang on, let me finish this thought") +// 2 = yield (stop, let them in) +// +// All arguments are per-mille integers. Holding requires BOTH that the material +// is worth finishing AND that the interrupter is not high-authority — either +// condition alone is not enough, because "what I'm saying is important" is +// exactly the reasoning that produces a system nobody can get a word in against. +fn conv_decide(salience_pm: Int, progress_pm: Int, authority_pm: Int, is_backchannel: Bool) -> Int { + if is_backchannel { + return 0 + } + let hold_score: Int = (salience_pm * 6 + progress_pm * 4) / 10 + if hold_score >= 600 { + if authority_pm < 800 { + return 1 + } + } + return 2 +} + +// ── Resume ─────────────────────────────────────────────────────────────────── +// +// The remaining plan, written where a later run can find it. This is what turns +// an interruption into a pause rather than a loss. + +fn conv_resume_path() -> String { + let home: String = env("PERIPH_HOME") + if str_eq(home, "") { + return "peripheral/.resume.json" + } + return home + "/.resume.json" +} + +// Minimal JSON string escaping. Written here rather than reached for from the +// runtime because the organ needs exactly two escapes and no dependency: a +// segment label containing a quote or a backslash must not be able to produce a +// resume file that fails to parse and silently loses the thread. +fn conv_escape(s: String) -> String { + let n: Int = str_len(s) + let out: String = "" + let i: Int = 0 + while i < n { + let c: Int = str_char_code(s, i) + if c == 34 { + out = out + "\\\"" + } else { + if c == 92 { + out = out + "\\\\" + } else { + if c >= 32 { + out = out + str_slice(s, i, i + 1) + } + } + } + i = i + 1 + } + return out +} + +fn conv_persist_resume(plan: [String], start_at: Int, reason: String) -> Bool { + let n: Int = plan_count(plan) + let body: String = "{\"resume_from\": " + int_to_str(start_at) + ", \"reason\": \"" + reason + "\", \"segments\": [" + let i: Int = start_at + let first: Bool = true + while i < n { + if first == false { + body = body + ", " + } + body = body + "{\"file\": \"" + plan_file(plan, i) + "\", \"salience\": " + int_to_str(plan_salience(plan, i)) + ", \"text\": \"" + conv_escape(plan_text(plan, i)) + "\"}" + first = false + i = i + 1 + } + body = body + "]}\n" + let ok: Bool = fs_write(conv_resume_path(), body) + organ_disclose("CONVERSE: meaning-plan persisted (" + int_to_str(n - start_at) + " segments remain) — Neuron can resume the thread.") + return ok +} + +fn conv_clear_resume() -> Bool { + return fs_write(conv_resume_path(), "") +} + +// Read a persisted plan back. Salience is already per-mille here (we wrote it), +// so it is NOT re-scaled — the manifest and the resume file are different +// formats on purpose, and conflating them silently divides every salience by a +// thousand. +fn conv_load_resume() -> [String] { + let plan: [String] = plan_new() + let raw: String = fs_read(conv_resume_path()) + if str_eq(raw, "") { + return plan + } + let segs: String = json_get_raw(raw, "segments") + let n: Int = json_array_len(segs) + let i: Int = 0 + while i < n { + let seg: String = json_array_get(segs, i) + plan = plan_add(plan, json_get_string(seg, "file"), json_get_int(seg, "salience"), json_get_string(seg, "text")) + i = i + 1 + } + return plan +} + +// ── The loop ───────────────────────────────────────────────────────────────── +// +// live_mic : open the microphone with AEC and let real speech drive barge-in. +// barge_ms : if >= 0, inject a barge event at that offset into the utterance +// instead. Deterministic, so the decision paths can be exercised +// without a room and a person — the same reason periph.swift has it. +// kind : "backchannel" or "bargein", for the injected case. +// authority : interrupter authority, per-mille. +// +// Returns the index the utterance stopped at, or -1 if it completed. + +fn conv_run(plan: [String], authority_pm: Int, barge_ms: Int, kind: String, live_mic: Bool) -> Int { + let n: Int = plan_count(plan) + if n <= 0 { + organ_disclose("CONVERSE: nothing to say.") + return 0 - 1 + } + if speaker_available() == 0 { + organ_disclose("CONVERSE: no speaker on this build — cannot hold a conversation.") + return 0 - 1 + } + + let mic_live: Bool = false + if live_mic { + if organ_may_listen() { + let m: Int = mic_monitor_start() + if m == 1 { + organ_disclose("CONVERSE: full-duplex — mic listening WHILE speaking, AEC on (won't self-interrupt).") + mic_live = true + } + if m == 2 { + organ_disclose("CONVERSE: full-duplex — mic listening, but AEC UNAVAILABLE; raising the VAD floor so we do not barge in on ourselves.") + mic_live = true + } + if m == 0 { + organ_disclose("CONVERSE: could not open the mic monitor — falling back to injected events.") + } + } + } + if mic_live == false { + organ_disclose("CONVERSE: deterministic mode (live mic off).") + } + + // Without AEC the mic hears the speaker, so the threshold has to sit above + // our own output. This is a mitigation and not a fix: the honest note is + // that barge-in is markedly less sensitive in this mode. + let vad_pm: Int = 20 + if mic_live { + if mic_monitor_start() == 2 { + vad_pm = 60 + } + } + + let elapsed_ms: Int = 0 + let prior_ms: Int = 0 + let handled: Bool = false + // An injected barge is ONE event, not a condition that stays true. Without + // this the deadline re-fires on every poll after a backchannel resume, and + // the utterance live-locks: paused, resumed, paused again, forever. + let injected_fired: Bool = false + let i: Int = 0 + + while i < n { + let file: String = plan_file(plan, i) + let sal: Int = plan_salience(plan, i) + let frames: Int = wav_frames(file) + let rate: Int = wav_rate(file) + if frames <= 0 { + organ_disclose("CONVERSE: missing or unreadable segment '" + file + "', skipping.") + i = i + 1 + } else { + let dur_ms: Int = frames * 1000 / rate + organ_disclose("CONVERSE: speaking segment " + int_to_str(i + 1) + "/" + int_to_str(n) + " (salience " + int_to_str(sal) + "/1000) — \"" + plan_text(plan, i) + "\"") + let started: Int = speaker_play_wav_async(file) + if started == 0 { + organ_disclose("CONVERSE: could not start playback for '" + file + "'.") + i = i + 1 + } else { + let seg_ms: Int = 0 + let done: Bool = false + let interrupted: Bool = false + let speech_ticks: Int = 0 + + while done == false { + sleep_ms(10) + seg_ms = seg_ms + 10 + + if speaker_playing() == 0 { + done = true + } else { + // The tick counter is an approximation — each pass costs + // more than the sleep it asked for. The DAC position is + // the truth, so drive the injected deadline off THAT and + // an injected barge lands where it was asked to land. + let pos_ms: Int = speaker_played_frames() * 1000 / rate + elapsed_ms = prior_ms + pos_ms + // --- onset detection: real speech, or an injected event --- + let onset: Bool = false + if mic_live { + let rms: Float = mic_monitor_rms() + let rms_pm: Int = float_to_int(rms * 1000.0) + if rms_pm > vad_pm { + speech_ticks = speech_ticks + 1 + } else { + speech_ticks = 0 + } + // ~60ms of continuous voice: short enough to feel + // instant, long enough that a door closing is not a turn. + if speech_ticks >= 3 { + if handled == false { + onset = true + } + } + } + if barge_ms >= 0 { + if injected_fired == false { + if elapsed_ms >= barge_ms { + onset = true + injected_fired = true + } + } + } + + if onset { + handled = true + // (1) BARGE-IN — pause on the spot. + speaker_pause() + let played: Int = speaker_played_frames() + let at_ms: Int = played * 1000 / rate + let progress_pm: Int = at_ms * 1000 / dur_ms + if progress_pm > 1000 { + progress_pm = 1000 + } + organ_disclose("CONVERSE: << user speech at " + int_to_str(at_ms) + "ms into segment " + int_to_str(i + 1) + " — PAUSED instantly >>") + + // (2) backchannel or real barge-in? + let is_bc: Bool = false + if barge_ms >= 0 { + if str_eq(kind, "backchannel") { + is_bc = true + } + } else { + // Live: look again ~250ms after onset. If the + // energy has already collapsed it was "mm-hm". + sleep_ms(250) + let r2: Float = mic_monitor_rms() + if float_to_int(r2 * 1000.0) < 15 { + is_bc = true + } + } + + // (3) yield, hold, or carry on + let d: Int = conv_decide(sal, progress_pm, authority_pm, is_bc) + if d == 0 { + organ_disclose("CONVERSE: read as BACKCHANNEL (\"mm-hm\") — keep going, resume seamlessly.") + handled = false + speech_ticks = 0 + speaker_resume() + } + if d == 1 { + organ_disclose("CONVERSE: HOLD the floor — \"hang on, let me finish this thought.\" (salience " + int_to_str(sal) + ", progress " + int_to_str(progress_pm) + ")") + speaker_resume() + // Finish THIS segment, then yield the remainder: + // holding is a request for a moment, not a claim + // on the rest of the conversation. + while speaker_playing() == 1 { + sleep_ms(20) + } + speaker_stop() + conv_persist_resume(plan, i + 1, "held-then-yield") + if mic_live { + mic_monitor_stop() + } + return i + 1 + } + if d == 2 { + organ_disclose("CONVERSE: YIELD — stop, let them in. Remembering where I was (resumable).") + speaker_stop() + conv_persist_resume(plan, i, "yield") + if mic_live { + mic_monitor_stop() + } + return i + } + } + } + } + if interrupted == false { + prior_ms = prior_ms + dur_ms + i = i + 1 + } + } + } + } + + conv_clear_resume() + organ_disclose("CONVERSE: utterance complete (uninterrupted).") + if mic_live { + mic_monitor_stop() + } + return 0 - 1 +} diff --git a/peripheral/src/organ_dsp.el b/peripheral/src/organ_dsp.el new file mode 100644 index 0000000..87068f0 --- /dev/null +++ b/peripheral/src/organ_dsp.el @@ -0,0 +1,1019 @@ +// organ_dsp.el — the AFFERENT DSP organ, own-core, ported from peripheral/src/periph.swift. +// +// The mirror of speech.el: where speech.el RENDERS meaning out through a voice +// (efferent), this module HEARS — it takes a raw 16-bit PCM RIFF/WAVE capture +// and metabolizes it into GEOMETRY: a compact 8-number audio descriptor, a +// voice-signature (F0 + formants F1-F5 by LPC), and an LPC analysis-resynthesis +// that speaks the heard voice back from its own signature. +// +// Everything here is float physics (math_sin/math_cos/math_sqrt), not the +// fixed-point integer path speech.el uses for synthesis — the analysis side +// needs the dynamic range that autocorrelation and Levinson-Durbin demand. +// +// Binary I/O note: an El String truncates at the first NUL, so a WAV can never +// be read through fs_read(). We read it through fs_read_b64_chunk(), which +// hands back plain-ASCII base64 of a byte window, and decode that base64 HERE, +// in El, into a [Int] of byte values. Chunks are a multiple of 3 bytes so each +// base64 window decodes cleanly with no padding except the final one. +// +// Every function is prefixed dsp_ so nothing here can collide with speech.el +// (write_wav / wav_le16 / wav_le32 / sp_* / voice_* all live there and are NOT +// redefined). imitate returns [Int] samples — hand them to speech.el's write_wav. + +// --------------------------------------------------------------------------- +// Base64 -> bytes (own-core; the only way binary reaches El intact) +// --------------------------------------------------------------------------- + +// Standard RFC 4648 alphabet A-Za-z0-9+/ -> 0..63. Padding '=' and any other +// character -> -1 (a sentinel; there is no exception handling in El). +fn dsp_b64_val(c: Int) -> Int { + if c >= 65 { + if c <= 90 { + return c - 65 + } + } + if c >= 97 { + if c <= 122 { + return c - 71 + } + } + if c >= 48 { + if c <= 57 { + return c + 4 + } + } + if c == 43 { + return 62 + } + if c == 47 { + return 63 + } + return 0 - 1 +} + +// Read a whole file as a list of byte values 0..255. Empty list on failure. +fn dsp_read_bytes(path: String) -> [Int] { + let bytes: [Int] = native_list_empty() + let size: Int = fs_size(path) + if size <= 0 { + return bytes + } + let chunk: Int = 60000 // multiple of 3 -> no interior padding + let off: Int = 0 + while off < size { + let s: String = fs_read_b64_chunk(path, off, chunk) + let sl: Int = str_len(s) + if sl < 4 { + return bytes + } + let i: Int = 0 + while i + 3 < sl { + let c0: Int = dsp_b64_val(str_char_code(s, i)) + let c1: Int = dsp_b64_val(str_char_code(s, i + 1)) + let c2: Int = dsp_b64_val(str_char_code(s, i + 2)) + let c3: Int = dsp_b64_val(str_char_code(s, i + 3)) + if c0 < 0 { + return bytes + } + if c1 < 0 { + return bytes + } + let b0: Int = c0 * 4 + c1 / 16 + bytes = native_list_append(bytes, b0) + if c2 >= 0 { + let lo1: Int = c1 - (c1 / 16) * 16 + let b1: Int = lo1 * 16 + c2 / 4 + bytes = native_list_append(bytes, b1) + if c3 >= 0 { + let lo2: Int = c2 - (c2 / 4) * 4 + let b2: Int = lo2 * 64 + c3 + bytes = native_list_append(bytes, b2) + } + } + i = i + 4 + } + off = off + chunk + } + return bytes +} + +fn dsp_rd16(b: [Int], o: Int) -> Int { + let b0: Int = native_list_get(b, o) + let b1: Int = native_list_get(b, o + 1) + return b0 + b1 * 256 +} + +fn dsp_rd32(b: [Int], o: Int) -> Int { + let b0: Int = native_list_get(b, o) + let b1: Int = native_list_get(b, o + 1) + let b2: Int = native_list_get(b, o + 2) + let b3: Int = native_list_get(b, o + 3) + return b0 + b1 * 256 + b2 * 65536 + b3 * 16777216 +} + +// Four bytes at o compared against a 4-char ASCII chunk id. +fn dsp_chunk_is(b: [Int], o: Int, id: String) -> Bool { + let k: Int = 0 + while k < 4 { + let got: Int = native_list_get(b, o + k) + let want: Int = str_char_code(id, k) + if got != want { + return false + } + k = k + 1 + } + return true +} + +// --------------------------------------------------------------------------- +// readWavSamples — 16-bit PCM RIFF/WAVE -> normalized samples. +// Walks chunks to find 'fmt ' and 'data', so JUNK/FLLR padding (which +// AVAudioRecorder emits) is stepped over rather than mistaken for audio. +// Channel 0 only if stereo. +// +// The returned list is PACKED: [ sr, ch, n, s0, s1, ... s(n-1) ] with the three +// header numbers carried as Floats (El has no tuples). Use dsp_wav_sr / +// dsp_wav_ch / dsp_wav_n / dsp_wav_pcm to open it. Empty list on failure. +// --------------------------------------------------------------------------- +fn dsp_read_wav(path: String) -> [Float] { + let out: [Float] = native_list_empty() + let d: [Int] = dsp_read_bytes(path) + let dn: Int = native_list_len(d) + if dn <= 44 { + return out + } + let ch: Int = 0 + let sr: Int = 0 + let bits: Int = 0 + let o: Int = 12 + while o + 8 <= dn { + let sz: Int = dsp_rd32(d, o + 4) + if dsp_chunk_is(d, o, "fmt ") { + if o + 24 <= dn { + ch = dsp_rd16(d, o + 10) + sr = dsp_rd32(d, o + 12) + bits = dsp_rd16(d, o + 22) + } + } + if dsp_chunk_is(d, o, "data") { + if bits != 16 { + return out + } + if ch <= 0 { + return out + } + // Faithful to periph.swift, including its `d.count - 1` bound and + // the `while i + 1 < end` test — the last sample of a file whose + // data chunk runs to EOF is dropped there, so it is dropped here. + let start: Int = o + 8 + let end: Int = start + sz + if end > dn - 1 { + end = dn - 1 + } + let samples: [Float] = native_list_empty() + let step: Int = 2 * ch + let i: Int = start + let count: Int = 0 + while i + 1 < end { + let v: Int = dsp_rd16(d, i) + if v >= 32768 { + v = v - 65536 + } + let f: Float = int_to_float(v) / 32768.0 + samples = native_list_append(samples, f) + count = count + 1 + i = i + step + } + let srf: Float = int_to_float(sr) + let chf: Float = int_to_float(ch) + let nf: Float = int_to_float(count) + out = native_list_append(out, srf) + out = native_list_append(out, chf) + out = native_list_append(out, nf) + let j: Int = 0 + while j < count { + let sv: Float = native_list_get(samples, j) + out = native_list_append(out, sv) + j = j + 1 + } + return out + } + let adv: Int = 8 + sz + (sz - (sz / 2) * 2) + if adv <= 0 { + return out + } + o = o + adv + } + return out +} + +fn dsp_wav_sr(w: [Float]) -> Int { + if native_list_len(w) < 3 { + return 0 + } + let v: Float = native_list_get(w, 0) + return float_to_int(v) +} + +fn dsp_wav_ch(w: [Float]) -> Int { + if native_list_len(w) < 3 { + return 0 + } + let v: Float = native_list_get(w, 1) + return float_to_int(v) +} + +fn dsp_wav_n(w: [Float]) -> Int { + if native_list_len(w) < 3 { + return 0 + } + let v: Float = native_list_get(w, 2) + return float_to_int(v) +} + +// The bare sample list, unpacked from the header-prefixed form. +fn dsp_wav_pcm(w: [Float]) -> [Float] { + let out: [Float] = native_list_empty() + let n: Int = dsp_wav_n(w) + let i: Int = 0 + while i < n { + let v: Float = native_list_get(w, i + 3) + out = native_list_append(out, v) + i = i + 1 + } + return out +} + +// --------------------------------------------------------------------------- +// small float helpers (El has no unary minus on Float in every position, and +// no min/max builtin — so they are written out) +// --------------------------------------------------------------------------- +fn dsp_fabs(v: Float) -> Float { + if v < 0.0 { + return 0.0 - v + } + return v +} + +// --------------------------------------------------------------------------- +// computeAudio — the compact audio descriptor. +// Returns the 8-number vector [seconds, sr, ch, rms, peak, zcr, centroid, f0]. +// Empty list if the WAV cannot be read (no exceptions in El — sentinels only). +// --------------------------------------------------------------------------- +fn dsp_compute_audio(path: String) -> [Float] { + let vec: [Float] = native_list_empty() + let w: [Float] = dsp_read_wav(path) + let n: Int = dsp_wav_n(w) + if n <= 0 { + return vec + } + let sr: Int = dsp_wav_sr(w) + let ch: Int = dsp_wav_ch(w) + let s: [Float] = dsp_wav_pcm(w) + let nf: Float = int_to_float(n) + let srf: Float = int_to_float(sr) + let seconds: Float = nf / srf + + // energy / peak / zero crossings + let sumsq: Float = 0.0 + let peak: Float = 0.0 + let zc: Float = 0.0 + let i: Int = 0 + while i < n { + let v: Float = native_list_get(s, i) + sumsq = sumsq + v * v + let av: Float = dsp_fabs(v) + if av > peak { + peak = av + } + if i > 0 { + let pv: Float = native_list_get(s, i - 1) + let a: Bool = pv < 0.0 + let b: Bool = v < 0.0 + if a != b { + zc = zc + 1.0 + } + } + i = i + 1 + } + let rms: Float = math_sqrt(sumsq / nf) + let zcr: Float = zc / nf * srf // ~2*dominant freq for tonal + + // Spectral centroid via a coarse 64-bin DFT on a centered 2048 window. + let ww: Int = 2048 + if n < ww { + ww = n + } + let off: Int = (n - ww) / 2 + if off < 0 { + off = 0 + } + let two: Float = 2.0 + let pi: Float = math_pi() + let num: Float = 0.0 + let den: Float = 0.0 + let bins: Int = 64 + let binsf: Float = 128.0 // Double(2*bins) + let k: Int = 1 + while k < bins { + let kf: Float = int_to_float(k) + let f: Float = kf * srf / binsf + let re: Float = 0.0 + let im: Float = 0.0 + let j: Int = 0 + while j < ww { + let jf: Float = int_to_float(j) + let ang: Float = (0.0 - two) * pi * kf * jf / binsf + let xv: Float = native_list_get(s, off + j) + let cv: Float = math_cos(ang) + let sv: Float = math_sin(ang) + re = re + xv * cv + im = im + xv * sv + j = j + 1 + } + let mag: Float = math_sqrt(re * re + im * im) + num = num + f * mag + den = den + mag + k = k + 1 + } + let centroid: Float = 0.0 + if den > 0.0 { + centroid = num / den + } + + // F0 by autocorrelation over the plausible speech range 70-400 Hz. + let lag_min: Int = sr / 400 + let lag_max: Int = sr / 70 + if lag_max > n - 1 { + lag_max = n - 1 + } + let bound: Int = off + ww + if bound > n { + bound = n + } + let best_lag: Int = 0 + let best_corr: Float = 0.0 + if lag_max > lag_min { + let lag: Int = lag_min + while lag <= lag_max { + let c: Float = 0.0 + let p: Int = 0 + while p + lag < bound { + let a1: Float = native_list_get(s, off + p) + let a2: Float = native_list_get(s, off + p + lag) + c = c + a1 * a2 + p = p + 1 + } + if c > best_corr { + best_corr = c + best_lag = lag + } + lag = lag + 1 + } + } + let f0: Float = 0.0 + if best_lag > 0 { + f0 = srf / int_to_float(best_lag) + } + + vec = native_list_append(vec, seconds) + vec = native_list_append(vec, srf) + let chf: Float = int_to_float(ch) + vec = native_list_append(vec, chf) + vec = native_list_append(vec, rms) + vec = native_list_append(vec, peak) + vec = native_list_append(vec, zcr) + vec = native_list_append(vec, centroid) + vec = native_list_append(vec, f0) + return vec +} + +// --------------------------------------------------------------------------- +// LPC core: hamming, autocorr, Levinson-Durbin, formant peak-pick, pitch. +// --------------------------------------------------------------------------- + +fn dsp_hamming(x: [Float]) -> [Float] { + let n: Int = native_list_len(x) + if n < 2 { + return x + } + let out: [Float] = native_list_empty() + let pi: Float = math_pi() + let dn: Float = int_to_float(n - 1) + let i: Int = 0 + while i < n { + let v: Float = native_list_get(x, i) + let ang: Float = 2.0 * pi * int_to_float(i) / dn + let cv: Float = math_cos(ang) + let wv: Float = 0.54 - 0.46 * cv + out = native_list_append(out, v * wv) + i = i + 1 + } + return out +} + +// r[lag] = sum_i x[i]*x[i-lag], lag = 0..p. Returns p+1 numbers. +fn dsp_autocorr(x: [Float], p: Int) -> [Float] { + let n: Int = native_list_len(x) + let r: [Float] = native_list_empty() + let lag: Int = 0 + while lag <= p { + let acc: Float = 0.0 + let i: Int = lag + while i < n { + let a: Float = native_list_get(x, i) + let b: Float = native_list_get(x, i - lag) + acc = acc + a * b + i = i + 1 + } + r = native_list_append(r, acc) + lag = lag + 1 + } + return r +} + +// Levinson-Durbin -> LPC coeffs a[0..p] with A(z) = 1 + sum a[k] z^-k. +// Returns p+2 numbers: a[0..p] followed by the residual energy at index p+1. +fn dsp_levinson(r: [Float], p: Int) -> [Float] { + let a: [Float] = native_list_empty() + a = native_list_append(a, 1.0) + let z: Int = 1 + while z <= p { + a = native_list_append(a, 0.0) + z = z + 1 + } + let err: Float = native_list_get(r, 0) + if err <= 0.0 { + a = native_list_append(a, 0.0) + return a + } + let i: Int = 1 + let stopped: Bool = false + while i <= p { + if stopped { + i = p + 1 + } else { + let acc: Float = native_list_get(r, i) + if i > 1 { + let j: Int = 1 + while j < i { + let aj: Float = native_list_get(a, j) + let rij: Float = native_list_get(r, i - j) + acc = acc + aj * rij + j = j + 1 + } + } + let k: Float = (0.0 - acc) / err + // na = a, then na[i] = k, then na[j] = a[j] + k*a[i-j] for 1<=j [Float] { + let peaks: [Float] = native_list_empty() + let p: Int = native_list_len(a) - 1 + if p < 1 { + return peaks + } + let steps: Int = 512 + let stepsf: Float = 512.0 + let srf: Float = int_to_float(sr) + let pi: Float = math_pi() + let mag: [Float] = native_list_empty() + let s: Int = 0 + while s < steps { + let wq: Float = pi * int_to_float(s) / stepsf // 0..pi -> 0..sr/2 + let re: Float = 0.0 + let im: Float = 0.0 + let k: Int = 0 + while k <= p { + let ak: Float = native_list_get(a, k) + let ang: Float = wq * int_to_float(k) + let cv: Float = math_cos(ang) + let sv: Float = math_sin(ang) + re = re + ak * cv + im = im - ak * sv + k = k + 1 + } + let d: Float = math_sqrt(re * re + im * im) + if d < 0.000000001 { + d = 0.000000001 + } + mag = native_list_append(mag, 1.0 / d) + s = s + 1 + } + let found: Int = 0 + let t: Int = 1 + while t < steps - 1 { + if found < 5 { + let m0: Float = native_list_get(mag, t - 1) + let m1: Float = native_list_get(mag, t) + let m2: Float = native_list_get(mag, t + 1) + let rise: Bool = m1 > m0 + let fall: Bool = m1 >= m2 + if rise { + if fall { + let f: Float = int_to_float(t) * srf / 2.0 / stepsf + if f > 150.0 { + if f < 5200.0 { + let thr: Float = m1 / 1.4142 + let lo: Int = t + let scan_lo: Bool = true + while scan_lo { + if lo > 0 { + let mv: Float = native_list_get(mag, lo) + if mv > thr { + lo = lo - 1 + } else { + scan_lo = false + } + } else { + scan_lo = false + } + } + let hi: Int = t + let scan_hi: Bool = true + while scan_hi { + if hi < steps - 1 { + let mv2: Float = native_list_get(mag, hi) + if mv2 > thr { + hi = hi + 1 + } else { + scan_hi = false + } + } else { + scan_hi = false + } + } + let bw: Float = int_to_float(hi - lo) * srf / 2.0 / stepsf + peaks = native_list_append(peaks, f) + peaks = native_list_append(peaks, bw) + found = found + 1 + } + } + } + } + } + t = t + 1 + } + return peaks +} + +// Autocorrelation pitch over 70-400 Hz with the 0.30 voicing threshold. +// Returns 0.0 for an unvoiced (or silent) frame. +fn dsp_pitch_of(frame: [Float], sr: Int) -> Float { + let n: Int = native_list_len(frame) + let lag_min: Int = sr / 400 + let lag_max: Int = sr / 70 + if lag_max > n - 1 { + lag_max = n - 1 + } + if lag_max <= lag_min { + return 0.0 + } + let r0: Float = 0.0 + let i: Int = 0 + while i < n { + let v: Float = native_list_get(frame, i) + r0 = r0 + v * v + i = i + 1 + } + if r0 < 0.00001 { + return 0.0 + } + let best_lag: Int = 0 + let best: Float = 0.0 + let lag: Int = lag_min + while lag <= lag_max { + let c: Float = 0.0 + let j: Int = lag + while j < n { + let a: Float = native_list_get(frame, j) + let b: Float = native_list_get(frame, j - lag) + c = c + a * b + j = j + 1 + } + if c > best { + best = c + best_lag = lag + } + lag = lag + 1 + } + if best_lag > 0 { + let ratio: Float = best / r0 + if ratio > 0.30 { + let srf: Float = int_to_float(sr) + return srf / int_to_float(best_lag) + } + } + return 0.0 +} + +// A copy of x[pos..pos+n) — the El stand-in for Swift's Array(x[a.. [Float] { + let out: [Float] = native_list_empty() + let total: Int = native_list_len(x) + let i: Int = 0 + while i < n { + if pos + i < total { + let v: Float = native_list_get(x, pos + i) + out = native_list_append(out, v) + } + i = i + 1 + } + return out +} + +// median == sorted()[count/2]. There is no list-set in El, so instead of +// sorting we find the value whose rank bracket contains index count/2 — +// numerically identical to the Swift expression, without a mutable buffer. +fn dsp_median(v: [Float]) -> Float { + let n: Int = native_list_len(v) + if n == 0 { + return 0.0 + } + let target: Int = n / 2 + let i: Int = 0 + while i < n { + let x: Float = native_list_get(v, i) + let less: Int = 0 + let eq: Int = 0 + let j: Int = 0 + while j < n { + let y: Float = native_list_get(v, j) + if y < x { + less = less + 1 + } + if y == x { + eq = eq + 1 + } + j = j + 1 + } + if less <= target { + if target < less + eq { + return x + } + } + i = i + 1 + } + return 0.0 +} + +fn dsp_min_of(v: [Float]) -> Float { + let n: Int = native_list_len(v) + if n == 0 { + return 0.0 + } + let m: Float = native_list_get(v, 0) + let i: Int = 1 + while i < n { + let x: Float = native_list_get(v, i) + if x < m { + m = x + } + i = i + 1 + } + return m +} + +fn dsp_max_of(v: [Float]) -> Float { + let n: Int = native_list_len(v) + if n == 0 { + return 0.0 + } + let m: Float = native_list_get(v, 0) + let i: Int = 1 + while i < n { + let x: Float = native_list_get(v, i) + if x > m { + m = x + } + i = i + 1 + } + return m +} + +// --------------------------------------------------------------------------- +// voiceprint — the voice-signature: median F0 over voiced frames + the median +// of each formant F1..F5 (and its bandwidth), plus the F0 min/max range. +// FRAME = 400 (25 ms @ 16k), HOP = 160 (10 ms), LPC order 16. +// +// Returns PACKED: [ f0med, f0lo, f0hi, nf, f1, b1, f2, b2, ... ] where nf is +// how many formants were recovered (0..5). Empty list if unreadable. +// --------------------------------------------------------------------------- +fn dsp_voiceprint(path: String) -> [Float] { + let out: [Float] = native_list_empty() + let w: [Float] = dsp_read_wav(path) + let n: Int = dsp_wav_n(w) + let frame_len: Int = 400 + let hop: Int = 160 + let order: Int = 16 + if n <= frame_len { + return out + } + let sr: Int = dsp_wav_sr(w) + let x: [Float] = dsp_wav_pcm(w) + + let f0s: [Float] = native_list_empty() + // five formant banks + five bandwidth banks, flat lists each + let f1s: [Float] = native_list_empty() + let f2s: [Float] = native_list_empty() + let f3s: [Float] = native_list_empty() + let f4s: [Float] = native_list_empty() + let f5s: [Float] = native_list_empty() + let b1s: [Float] = native_list_empty() + let b2s: [Float] = native_list_empty() + let b3s: [Float] = native_list_empty() + let b4s: [Float] = native_list_empty() + let b5s: [Float] = native_list_empty() + + let pos: Int = 0 + while pos + frame_len <= n { + let raw: [Float] = dsp_slice(x, pos, frame_len) + let f0: Float = dsp_pitch_of(raw, sr) + if f0 > 0.0 { + f0s = native_list_append(f0s, f0) + let win: [Float] = dsp_hamming(raw) + let r: [Float] = dsp_autocorr(win, order) + let r0: Float = native_list_get(r, 0) + if r0 > 0.000001 { + let al: [Float] = dsp_levinson(r, order) + // strip the trailing residual energy: coeffs are a[0..order] + let a: [Float] = native_list_empty() + let ci: Int = 0 + while ci <= order { + let av: Float = native_list_get(al, ci) + a = native_list_append(a, av) + ci = ci + 1 + } + let fs: [Float] = dsp_formants(a, sr) + let nfs: Int = native_list_len(fs) / 2 + if nfs > 0 { + let fv: Float = native_list_get(fs, 0) + let bv: Float = native_list_get(fs, 1) + f1s = native_list_append(f1s, fv) + b1s = native_list_append(b1s, bv) + } + if nfs > 1 { + let fv2: Float = native_list_get(fs, 2) + let bv2: Float = native_list_get(fs, 3) + f2s = native_list_append(f2s, fv2) + b2s = native_list_append(b2s, bv2) + } + if nfs > 2 { + let fv3: Float = native_list_get(fs, 4) + let bv3: Float = native_list_get(fs, 5) + f3s = native_list_append(f3s, fv3) + b3s = native_list_append(b3s, bv3) + } + if nfs > 3 { + let fv4: Float = native_list_get(fs, 6) + let bv4: Float = native_list_get(fs, 7) + f4s = native_list_append(f4s, fv4) + b4s = native_list_append(b4s, bv4) + } + if nfs > 4 { + let fv5: Float = native_list_get(fs, 8) + let bv5: Float = native_list_get(fs, 9) + f5s = native_list_append(f5s, fv5) + b5s = native_list_append(b5s, bv5) + } + } + } + pos = pos + hop + } + + let f0med: Float = dsp_median(f0s) + let f0lo: Float = dsp_min_of(f0s) + let f0hi: Float = dsp_max_of(f0s) + out = native_list_append(out, f0med) + out = native_list_append(out, f0lo) + out = native_list_append(out, f0hi) + + let nf: Int = 0 + if native_list_len(f1s) > 0 { + nf = nf + 1 + } + if native_list_len(f2s) > 0 { + nf = nf + 1 + } + if native_list_len(f3s) > 0 { + nf = nf + 1 + } + if native_list_len(f4s) > 0 { + nf = nf + 1 + } + if native_list_len(f5s) > 0 { + nf = nf + 1 + } + let nff: Float = int_to_float(nf) + out = native_list_append(out, nff) + if native_list_len(f1s) > 0 { + let m: Float = dsp_median(f1s) + let b: Float = dsp_median(b1s) + out = native_list_append(out, m) + out = native_list_append(out, b) + } + if native_list_len(f2s) > 0 { + let m2: Float = dsp_median(f2s) + let bb2: Float = dsp_median(b2s) + out = native_list_append(out, m2) + out = native_list_append(out, bb2) + } + if native_list_len(f3s) > 0 { + let m3: Float = dsp_median(f3s) + let bb3: Float = dsp_median(b3s) + out = native_list_append(out, m3) + out = native_list_append(out, bb3) + } + if native_list_len(f4s) > 0 { + let m4: Float = dsp_median(f4s) + let bb4: Float = dsp_median(b4s) + out = native_list_append(out, m4) + out = native_list_append(out, bb4) + } + if native_list_len(f5s) > 0 { + let m5: Float = dsp_median(f5s) + let bb5: Float = dsp_median(b5s) + out = native_list_append(out, m5) + out = native_list_append(out, bb5) + } + return out +} + +// --------------------------------------------------------------------------- +// imitate — LPC analysis-resynthesis. Per frame: autocorr + Levinson, gain = +// sqrt(residual energy), excitation = an energy-normalized glottal impulse +// train at F0 for voiced frames or white noise for unvoiced, run through the +// all-pole filter using the past-output state. The whole output is normalized +// to peak 0.9 and returned as int16 samples — hand them to speech.el's +// write_wav(samples, sr, path). +// --------------------------------------------------------------------------- +fn dsp_imitate(path: String) -> [Int] { + let res: [Int] = native_list_empty() + let w: [Float] = dsp_read_wav(path) + let n: Int = dsp_wav_n(w) + let frame_len: Int = 400 + let hop: Int = 160 + let order: Int = 16 + if n <= frame_len { + return res + } + let sr: Int = dsp_wav_sr(w) + let srf: Float = int_to_float(sr) + let x: [Float] = dsp_wav_pcm(w) + + let out: [Float] = native_list_empty() + // past outputs, order deep + let state: [Float] = native_list_empty() + let si: Int = 0 + while si < order { + state = native_list_append(state, 0.0) + si = si + 1 + } + let phase: Float = 0.0 + let last_f0: Float = 0.0 + let nstate: Int = 22695 // LCG for the unvoiced source + let written: Int = 0 + + let pos: Int = 0 + while pos + frame_len <= n { + let raw: [Float] = dsp_slice(x, pos, frame_len) + let win: [Float] = dsp_hamming(raw) + let r: [Float] = dsp_autocorr(win, order) + let f0: Float = dsp_pitch_of(raw, sr) + let r0: Float = native_list_get(r, 0) + if r0 < 0.0000001 { + // frame skipped in periph.swift -> those output samples stay zero + let z: Int = 0 + while z < hop { + if written < n { + out = native_list_append(out, 0.0) + written = written + 1 + } + z = z + 1 + } + } else { + let al: [Float] = dsp_levinson(r, order) + let errv: Float = native_list_get(al, order + 1) + let ge: Float = errv + if ge < 0.0 { + ge = 0.0 + } + let gain: Float = math_sqrt(ge) + let use_f0: Float = f0 + if f0 <= 0.0 { + use_f0 = 0.0 + if last_f0 > 0.0 { + use_f0 = last_f0 + } + } + last_f0 = f0 + let i: Int = 0 + while i < hop { + if written < n { + let e: Float = 0.0 + if use_f0 > 0.0 { + phase = phase + use_f0 / srf + if phase >= 1.0 { + phase = phase - 1.0 + e = math_sqrt(srf / use_f0) + } + } else { + nstate = nstate * 1103515245 + 12345 + nstate = nstate - (nstate / 2147483648) * 2147483648 + if nstate < 0 { + nstate = 0 - nstate + } + let u: Float = int_to_float(nstate) / 2147483648.0 + e = u * 2.0 - 1.0 + } + let y: Float = gain * e + let k: Int = 1 + while k <= order { + let ak: Float = native_list_get(al, k) + let sk: Float = native_list_get(state, k - 1) + y = y - ak * sk + k = k + 1 + } + let ns: [Float] = native_list_empty() + ns = native_list_append(ns, y) + let m: Int = 0 + while m < order - 1 { + let sv: Float = native_list_get(state, m) + ns = native_list_append(ns, sv) + m = m + 1 + } + state = ns + out = native_list_append(out, y) + written = written + 1 + } + i = i + 1 + } + } + pos = pos + hop + } + // the tail past the last full frame stays silent, exactly as in periph.swift + while written < n { + out = native_list_append(out, 0.0) + written = written + 1 + } + + // normalize to peak 0.9, then to int16 + let peak: Float = 0.0 + let q: Int = 0 + while q < n { + let v: Float = native_list_get(out, q) + let av: Float = dsp_fabs(v) + if av > peak { + peak = av + } + q = q + 1 + } + let scale: Float = 1.0 + if peak > 0.000000001 { + scale = 0.9 / peak + } + let t: Int = 0 + while t < n { + let v2: Float = native_list_get(out, t) + let sv2: Float = v2 * scale * 32767.0 + if sv2 > 32767.0 { + sv2 = 32767.0 + } + if sv2 < 0.0 - 32767.0 { + sv2 = 0.0 - 32767.0 + } + let iv: Int = float_to_int(sv2) + res = native_list_append(res, iv) + t = t + 1 + } + return res +}