organ: el speaks — the peripheral becomes a capability of the language #159
@@ -0,0 +1,6 @@
|
||||
|
||||
# organ: local device state and its own engram store — never production's
|
||||
peripheral/.consent.json
|
||||
peripheral/.resume.json
|
||||
peripheral/.engram/
|
||||
peripheral/organ
|
||||
@@ -314,6 +314,25 @@ el_val_t speaker_played_frames(void) {
|
||||
return (el_val_t)g_aq_state->pos;
|
||||
}
|
||||
|
||||
/* Pause where we are, keeping the queue and its position intact.
|
||||
*
|
||||
* This is the difference between barge-in and "finish the buffer". The moment
|
||||
* the microphone hears speech, output must stop AT THAT SAMPLE — a listener
|
||||
* experiences even 200ms of continued talking as being talked over. Pause
|
||||
* rather than stop because the interruption might turn out to be a backchannel
|
||||
* ("mm-hm"), and the right response to a backchannel is to carry on as though
|
||||
* nothing happened, which requires the queue to still be exactly where it was.
|
||||
* A stop-and-restart would re-attack the buffer and be audible as a stutter. */
|
||||
el_val_t speaker_pause(void) {
|
||||
if (!g_aq) return (el_val_t)0;
|
||||
return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0);
|
||||
}
|
||||
|
||||
el_val_t speaker_resume(void) {
|
||||
if (!g_aq) return (el_val_t)0;
|
||||
return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0);
|
||||
}
|
||||
|
||||
el_val_t speaker_stop(void) {
|
||||
if (!g_aq) return (el_val_t)0;
|
||||
/* immediate: do NOT let the queue finish what it is holding */
|
||||
@@ -322,6 +341,131 @@ el_val_t speaker_stop(void) {
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
/* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer.
|
||||
* Returns frames, or 0 on any failure; *out is set only on success. Shared by
|
||||
* the blocking and async WAV paths. */
|
||||
static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) {
|
||||
if (!path || !out) return 0;
|
||||
FILE* f = fopen(path, "rb");
|
||||
if (!f) return 0;
|
||||
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; }
|
||||
long size = ftell(f);
|
||||
if (size <= 44) { fclose(f); return 0; }
|
||||
rewind(f);
|
||||
unsigned char* d = (unsigned char*)malloc((size_t)size);
|
||||
if (!d) { fclose(f); return 0; }
|
||||
size_t got = fread(d, 1, (size_t)size, f);
|
||||
fclose(f);
|
||||
if (got != (size_t)size) { free(d); return 0; }
|
||||
if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; }
|
||||
|
||||
int32_t sr = 0, channels = 0, bits = 0;
|
||||
long dataOff = -1, dataLen = 0, o = 12;
|
||||
/* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders
|
||||
* routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads
|
||||
* padding as audio. */
|
||||
while (o + 8 <= size) {
|
||||
long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24);
|
||||
if (sz < 0) break;
|
||||
if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) {
|
||||
channels = (int32_t)(d[o+10] | (d[o+11] << 8));
|
||||
sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24));
|
||||
bits = (int32_t)(d[o+22] | (d[o+23] << 8));
|
||||
} else if (memcmp(d + o, "data", 4) == 0) {
|
||||
dataOff = o + 8;
|
||||
dataLen = sz;
|
||||
if (dataOff + dataLen > size) dataLen = size - dataOff;
|
||||
}
|
||||
o += 8 + sz + (sz & 1);
|
||||
}
|
||||
if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; }
|
||||
|
||||
long frames = dataLen / (2 * channels);
|
||||
int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t));
|
||||
if (!pcm) { free(d); return 0; }
|
||||
for (long i = 0; i < frames; i++) {
|
||||
long b = dataOff + i * 2 * channels;
|
||||
pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8));
|
||||
}
|
||||
free(d);
|
||||
*out = pcm;
|
||||
if (out_sr) *out_sr = sr;
|
||||
return (int64_t)frames;
|
||||
}
|
||||
|
||||
/* Async WAV playback. converse speaks PRE-RENDERED segments and must keep
|
||||
* listening while it does, so it needs the file on the queue without blocking
|
||||
* and needs to be able to stop it mid-buffer. Going through the file rather
|
||||
* than an El [Int] also avoids marshalling a million-element list per segment
|
||||
* for audio the caller never intends to look at. */
|
||||
el_val_t speaker_play_wav_async(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
if (!p) return (el_val_t)0;
|
||||
|
||||
el_audio_teardown();
|
||||
|
||||
int32_t sr = 0;
|
||||
int16_t* pcm = NULL;
|
||||
int64_t frames = el_wav_load(p, &pcm, &sr);
|
||||
if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; }
|
||||
|
||||
g_aq_pcm = pcm;
|
||||
g_aq_sr = sr;
|
||||
g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState));
|
||||
if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; }
|
||||
g_aq_state->pcm = g_aq_pcm;
|
||||
g_aq_state->frames = frames;
|
||||
|
||||
AudioStreamBasicDescription fmt;
|
||||
memset(&fmt, 0, sizeof(fmt));
|
||||
fmt.mSampleRate = (Float64)sr;
|
||||
fmt.mFormatID = kAudioFormatLinearPCM;
|
||||
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
|
||||
fmt.mFramesPerPacket = 1;
|
||||
fmt.mChannelsPerFrame = 1;
|
||||
fmt.mBitsPerChannel = 16;
|
||||
fmt.mBytesPerFrame = 2;
|
||||
fmt.mBytesPerPacket = 2;
|
||||
|
||||
if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) {
|
||||
el_audio_teardown();
|
||||
return (el_val_t)0;
|
||||
}
|
||||
for (int i = 0; i < EL_AQ_NBUF; i++) {
|
||||
int64_t remain = g_aq_state->frames - g_aq_state->pos;
|
||||
if (remain <= 0) break;
|
||||
AudioQueueBufferRef b = NULL;
|
||||
if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break;
|
||||
int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
|
||||
memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t));
|
||||
b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t));
|
||||
g_aq_state->pos += k;
|
||||
if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break;
|
||||
g_aq_state->inflight++;
|
||||
}
|
||||
if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; }
|
||||
if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; }
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
/* Total frames and sample rate of a WAV, without playing it — wav-info, and the
|
||||
* duration converse needs to compute progress through a segment. */
|
||||
el_val_t wav_frames(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
int16_t* pcm = NULL; int32_t sr = 0;
|
||||
int64_t n = el_wav_load(p, &pcm, &sr);
|
||||
free(pcm);
|
||||
return (el_val_t)n;
|
||||
}
|
||||
|
||||
el_val_t wav_rate(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
int16_t* pcm = NULL; int32_t sr = 0;
|
||||
int64_t n = el_wav_load(p, &pcm, &sr);
|
||||
free(pcm);
|
||||
return (el_val_t)(n > 0 ? sr : 0);
|
||||
}
|
||||
|
||||
/* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows
|
||||
* how to write a WAV and a caller may reasonably want to hear one back without
|
||||
* re-rendering it; the parse is deliberately minimal and chunk-walking, so the
|
||||
|
||||
@@ -42,10 +42,23 @@ el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) {
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
el_val_t speaker_play_wav_async(el_val_t path) {
|
||||
(void)path;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
el_val_t speaker_pause(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_resume(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_playing(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_stop(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_played_frames(void) { return (el_val_t)0; }
|
||||
|
||||
/* WAV geometry is pure parsing and would work fine here, but reporting a
|
||||
* duration for audio this build cannot play would invite a caller to sequence
|
||||
* around a silence. Refuse consistently with the rest of the file. */
|
||||
el_val_t wav_frames(el_val_t path) { (void)path; return (el_val_t)0; }
|
||||
el_val_t wav_rate(el_val_t path) { (void)path; return (el_val_t)0; }
|
||||
|
||||
/* ── Microphone ──────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t mic_available(void) { return (el_val_t)0; }
|
||||
|
||||
@@ -126,10 +126,18 @@ el_val_t speaker_play_wav(el_val_t path); /* 16-bit mono RIFF/WAVE; 1 o
|
||||
* the hardware is still going; speaker_played_frames() is how far it actually
|
||||
* got, which is what makes an interrupted utterance resumable at the sample. */
|
||||
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate);
|
||||
el_val_t speaker_play_wav_async(el_val_t path);
|
||||
el_val_t speaker_pause(void); /* stop AT THIS SAMPLE, keep position */
|
||||
el_val_t speaker_resume(void); /* carry on from exactly there */
|
||||
el_val_t speaker_playing(void);
|
||||
el_val_t speaker_stop(void);
|
||||
el_val_t speaker_played_frames(void);
|
||||
|
||||
/* WAV geometry without playing — wav-info, and the segment duration converse
|
||||
* needs to turn elapsed time into progress. */
|
||||
el_val_t wav_frames(el_val_t path);
|
||||
el_val_t wav_rate(el_val_t path);
|
||||
|
||||
/* Microphone (afferent). Fails CLOSED: returns 0 unless the OS has granted
|
||||
* capture access. mic_capture_pcm16 blocks for `seconds` and returns an [Int]
|
||||
* of 16-bit mono samples at `sample_rate` — the raw stream is handed to El and
|
||||
|
||||
+165
-61
@@ -1,80 +1,184 @@
|
||||
# peripheral — Neuron's I/O organ (own-core, local, consent-gated)
|
||||
# peripheral — Neuron's I/O organ, in El
|
||||
|
||||
The interface made physical. Two afferent senses in, one efferent voice out —
|
||||
all reached the way the agentic surface reaches any tool.
|
||||
**El speaks.** The engram stores geometry and does not speak; the speaking
|
||||
belongs to the language and its runtime.
|
||||
|
||||
Until this landed, the organ was a 939-line Swift program (`src/periph.swift`)
|
||||
that shelled out to `afplay`. Neuron's mouth and ears were a separate binary
|
||||
standing next to the language, and "speak" meant "ask that binary to speak."
|
||||
That program is now **reference material, not the implementation.**
|
||||
|
||||
```
|
||||
MIC (hear) afferent device -> capture -> descriptor -> ingest -> geometry
|
||||
CAMERA (see) afferent device -> capture -> descriptor -> ingest -> scene-geometry
|
||||
SPEAKER(speak) efferent render WAV -> PLAY ALOUD out the speaker
|
||||
SPEAKER (speak) efferent samples ──────────────► CoreAudio ──► the room
|
||||
MIC (hear) afferent device ──► samples ──► descriptor ──► engram
|
||||
CAMERA (see) afferent device ──► frame ──► descriptor ──► engram
|
||||
```
|
||||
|
||||
Closes the conversational loop: **hear (mic) -> understand (engram) -> speak (speaker)**.
|
||||
## The split, and why it falls where it does
|
||||
|
||||
Exactly **two** things here are not El, and they are the two things El cannot
|
||||
express as arithmetic:
|
||||
|
||||
| Not El (realizers) | Why |
|
||||
|---|---|
|
||||
| `lang/runtime/el_audio_darwin.m` | Handing a buffer to the DAC and waiting for it to drain. There is no way to say "the hardware has now played these samples" in El, and there should not be. |
|
||||
| `lang/runtime/el_capture_darwin.m` | Asking the OS for samples off a microphone or frames off a camera, plus the TCC permission dance. |
|
||||
|
||||
**Everything else is El**, because everything else is arithmetic:
|
||||
|
||||
| In El | Where |
|
||||
|---|---|
|
||||
| WAV encode / decode (chunk-walking, JUNK/FLLR tolerant) | `src/organ_dsp.el`, `elp/src/speech.el` |
|
||||
| LPC autocorrelation + Levinson-Durbin (order 16 @ 16 kHz) | `src/organ_dsp.el` |
|
||||
| Formant extraction off the all-pole spectral envelope | `src/organ_dsp.el` |
|
||||
| Source-filter resynthesis (glottal impulse train through the filter) | `src/organ_dsp.el` |
|
||||
| Audio descriptor `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` | `src/organ_dsp.el` |
|
||||
| Voice descriptor `[F0, F1..F5, bandwidths]` | `src/organ_dsp.el` |
|
||||
| Scene descriptor `[w, h, meanRGB, brightness, 3×3 luminance grid]` | `src/organ.el` |
|
||||
| Consent, disclosure, the voice-from-engram fetch | `src/organ.el` |
|
||||
| Barge-in, yield-or-hold, backchannel, resume | `src/organ_converse.el` |
|
||||
| The command surface | `src/organ_cli.el` |
|
||||
|
||||
Both realizers are their **own translation units**, declared in
|
||||
`lang/runtime/el_runtime.h`, and deliberately **not** patches to
|
||||
`el_runtime.c`. Acquiring a device must not mean editing the middle of the
|
||||
language — the same rule the realizer registry follows for modalities.
|
||||
`lang/runtime/el_peripheral_null.c` provides the identical entry points
|
||||
everywhere else, so El that speaks links on any platform and truthfully reports
|
||||
having no speaker rather than going quietly silent.
|
||||
|
||||
## The voice comes from the engram
|
||||
|
||||
A voice is **geometry in the engram**, not a JSON file next to the code and
|
||||
certainly not constants in a source file. The organ fetches it the way anything
|
||||
retrieves a memory — it asks:
|
||||
|
||||
```el
|
||||
let g: [Int] = organ_voice_fetch("will")
|
||||
// [peripheral] VOICE: fetched 'will' FROM THE ENGRAM —
|
||||
// f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531
|
||||
```
|
||||
|
||||
`organ_voice_fetch` issues an engram query and reads the geometry off the node
|
||||
that comes back. Nothing opens a file. If the region is not in the graph it
|
||||
returns **empty**, not a plausible default — a caller has to be able to tell
|
||||
"this is how they sound" from "I never heard them."
|
||||
|
||||
The reverse direction is `ingest-voice`: an LPC voiceprint becomes a node, and
|
||||
from then on the voice is a memory rather than a measurement someone wrote down.
|
||||
|
||||
## What the organ never does
|
||||
|
||||
**It never learns a word.** Pronunciation, vocabulary and phonemes belong to the
|
||||
language faculty and are already built as ingested geometry — *the engram knows
|
||||
how to pronounce*. The seam is `synth_codes(codes, voice, pmap)`: the codes and
|
||||
the phoneme map arrive from the language side as geometry, and the organ's whole
|
||||
job is turning them into samples and getting the samples out the speaker, plus
|
||||
the same trip in reverse for the senses. There is no lexicon here and no
|
||||
grapheme-to-phoneme rule, by design.
|
||||
|
||||
## Rails
|
||||
- **Own-core.** macOS-native only: AVFoundation (camera/mic), CoreAudio voice-
|
||||
processing (AEC), afplay (speaker), ImageIO/CoreGraphics (frames), hand-rolled
|
||||
DSP (WAV, LPC, formant synthesis). No cloud, no heavy deps.
|
||||
- **Local-only.** Raw streams are written to `out/` and never egress. `.gitignore`
|
||||
keeps captured media out of git.
|
||||
- **Consent-gated (two locks).** A Neuron-level grant (`grant`/`revoke`) *and* the
|
||||
OS TCC permission. Sensitive senses (camera/mic) fail closed without both.
|
||||
- **Disclosed.** Every device touch prints a `[peripheral]` line on stderr.
|
||||
|
||||
- **Own-core.** CoreAudio / AVFoundation / ImageIO — all ship with macOS. No
|
||||
cloud, no model, no heavy dependency. There is **no network code in the organ
|
||||
at all**, by construction.
|
||||
- **Local-only.** Raw streams stay on the machine. What leaves a capture is a
|
||||
descriptor of a few dozen numbers. A 1920×1080 frame becomes 15 integers
|
||||
(~414,000× smaller); three seconds of audio becomes 8.
|
||||
- **Consent, two locks.** A Neuron-level grant **and** the OS TCC permission.
|
||||
Camera and mic **fail closed** without both. The speaker is disclosed but not
|
||||
gated — you cannot secretly speak aloud, and gating it would mean Neuron needs
|
||||
permission to answer.
|
||||
- **Disclosed.** Every device touch prints a `[peripheral]` line on **stderr**
|
||||
(via `eprintln`, flushed immediately), so a disclosure lands before the device
|
||||
is touched and never contaminates the program's stdout.
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
./peripheral/build.sh /tmp/organ
|
||||
```
|
||||
swiftc -O -o bin/periph src/periph.swift \
|
||||
-framework AVFoundation -framework CoreMedia -framework Foundation \
|
||||
-framework CoreGraphics -framework ImageIO -framework CoreImage
|
||||
```
|
||||
|
||||
Concatenates the El modules, compiles with `elc`, links the two realizers.
|
||||
Run it **from the repo root** or the `.psv` phoneme data will not resolve.
|
||||
|
||||
## Commands
|
||||
|
||||
```
|
||||
periph grant|revoke <camera|mic> # Neuron-level consent
|
||||
periph status
|
||||
periph speak <file.wav> # SPEAK ALOUD (efferent)
|
||||
periph tone <out.wav> [hz] [sec] # own-core WAV synth
|
||||
periph listen <sec> <out.wav> # MIC capture (afferent), 16k mono
|
||||
periph see <out.jpg> # CAMERA one frame (afferent)
|
||||
periph feat-audio <wav> | feat-image <jpg> # capture -> compact descriptor
|
||||
periph ingest-audio|ingest-image <file> <engramURL> # descriptor -> engram node (geometry)
|
||||
periph voiceprint <voice.wav> # extract F0 + formants F1-F5
|
||||
periph imitate <voice.wav> <out.wav> # speak back in that voice (LPC resynthesis)
|
||||
periph hear-imitate <sec> <out.wav> # MIC -> signature -> imitate -> SPEAK ALOUD
|
||||
periph converse <manifest.json> [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] [--live-mic]
|
||||
organ grant|revoke <camera|mic> Neuron-level consent
|
||||
organ status consent + device state
|
||||
organ speak <file.wav> play a WAV aloud (efferent)
|
||||
organ tone [hz] [ms] synthesize and play — no file at all
|
||||
organ say <voice> <CODE> [CODE...] fetch voice FROM THE ENGRAM, render, speak
|
||||
organ listen <sec> <out.wav> mic capture 16k mono (afferent)
|
||||
organ see <out.jpg> one camera frame (afferent)
|
||||
organ wav-info <file.wav> WAV geometry
|
||||
organ feat-audio <file.wav> compact audio descriptor (8 numbers)
|
||||
organ feat-image compact scene-geometry from the camera
|
||||
organ voiceprint <voice.wav> F0 + formants F1-F5 (LPC)
|
||||
organ imitate <in.wav> <out.wav> LPC analysis-resynthesis
|
||||
organ hear-imitate <sec> <out.wav> mic -> signature -> imitate -> speak aloud
|
||||
organ ingest-audio <file.wav> descriptor -> engram node (geometry)
|
||||
organ ingest-voice <voice.wav> <n> voiceprint -> engram voice region
|
||||
organ converse <manifest.json> [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume]
|
||||
```
|
||||
|
||||
## The afferent metabolism
|
||||
A capture is never shipped raw. It becomes a **compact descriptor** — the afferent
|
||||
twin of the music instrument-signature:
|
||||
- audio -> `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` (~2400-6000x smaller)
|
||||
- image -> `[w, h, meanRGB, brightness, 3x3 luminance grid]` (~400000x smaller)
|
||||
- voice -> `[F0, F1..F5, bandwidths]` (11 numbers)
|
||||
## Interruptibility
|
||||
|
||||
That descriptor is what the ingest organ (engram `POST /api/nodes`) turns into an
|
||||
embedded node = geometry.
|
||||
`converse` speaks an ordered, salience-tagged **meaning-plan** while listening:
|
||||
|
||||
## Voice by imitation
|
||||
`voiceprint`/`imitate` are own-core LPC (autocorrelation + Levinson-Durbin, order
|
||||
16 @ 16 kHz), formant extraction from the LPC spectral envelope, and source-filter
|
||||
resynthesis (glottal impulse train at F0 through the all-pole formant filter). A
|
||||
voice is grabbed by ear as ~a dozen numbers and spoken back — **no training, no
|
||||
stolen voice.** Measured fidelity on real speech: resynthesized formants match the
|
||||
source within 2-3%. The full phoneme->formant path for *novel* sentences is the
|
||||
speech faculty's seam (`elp` audio surface profile); this engine provides the
|
||||
formant synthesis primitive it renders through.
|
||||
- **barge-in** — output stops at the sample, not at the end of the buffer. The
|
||||
realizer exposes `pause`/`resume` and reports `played_frames` (the real DAC
|
||||
position) precisely so this is possible.
|
||||
- **yield-or-hold** — a decision, not a rule: `hold = salience·0.6 +
|
||||
progress·0.4`, and holding also requires that the interrupter not be
|
||||
high-authority. Otherwise yield, because the polite default is the right one.
|
||||
- **backchannel** — "mm-hm" is brief and low-energy; resume seamlessly.
|
||||
- **resumable** — on yield the remaining plan persists to `.resume.json`;
|
||||
`--resume` picks the thread back up. An interruption should cost a turn, not
|
||||
the content.
|
||||
|
||||
## Interruptibility (native turn-taking)
|
||||
`converse` plays the utterance as an ordered, salience-tagged **meaning-plan**
|
||||
while the mic listens (full-duplex, AEC on so it never barges in on its own voice):
|
||||
- **barge-in**: user speech -> pause on the spot (sample-accurate), not "finish the buffer."
|
||||
- **yield-or-hold**: a decision grounded in the current segment's salience + progress
|
||||
+ the interrupter's authority — YIELD (stop) or HOLD ("hang on, let me finish").
|
||||
- **backchannel** ("mm-hm"): brief/low -> keep going, resume seamlessly.
|
||||
- **resumable**: on yield the remaining plan persists (`.resume.json`); `--resume`
|
||||
picks the thread back up ("as I was saying").
|
||||
Live full-duplex uses `--live-mic` with the OS voice-processing unit (AEC) so
|
||||
Neuron does not barge in on its own voice. `--barge-at` injects the event
|
||||
deterministically for testing.
|
||||
|
||||
Live full-duplex uses `--live-mic` (OS AEC). Injected `--barge-at` drives the
|
||||
decision loop deterministically for testing.
|
||||
```
|
||||
```
|
||||
## Measured against the Swift original
|
||||
|
||||
Same input (`out/mic_room.wav`, 16 kHz mono, 48121 samples), Swift `periph`
|
||||
vs the El organ:
|
||||
|
||||
| | Swift | El |
|
||||
|---|---|---|
|
||||
| seconds | 3.0075625 | 3.0076 |
|
||||
| rms | 0.0047766496761 | 0.004777 |
|
||||
| peak | 0.01806640625 | 0.018066 |
|
||||
| zcr_hz | 416.28395087 | 416.2840 |
|
||||
| centroid_hz | 727.60529169 | 727.6053 |
|
||||
| f0_hz | 400 | 400.0000 |
|
||||
| formants F1–F5 | 1734.375 / 3343.75 / 3875 / 4359.375 / 4468.75 | identical |
|
||||
| bandwidths B1–B5 | 2000 / 2968.75 / 4203.125 / 4687.5 / 5000 | identical |
|
||||
|
||||
Agreement to every printed digit. `imitate` cannot match bit-for-bit because the
|
||||
Swift excites unvoiced frames with `Double.random` — two Swift runs correlate
|
||||
0.957 with **each other**; El correlates **0.958** with Swift. The port is as
|
||||
close to the original as the original is to itself, and the deterministic prefix
|
||||
is bit-identical.
|
||||
|
||||
## Honest status
|
||||
|
||||
- **Works:** speaker (CoreAudio, no `afplay`, no subprocess — verified: zero
|
||||
`afplay`/Swift strings in the binary, no child process during playback), mic
|
||||
capture, camera capture, all descriptors, LPC voiceprint, imitate,
|
||||
hear-imitate, voice fetch/ingest against the engram, converse (yield, hold,
|
||||
yield-to-authority, backchannel, resume — all exercised with real audio).
|
||||
- **Coarse, and labelled so:** a fetched voice is one formant triple with no
|
||||
coarticulation and no prosody. It is an impression, explicitly **not a
|
||||
clone**, and `prov=COARSE` says so on the node.
|
||||
- **Not verified here:** live `--live-mic` barge-in in a real room with a real
|
||||
interrupter. The AEC path is implemented and the deterministic path is proven;
|
||||
the acoustic behaviour is not something a headless run can establish.
|
||||
- **Not in the engram yet:** the structured `Voice` / `VowelTarget` geometry
|
||||
nodes live in the organ's own store and in snapshot files from earlier work,
|
||||
but the **production engram does not carry them**. Getting them there is an
|
||||
ingest, not a code change.
|
||||
- `src/periph.swift` is kept as the reference the port was measured against.
|
||||
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# build.sh — build the El organ.
|
||||
#
|
||||
# El has no import system on this path, so the modules are concatenated in
|
||||
# dependency order (the same thing elp/tests/run.sh does) and handed to elc as
|
||||
# one unit. The two device realizers are then linked in.
|
||||
#
|
||||
# MUST be run from the repo root, or the .psv phoneme geometry will not resolve
|
||||
# and the render silently degrades.
|
||||
set -uo pipefail
|
||||
|
||||
OUT="${1:-./peripheral/organ}"
|
||||
REPO="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO"
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
# Dependency order. The elp modules supply the render (synth_codes) and the
|
||||
# phoneme-geometry read; the organ supplies everything else.
|
||||
cat elp/src/voice-profile.el \
|
||||
elp/src/accent.el \
|
||||
elp/src/voice-ingest.el \
|
||||
elp/src/speech-ingest.el \
|
||||
elp/src/speech.el \
|
||||
peripheral/src/organ.el \
|
||||
peripheral/src/organ_dsp.el \
|
||||
peripheral/src/organ_converse.el \
|
||||
peripheral/src/organ_cli.el \
|
||||
| grep -v '^import ' > "$WORK/organ.el"
|
||||
|
||||
cd "$REPO/lang"
|
||||
./dist/platform/elc "$WORK/organ.el" > "$WORK/organ.c" || { echo "elc failed" >&2; exit 1; }
|
||||
|
||||
SSL_PREFIX="$(brew --prefix openssl@3 2>/dev/null || echo /usr/local)"
|
||||
|
||||
# The peripheral realizers are per-platform: Darwin gets the real devices,
|
||||
# anything else gets el_peripheral_null.c and honestly reports having none.
|
||||
case "$(uname)" in
|
||||
Darwin)
|
||||
# The Objective-C realizers are compiled SEPARATELY, with -fobjc-arc. The
|
||||
# capture realizer is written against ARC (it holds AVFoundation objects);
|
||||
# compiling it MRR silently changes its memory semantics, which on a device
|
||||
# path shows up as a use-after-free under load rather than as an error here.
|
||||
cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_audio_darwin.m -o "$WORK/el_audio.o" || exit 1
|
||||
cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_capture_darwin.m -o "$WORK/el_capture.o" || exit 1
|
||||
PERIPH_SRC="$WORK/el_audio.o $WORK/el_capture.o"
|
||||
PERIPH_LIBS="-framework AudioToolbox -framework AVFoundation -framework CoreMedia
|
||||
-framework CoreVideo -framework CoreGraphics -framework ImageIO
|
||||
-framework Foundation"
|
||||
;;
|
||||
*)
|
||||
PERIPH_SRC="runtime/el_peripheral_null.c"
|
||||
PERIPH_LIBS=""
|
||||
;;
|
||||
esac
|
||||
|
||||
cc -O1 -I runtime -I"$SSL_PREFIX/include" -L"$SSL_PREFIX/lib" \
|
||||
-o "$OUT" "$WORK/organ.c" \
|
||||
runtime/el_runtime.c runtime/el_seed.c \
|
||||
runtime/engram_cognition.c runtime/engram_geometry.c runtime/engram_reason.c \
|
||||
runtime/engram_store.c runtime/engram_verify.c runtime/engram_vindex.c \
|
||||
runtime/eg_cosine_batch.c runtime/eg_cosine_batch_strategy_cpu.c \
|
||||
$PERIPH_SRC $PERIPH_LIBS \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm || { echo "link failed" >&2; exit 1; }
|
||||
|
||||
echo "built: $OUT"
|
||||
+195
-3
@@ -333,12 +333,204 @@ fn organ_voice_ingest(name: String, f0: Int, f0_end: Int, kf: Int, f1: Int, f2:
|
||||
// Turn the fetched geometry into the voice slot-map the render consumes. Kept
|
||||
// separate from the fetch so the organ never invents a voice: if the fetch came
|
||||
// back empty this returns empty too, and the caller has to deal with it.
|
||||
//
|
||||
// The slot-map is built here rather than by calling the render's own
|
||||
// constructor, so the organ carries NO dependency on the language faculty's
|
||||
// modules — it only has to agree with them about a wire format, which is the
|
||||
// looser and more honest coupling. (The layout is the same key/value [String]
|
||||
// convention lang_get / surface_get / voice_get all read.)
|
||||
fn organ_voice_profile(name: String, g: [Int]) -> [String] {
|
||||
let empty: [String] = native_list_empty()
|
||||
let r: [String] = native_list_empty()
|
||||
if native_list_len(g) < 6 {
|
||||
return empty
|
||||
return r
|
||||
}
|
||||
return voice_new(name, native_list_get(g, 0), native_list_get(g, 1), native_list_get(g, 2), 1000, 1000, 8)
|
||||
r = native_list_append(r, "name")
|
||||
r = native_list_append(r, name)
|
||||
r = native_list_append(r, "f0")
|
||||
r = native_list_append(r, int_to_str(native_list_get(g, 0)))
|
||||
r = native_list_append(r, "f0_end")
|
||||
r = native_list_append(r, int_to_str(native_list_get(g, 1)))
|
||||
r = native_list_append(r, "kf")
|
||||
r = native_list_append(r, int_to_str(native_list_get(g, 2)))
|
||||
r = native_list_append(r, "dur")
|
||||
r = native_list_append(r, "1000")
|
||||
r = native_list_append(r, "tilt")
|
||||
r = native_list_append(r, "1000")
|
||||
r = native_list_append(r, "breath")
|
||||
r = native_list_append(r, "8")
|
||||
return r
|
||||
}
|
||||
|
||||
// ── Scene geometry (afferent, camera) ────────────────────────────────────────
|
||||
//
|
||||
// The image half of the afferent metabolism, and the same principle as the
|
||||
// audio descriptor: a frame is never handed on raw. The realizer returns a
|
||||
// small pixel grid; THIS computes the descriptor, in El, because averaging
|
||||
// pixels is arithmetic and arithmetic is not a device concern.
|
||||
//
|
||||
// Returns 15 numbers — [w, h, meanR, meanG, meanB, brightness_pm, and a 3x3
|
||||
// luminance grid] — standing in for a multi-megapixel frame. The 3x3 grid is
|
||||
// the smallest thing that still says WHERE the light is, which is most of what
|
||||
// makes a scene comparable to another scene; a single brightness average would
|
||||
// make a lamp on the left indistinguishable from a lamp on the right.
|
||||
//
|
||||
// Luminance is Rec. 601 (0.299R + 0.587G + 0.114B), in integer per-mille, so
|
||||
// the descriptor is reproducible rather than subject to float drift.
|
||||
fn organ_image_descriptor() -> [Int] {
|
||||
let out: [Int] = native_list_empty()
|
||||
let frame: Any = camera_capture_rgb()
|
||||
if frame == 0 {
|
||||
return out
|
||||
}
|
||||
let w: Int = el_map_get(frame, "width")
|
||||
let h: Int = el_map_get(frame, "height")
|
||||
let gw: Int = el_map_get(frame, "grid_w")
|
||||
let gh: Int = el_map_get(frame, "grid_h")
|
||||
let px: [Int] = el_map_get(frame, "pixels")
|
||||
let np: Int = native_list_len(px)
|
||||
if np < 3 {
|
||||
return out
|
||||
}
|
||||
let count: Int = np / 3
|
||||
let rsum: Int = 0
|
||||
let gsum: Int = 0
|
||||
let bsum: Int = 0
|
||||
// 3x3 accumulators, row-major
|
||||
let cell: [Int] = native_list_empty()
|
||||
let cn: [Int] = native_list_empty()
|
||||
let z: Int = 0
|
||||
while z < 9 {
|
||||
cell = native_list_append(cell, 0)
|
||||
cn = native_list_append(cn, 0)
|
||||
z = z + 1
|
||||
}
|
||||
// El has no list-set, so the cells are summed into parallel scalars and
|
||||
// reassembled — nine explicit accumulators would be worse to read than one
|
||||
// pass per cell over a grid this small.
|
||||
let c0: Int = 0
|
||||
let c1: Int = 0
|
||||
let c2: Int = 0
|
||||
let c3: Int = 0
|
||||
let c4: Int = 0
|
||||
let c5: Int = 0
|
||||
let c6: Int = 0
|
||||
let c7: Int = 0
|
||||
let c8: Int = 0
|
||||
let n0: Int = 0
|
||||
let n1: Int = 0
|
||||
let n2: Int = 0
|
||||
let n3: Int = 0
|
||||
let n4: Int = 0
|
||||
let n5: Int = 0
|
||||
let n6: Int = 0
|
||||
let n7: Int = 0
|
||||
let n8: Int = 0
|
||||
let i: Int = 0
|
||||
while i < count {
|
||||
let r: Int = native_list_get(px, i * 3)
|
||||
let g: Int = native_list_get(px, i * 3 + 1)
|
||||
let b: Int = native_list_get(px, i * 3 + 2)
|
||||
rsum = rsum + r
|
||||
gsum = gsum + g
|
||||
bsum = bsum + b
|
||||
let lum: Int = (299 * r + 587 * g + 114 * b) / 1000
|
||||
let x: Int = i - (i / gw) * gw
|
||||
let y: Int = i / gw
|
||||
let cx: Int = x * 3 / gw
|
||||
let cy: Int = y * 3 / gh
|
||||
if cx > 2 {
|
||||
cx = 2
|
||||
}
|
||||
if cy > 2 {
|
||||
cy = 2
|
||||
}
|
||||
let idx: Int = cy * 3 + cx
|
||||
if idx == 0 {
|
||||
c0 = c0 + lum
|
||||
n0 = n0 + 1
|
||||
}
|
||||
if idx == 1 {
|
||||
c1 = c1 + lum
|
||||
n1 = n1 + 1
|
||||
}
|
||||
if idx == 2 {
|
||||
c2 = c2 + lum
|
||||
n2 = n2 + 1
|
||||
}
|
||||
if idx == 3 {
|
||||
c3 = c3 + lum
|
||||
n3 = n3 + 1
|
||||
}
|
||||
if idx == 4 {
|
||||
c4 = c4 + lum
|
||||
n4 = n4 + 1
|
||||
}
|
||||
if idx == 5 {
|
||||
c5 = c5 + lum
|
||||
n5 = n5 + 1
|
||||
}
|
||||
if idx == 6 {
|
||||
c6 = c6 + lum
|
||||
n6 = n6 + 1
|
||||
}
|
||||
if idx == 7 {
|
||||
c7 = c7 + lum
|
||||
n7 = n7 + 1
|
||||
}
|
||||
if idx == 8 {
|
||||
c8 = c8 + lum
|
||||
n8 = n8 + 1
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
let rA: Int = rsum / count
|
||||
let gA: Int = gsum / count
|
||||
let bA: Int = bsum / count
|
||||
let bright: Int = (299 * rA + 587 * gA + 114 * bA) / 255
|
||||
out = native_list_append(out, w)
|
||||
out = native_list_append(out, h)
|
||||
out = native_list_append(out, rA)
|
||||
out = native_list_append(out, gA)
|
||||
out = native_list_append(out, bA)
|
||||
out = native_list_append(out, bright)
|
||||
if n0 < 1 {
|
||||
n0 = 1
|
||||
}
|
||||
if n1 < 1 {
|
||||
n1 = 1
|
||||
}
|
||||
if n2 < 1 {
|
||||
n2 = 1
|
||||
}
|
||||
if n3 < 1 {
|
||||
n3 = 1
|
||||
}
|
||||
if n4 < 1 {
|
||||
n4 = 1
|
||||
}
|
||||
if n5 < 1 {
|
||||
n5 = 1
|
||||
}
|
||||
if n6 < 1 {
|
||||
n6 = 1
|
||||
}
|
||||
if n7 < 1 {
|
||||
n7 = 1
|
||||
}
|
||||
if n8 < 1 {
|
||||
n8 = 1
|
||||
}
|
||||
out = native_list_append(out, c0 / n0)
|
||||
out = native_list_append(out, c1 / n1)
|
||||
out = native_list_append(out, c2 / n2)
|
||||
out = native_list_append(out, c3 / n3)
|
||||
out = native_list_append(out, c4 / n4)
|
||||
out = native_list_append(out, c5 / n5)
|
||||
out = native_list_append(out, c6 / n6)
|
||||
out = native_list_append(out, c7 / n7)
|
||||
out = native_list_append(out, c8 / n8)
|
||||
organ_disclose("FEAT(image): 15-number scene-geometry vs " + int_to_str(w * h * 3) + " pixel-channels — the descriptor travels, the frame does not.")
|
||||
return out
|
||||
}
|
||||
|
||||
// ── Own-core tone ────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
// organ_cli.el — the organ's command surface. main() lives here.
|
||||
//
|
||||
// One binary, the same verbs the Swift program had, and nothing behind them
|
||||
// except El and two thin device realizers. This file is the proof surface: if
|
||||
// `organ speak` makes a sound and no Swift binary is in the process tree, the
|
||||
// claim in organ.el's header is true.
|
||||
//
|
||||
// Verbs, and what each one demonstrates:
|
||||
//
|
||||
// grant/revoke/status the Neuron-level consent lock, inspectable
|
||||
// speak <wav> efferent — audio out of El's own speaker
|
||||
// tone <hz> <ms> own-core synthesis: computed in El, played by El,
|
||||
// never touching the disk
|
||||
// say <name> <codes...> fetch a VOICE FROM THE ENGRAM and render through it
|
||||
// listen <sec> <out> afferent — mic capture, consent-gated, fails closed
|
||||
// see <out.jpg> afferent — one camera frame, same two locks
|
||||
// wav-info <wav> WAV geometry, parsed in El
|
||||
// feat-audio <wav> capture -> compact descriptor (8 numbers)
|
||||
// feat-image <jpg> frame -> compact scene-geometry
|
||||
// voiceprint <wav> F0 + formants F1-F5 by LPC, in El
|
||||
// imitate <in> <out> LPC analysis-resynthesis, in El
|
||||
// hear-imitate <sec> the closed loop: hear a voice, take its signature,
|
||||
// speak back in it
|
||||
// ingest-audio <wav> descriptor -> engram node (the capture becomes geometry)
|
||||
// ingest-voice <wav> <n> voiceprint -> engram voice region (how a voice is learned)
|
||||
// converse <manifest> full-duplex interruptible utterance
|
||||
//
|
||||
// The descriptors are the point of the afferent half. A capture is NEVER handed
|
||||
// on raw: a three-second recording is ~48,000 samples and what leaves this
|
||||
// process is eight numbers. That is both the privacy rail (the stream stays
|
||||
// local because only its shape travels) and the reason the engram can hold a
|
||||
// perception at all — geometry is storable, a waveform is not.
|
||||
|
||||
fn cli_usage() -> Bool {
|
||||
println("organ — Neuron's I/O organ, native El (own-core, local, consent-gated)")
|
||||
println(" grant|revoke <camera|mic> Neuron-level consent")
|
||||
println(" status consent + device state")
|
||||
println(" speak <file.wav> play a WAV aloud (efferent)")
|
||||
println(" tone [hz] [ms] synthesize and play, no file at all")
|
||||
println(" say <voice> <CODE> [CODE...] fetch voice FROM THE ENGRAM, render, speak")
|
||||
println(" listen <sec> <out.wav> mic capture 16k mono (afferent)")
|
||||
println(" see <out.jpg> one camera frame (afferent)")
|
||||
println(" wav-info <file.wav> WAV geometry")
|
||||
println(" feat-audio <file.wav> compact audio descriptor (8 numbers)")
|
||||
println(" feat-image compact scene-geometry from the camera")
|
||||
println(" voiceprint <voice.wav> F0 + formants F1-F5 (LPC)")
|
||||
println(" imitate <in.wav> <out.wav> LPC analysis-resynthesis")
|
||||
println(" hear-imitate <sec> <out.wav> mic -> signature -> imitate -> speak aloud")
|
||||
println(" ingest-audio <file.wav> descriptor -> engram node (geometry)")
|
||||
println(" ingest-voice <voice.wav> <n> voiceprint -> engram voice region")
|
||||
println(" converse <manifest.json> [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume]")
|
||||
return true
|
||||
}
|
||||
|
||||
// The engram the organ reads and writes. Its own store, never production's.
|
||||
fn cli_engram_dir() -> String {
|
||||
let d: String = env("ORGAN_ENGRAM")
|
||||
if str_eq(d, "") {
|
||||
return "peripheral/.engram"
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
fn cli_open_engram() -> Bool {
|
||||
let dir: String = cli_engram_dir()
|
||||
fs_mkdir(dir)
|
||||
let ok: Int = engram_store_boot(dir)
|
||||
if ok == 1 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ── formatting helpers ───────────────────────────────────────────────────────
|
||||
|
||||
fn cli_f(v: Float, dec: Int) -> String {
|
||||
return format_float(v, dec)
|
||||
}
|
||||
|
||||
// ── the descriptor, printed and ingested ─────────────────────────────────────
|
||||
//
|
||||
// [seconds, sr, ch, rms, peak, zcr, centroid, f0] — the same eight numbers the
|
||||
// Swift produced, computed in El, and the compression ratio is the headline:
|
||||
// a few dozen bytes standing in for a few hundred kilobytes.
|
||||
fn cli_audio_descriptor_text(v: [Float], path: String) -> String {
|
||||
let secs: Float = native_list_get(v, 0)
|
||||
let sr: Float = native_list_get(v, 1)
|
||||
let ch: Float = native_list_get(v, 2)
|
||||
let rms: Float = native_list_get(v, 3)
|
||||
let peak: Float = native_list_get(v, 4)
|
||||
let zcr: Float = native_list_get(v, 5)
|
||||
let cen: Float = native_list_get(v, 6)
|
||||
let f0: Float = native_list_get(v, 7)
|
||||
return "Heard sound (afferent, mic): " + cli_f(secs, 2) + "s at " + cli_f(sr, 0) + "Hz. RMS energy " + cli_f(rms, 4) + ", peak " + cli_f(peak, 4) + ", zero-crossing rate " + cli_f(zcr, 0) + "Hz, spectral centroid " + cli_f(cen, 0) + "Hz, estimated voice pitch F0 " + cli_f(f0, 0) + "Hz. Compact voice/sound signature (8 numbers) — phonetic geometry seed."
|
||||
}
|
||||
|
||||
fn cli_feat_audio(path: String) -> Bool {
|
||||
let v: [Float] = dsp_compute_audio(path)
|
||||
if native_list_len(v) < 8 {
|
||||
println("{\"ok\": false, \"op\": \"feat-audio\", \"error\": \"cannot read PCM\"}")
|
||||
return false
|
||||
}
|
||||
organ_disclose("FEAT(audio): 8-number signature vs " + int_to_str(float_to_int(native_list_get(v, 0) * native_list_get(v, 1))) + " raw samples.")
|
||||
println("{\"ok\": true, \"op\": \"feat-audio\", \"file\": \"" + path + "\", \"seconds\": " + cli_f(native_list_get(v, 0), 4) + ", \"sample_rate\": " + cli_f(native_list_get(v, 1), 0) + ", \"channels\": " + cli_f(native_list_get(v, 2), 0) + ", \"rms\": " + cli_f(native_list_get(v, 3), 6) + ", \"peak\": " + cli_f(native_list_get(v, 4), 6) + ", \"zcr_hz\": " + cli_f(native_list_get(v, 5), 4) + ", \"centroid_hz\": " + cli_f(native_list_get(v, 6), 4) + ", \"f0_hz\": " + cli_f(native_list_get(v, 7), 4) + "}")
|
||||
return true
|
||||
}
|
||||
|
||||
fn cli_voiceprint(path: String) -> Bool {
|
||||
let v: [Float] = dsp_voiceprint(path)
|
||||
if native_list_len(v) < 4 {
|
||||
println("{\"ok\": false, \"op\": \"voiceprint\", \"error\": \"cannot read speech\"}")
|
||||
return false
|
||||
}
|
||||
let nf: Int = float_to_int(native_list_get(v, 3))
|
||||
let fs: String = ""
|
||||
let bs: String = ""
|
||||
let i: Int = 0
|
||||
while i < nf {
|
||||
if i > 0 {
|
||||
fs = fs + ", "
|
||||
bs = bs + ", "
|
||||
}
|
||||
fs = fs + cli_f(native_list_get(v, 4 + i * 2), 3)
|
||||
bs = bs + cli_f(native_list_get(v, 5 + i * 2), 3)
|
||||
i = i + 1
|
||||
}
|
||||
println("{\"ok\": true, \"op\": \"voiceprint\", \"file\": \"" + path + "\", \"f0_hz\": " + cli_f(native_list_get(v, 0), 4) + ", \"f0_range\": [" + cli_f(native_list_get(v, 1), 4) + ", " + cli_f(native_list_get(v, 2), 4) + "], \"formants_hz\": [" + fs + "], \"bandwidths_hz\": [" + bs + "]}")
|
||||
return true
|
||||
}
|
||||
|
||||
// ── main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn main() {
|
||||
let a: [String] = args()
|
||||
let n: Int = native_list_len(a)
|
||||
if n < 1 {
|
||||
cli_usage()
|
||||
return
|
||||
}
|
||||
let cmd: String = native_list_get(a, 0)
|
||||
|
||||
// ---- consent -----------------------------------------------------------
|
||||
if str_eq(cmd, "grant") {
|
||||
if n < 2 {
|
||||
println("grant needs a device")
|
||||
return
|
||||
}
|
||||
organ_grant(native_list_get(a, 1))
|
||||
println("{\"ok\": true, \"op\": \"grant\", \"consent\": \"" + organ_consent_status() + "\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "revoke") {
|
||||
if n < 2 {
|
||||
println("revoke needs a device")
|
||||
return
|
||||
}
|
||||
organ_revoke(native_list_get(a, 1))
|
||||
println("{\"ok\": true, \"op\": \"revoke\", \"consent\": \"" + organ_consent_status() + "\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "status") {
|
||||
println("{\"ok\": true, \"op\": \"status\", \"consent\": \"" + organ_consent_status() + "\", \"speaker\": \"" + speaker_name() + "\", \"speaker_available\": " + int_to_str(speaker_available()) + ", \"mic_os_authorized\": " + int_to_str(mic_available()) + ", \"camera_os_authorized\": " + int_to_str(camera_available()) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- efferent ----------------------------------------------------------
|
||||
if str_eq(cmd, "speak") {
|
||||
if n < 2 {
|
||||
println("speak needs a wav")
|
||||
return
|
||||
}
|
||||
let ok: Bool = organ_speak_wav(native_list_get(a, 1))
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"speak\", \"played_aloud\": " + bool_to_str(ok) + "}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "tone") {
|
||||
let hz: Int = 220
|
||||
let ms: Int = 1000
|
||||
if n >= 2 {
|
||||
hz = str_to_int(native_list_get(a, 1))
|
||||
}
|
||||
if n >= 3 {
|
||||
ms = str_to_int(native_list_get(a, 2))
|
||||
}
|
||||
let s: [Int] = organ_tone(hz, ms, 16000)
|
||||
let ok: Bool = organ_speak_samples(s, 16000)
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"tone\", \"hz\": " + int_to_str(hz) + ", \"ms\": " + int_to_str(ms) + ", \"samples\": " + int_to_str(native_list_len(s)) + ", \"file\": null}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- the voice, from the engram ----------------------------------------
|
||||
if str_eq(cmd, "say") {
|
||||
if n < 3 {
|
||||
println("say needs <voice> <CODE> [CODE...]")
|
||||
return
|
||||
}
|
||||
cli_open_engram()
|
||||
let vname: String = native_list_get(a, 1)
|
||||
let g: [Int] = organ_voice_fetch(vname)
|
||||
if native_list_len(g) < 6 {
|
||||
println("{\"ok\": false, \"op\": \"say\", \"error\": \"no voice region '" + vname + "' in the engram\"}")
|
||||
return
|
||||
}
|
||||
// Codes and the phoneme map come from the LANGUAGE side. The organ does
|
||||
// not know what a word is and never looks one up.
|
||||
let pmap: [String] = ingest_phonetics("elp/data/phonetics.psv")
|
||||
let codes: [String] = native_list_empty()
|
||||
let i: Int = 2
|
||||
while i < n {
|
||||
codes = native_list_append(codes, native_list_get(a, i))
|
||||
i = i + 1
|
||||
}
|
||||
let voice: [String] = organ_voice_profile(vname, g)
|
||||
let s: [Int] = synth_codes(codes, voice, pmap)
|
||||
let ok: Bool = organ_speak_samples(s, 16000)
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"say\", \"voice\": \"" + vname + "\", \"f0\": " + int_to_str(native_list_get(g, 0)) + ", \"kf\": " + int_to_str(native_list_get(g, 2)) + ", \"codes\": " + int_to_str(native_list_len(codes)) + ", \"samples\": " + int_to_str(native_list_len(s)) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- afferent ----------------------------------------------------------
|
||||
if str_eq(cmd, "listen") {
|
||||
if n < 3 {
|
||||
println("listen needs <sec> <out.wav>")
|
||||
return
|
||||
}
|
||||
let secs: Int = str_to_int(native_list_get(a, 1))
|
||||
let out: String = native_list_get(a, 2)
|
||||
if organ_may_listen() == false {
|
||||
println("{\"ok\": false, \"op\": \"listen\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
organ_disclose("MIC: capturing " + int_to_str(secs) + "s (16 kHz mono, LOCAL, never egresses).")
|
||||
let s: [Int] = mic_capture_pcm16(secs, 16000)
|
||||
let got: Int = native_list_len(s)
|
||||
if got <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"listen\", \"error\": \"capture returned nothing\"}")
|
||||
return
|
||||
}
|
||||
let ok: Bool = write_wav(s, 16000, out)
|
||||
organ_disclose("MIC: captured " + int_to_str(got) + " frames — ready to hand to the ingest organ.")
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"listen\", \"file\": \"" + out + "\", \"frames\": " + int_to_str(got) + ", \"sample_rate\": 16000}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "see") {
|
||||
if n < 2 {
|
||||
println("see needs an out path")
|
||||
return
|
||||
}
|
||||
if organ_may_see() == false {
|
||||
println("{\"ok\": false, \"op\": \"see\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
organ_disclose("CAMERA: capturing one frame (LOCAL, never egresses).")
|
||||
let ok: Int = camera_capture_jpeg(native_list_get(a, 1))
|
||||
println("{\"ok\": " + int_to_str(ok) + ", \"op\": \"see\", \"file\": \"" + native_list_get(a, 1) + "\"}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- descriptors -------------------------------------------------------
|
||||
if str_eq(cmd, "wav-info") {
|
||||
if n < 2 {
|
||||
println("wav-info needs a wav")
|
||||
return
|
||||
}
|
||||
let p: String = native_list_get(a, 1)
|
||||
let w: [Float] = dsp_read_wav(p)
|
||||
if dsp_wav_n(w) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"wav-info\"}")
|
||||
return
|
||||
}
|
||||
println("{\"ok\": true, \"op\": \"wav-info\", \"sample_rate\": " + int_to_str(dsp_wav_sr(w)) + ", \"channels\": " + int_to_str(dsp_wav_ch(w)) + ", \"frames\": " + int_to_str(dsp_wav_n(w)) + "}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "feat-audio") {
|
||||
if n < 2 {
|
||||
println("feat-audio needs a wav")
|
||||
return
|
||||
}
|
||||
cli_feat_audio(native_list_get(a, 1))
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "feat-image") {
|
||||
if organ_may_see() == false {
|
||||
println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
let f: [Int] = organ_image_descriptor()
|
||||
if native_list_len(f) < 15 {
|
||||
println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"no frame\"}")
|
||||
return
|
||||
}
|
||||
let grid: String = ""
|
||||
let i: Int = 6
|
||||
while i < 15 {
|
||||
if i > 6 {
|
||||
grid = grid + ", "
|
||||
}
|
||||
grid = grid + int_to_str(native_list_get(f, i))
|
||||
i = i + 1
|
||||
}
|
||||
println("{\"ok\": true, \"op\": \"feat-image\", \"width\": " + int_to_str(native_list_get(f, 0)) + ", \"height\": " + int_to_str(native_list_get(f, 1)) + ", \"mean_rgb\": [" + int_to_str(native_list_get(f, 2)) + ", " + int_to_str(native_list_get(f, 3)) + ", " + int_to_str(native_list_get(f, 4)) + "], \"brightness_pm\": " + int_to_str(native_list_get(f, 5)) + ", \"luma_grid\": [" + grid + "]}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "voiceprint") {
|
||||
if n < 2 {
|
||||
println("voiceprint needs a wav")
|
||||
return
|
||||
}
|
||||
cli_voiceprint(native_list_get(a, 1))
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "imitate") {
|
||||
if n < 3 {
|
||||
println("imitate needs <in.wav> <out.wav>")
|
||||
return
|
||||
}
|
||||
let s: [Int] = dsp_imitate(native_list_get(a, 1))
|
||||
if native_list_len(s) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"imitate\"}")
|
||||
return
|
||||
}
|
||||
let ok: Bool = write_wav(s, 16000, native_list_get(a, 2))
|
||||
organ_disclose("IMITATE: rebuilt the voice from its own LPC signature (own-core, no training, no stolen voice).")
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"imitate\", \"out\": \"" + native_list_get(a, 2) + "\", \"samples\": " + int_to_str(native_list_len(s)) + ", \"method\": \"LPC analysis-resynthesis\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "hear-imitate") {
|
||||
if n < 3 {
|
||||
println("hear-imitate needs <sec> <out.wav>")
|
||||
return
|
||||
}
|
||||
let secs: Int = str_to_int(native_list_get(a, 1))
|
||||
let out: String = native_list_get(a, 2)
|
||||
if organ_may_listen() == false {
|
||||
println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
let heard: String = out + ".heard.wav"
|
||||
organ_disclose("HEAR-IMITATE: open the ear, listen " + int_to_str(secs) + "s, take the voice, speak it back.")
|
||||
let s: [Int] = mic_capture_pcm16(secs, 16000)
|
||||
if native_list_len(s) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"capture returned nothing\"}")
|
||||
return
|
||||
}
|
||||
write_wav(s, 16000, heard)
|
||||
let re: [Int] = dsp_imitate(heard)
|
||||
if native_list_len(re) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"could not model the voice\"}")
|
||||
return
|
||||
}
|
||||
write_wav(re, 16000, out)
|
||||
let ok: Bool = organ_speak_samples(re, 16000)
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"hear-imitate\", \"heard\": \"" + heard + "\", \"out\": \"" + out + "\", \"spoke_aloud\": " + bool_to_str(ok) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- the afferent wire: descriptor -> geometry --------------------------
|
||||
if str_eq(cmd, "ingest-audio") {
|
||||
if n < 2 {
|
||||
println("ingest-audio needs a wav")
|
||||
return
|
||||
}
|
||||
let p: String = native_list_get(a, 1)
|
||||
let v: [Float] = dsp_compute_audio(p)
|
||||
if native_list_len(v) < 8 {
|
||||
println("{\"ok\": false, \"op\": \"ingest-audio\"}")
|
||||
return
|
||||
}
|
||||
cli_open_engram()
|
||||
let content: String = cli_audio_descriptor_text(v, p)
|
||||
let id: String = engram_node(content, "Observation", 70)
|
||||
engram_store_checkpoint()
|
||||
organ_disclose("INGEST: the capture is now GEOMETRY in the engram (node " + id + ") — the descriptor travelled, the stream did not.")
|
||||
println("{\"ok\": true, \"op\": \"ingest-audio\", \"node_id\": \"" + id + "\", \"content\": \"" + content + "\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "ingest-voice") {
|
||||
if n < 3 {
|
||||
println("ingest-voice needs <voice.wav> <name>")
|
||||
return
|
||||
}
|
||||
let p: String = native_list_get(a, 1)
|
||||
let name: String = native_list_get(a, 2)
|
||||
let v: [Float] = dsp_voiceprint(p)
|
||||
if native_list_len(v) < 10 {
|
||||
println("{\"ok\": false, \"op\": \"ingest-voice\", \"error\": \"no voiced frames\"}")
|
||||
return
|
||||
}
|
||||
cli_open_engram()
|
||||
let f0: Int = float_to_int(native_list_get(v, 0))
|
||||
let f1: Int = float_to_int(native_list_get(v, 4))
|
||||
let f2: Int = float_to_int(native_list_get(v, 6))
|
||||
let f3: Int = float_to_int(native_list_get(v, 8))
|
||||
// kf is the vocal-tract scale: this speaker's F1 against the nominal
|
||||
// /AA/ F1 of 730 Hz. One number standing for a tract length.
|
||||
let kf: Int = 1000 * f1 / 730
|
||||
let f0e: Int = f0 * 85 / 100
|
||||
let id: String = organ_voice_ingest(name, f0, f0e, kf, f1, f2, f3, "el-organ-lpc-voiceprint", "COARSE")
|
||||
engram_store_checkpoint()
|
||||
println("{\"ok\": true, \"op\": \"ingest-voice\", \"node_id\": \"" + id + "\", \"name\": \"" + name + "\", \"f0\": " + int_to_str(f0) + ", \"kf\": " + int_to_str(kf) + ", \"f1\": " + int_to_str(f1) + ", \"f2\": " + int_to_str(f2) + ", \"f3\": " + int_to_str(f3) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- converse ----------------------------------------------------------
|
||||
if str_eq(cmd, "converse") {
|
||||
if n < 2 {
|
||||
println("converse needs a manifest")
|
||||
return
|
||||
}
|
||||
let mf: String = native_list_get(a, 1)
|
||||
let authority: Int = 500
|
||||
let barge: Int = 0 - 1
|
||||
let kind: String = "bargein"
|
||||
let live: Bool = false
|
||||
let resume: Bool = false
|
||||
let i: Int = 2
|
||||
while i < n {
|
||||
let f: String = native_list_get(a, i)
|
||||
if str_eq(f, "--authority") {
|
||||
if i + 1 < n {
|
||||
authority = str_to_int(native_list_get(a, i + 1))
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
if str_eq(f, "--barge-at") {
|
||||
if i + 1 < n {
|
||||
let spec: String = native_list_get(a, i + 1)
|
||||
let c: Int = str_index_of(spec, ":")
|
||||
if c < 0 {
|
||||
barge = str_to_int(spec)
|
||||
} else {
|
||||
barge = str_to_int(str_slice(spec, 0, c))
|
||||
kind = str_slice(spec, c + 1, str_len(spec))
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
if str_eq(f, "--live-mic") {
|
||||
live = true
|
||||
}
|
||||
if str_eq(f, "--resume") {
|
||||
resume = true
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
let plan: [String] = conv_load_manifest(mf)
|
||||
if resume {
|
||||
plan = conv_load_resume()
|
||||
organ_disclose("CONVERSE: resuming — \"as I was saying...\" (" + int_to_str(plan_count(plan)) + " segments left).")
|
||||
} else {
|
||||
organ_disclose("CONVERSE: utterance = \"" + conv_utterance(mf) + "\" (" + int_to_str(plan_count(plan)) + " segments).")
|
||||
}
|
||||
let stopped: Int = conv_run(plan, authority, barge, kind, live)
|
||||
println("{\"ok\": true, \"op\": \"converse\", \"stopped_at\": " + int_to_str(stopped) + ", \"complete\": " + bool_to_str(stopped < 0) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
cli_usage()
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
// organ_converse.el — full-duplex, interruptible speech. The turn-taking organ.
|
||||
//
|
||||
// WHAT THIS IS FOR. A system that plays an utterance to completion and only
|
||||
// then listens is not conversational, it is a loudspeaker with a queue. Being
|
||||
// interruptible is not a feature bolted onto speech; it is most of what makes
|
||||
// speech social. So the utterance is not a blob of audio — it is an ordered,
|
||||
// SALIENCE-TAGGED MEANING-PLAN, and the organ speaks it while listening, decides
|
||||
// what to do when interrupted, and can pick the thread back up afterwards.
|
||||
//
|
||||
// THREE THINGS HAVE TO BE TRUE, and each one is a place naive implementations
|
||||
// go wrong:
|
||||
//
|
||||
// Barge-in is AT THE SAMPLE. When the mic hears speech, output stops on the
|
||||
// spot — not at the end of the current buffer, not at the end of the segment.
|
||||
// A listener experiences even a fifth of a second of continued talking as
|
||||
// being talked over. This is why the speaker realizer has pause/resume and
|
||||
// reports played_frames: "finish the buffer" is not barge-in.
|
||||
//
|
||||
// Yield-or-hold is a DECISION, not a rule. Stopping every time anyone makes a
|
||||
// noise is its own failure — it means Neuron can never finish a sentence that
|
||||
// matters. So the choice is grounded: how salient is what I am mid-saying,
|
||||
// how close am I to done, and how much authority does the interrupter have.
|
||||
// Holding the floor is justified when what I am saying matters AND finishing
|
||||
// is cheap AND the interrupter is not high-priority. Otherwise yield, because
|
||||
// the polite default is the right default.
|
||||
//
|
||||
// A backchannel is NOT an interruption. "mm-hm" means keep going. Treating it
|
||||
// as a barge-in makes the system stop every three seconds during ordinary
|
||||
// listening behaviour, which is worse than not listening at all. It is
|
||||
// distinguished by being brief and low-energy: sample again shortly after
|
||||
// onset, and if the speech already died away it was a backchannel.
|
||||
//
|
||||
// AND THE UTTERANCE SURVIVES. On yield, the remaining plan is persisted, so
|
||||
// Neuron can resume — "as I was saying" — instead of losing the thought. An
|
||||
// interruption should cost a turn, not the content.
|
||||
//
|
||||
// The AEC rail: the microphone runs with the OS voice-processing unit enabled
|
||||
// so it does not hear our own speaker. Without it Neuron barges in on its own
|
||||
// voice on the first syllable and the whole loop is unusable in a real room.
|
||||
//
|
||||
// Note what is NOT here: nothing about words. A segment carries a `text` field
|
||||
// purely as a label for disclosure. The organ speaks pre-rendered audio and
|
||||
// never inspects language — that is the language faculty's, and the seam holds.
|
||||
|
||||
// ── The meaning-plan ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Stored as a flat [String] with stride 3 — file, salience-per-mille, text —
|
||||
// because El has no record type and parallel lists drift out of step under
|
||||
// editing. Salience is an integer per-mille rather than a Float so the decision
|
||||
// arithmetic stays exact and reproducible; a turn-taking decision that varies
|
||||
// with floating-point rounding is not one you can debug.
|
||||
|
||||
fn plan_new() -> [String] {
|
||||
return native_list_empty()
|
||||
}
|
||||
|
||||
fn plan_add(plan: [String], file: String, salience_pm: Int, text: String) -> [String] {
|
||||
let p: [String] = plan
|
||||
p = native_list_append(p, file)
|
||||
p = native_list_append(p, int_to_str(salience_pm))
|
||||
p = native_list_append(p, text)
|
||||
return p
|
||||
}
|
||||
|
||||
fn plan_count(plan: [String]) -> Int {
|
||||
return native_list_len(plan) / 3
|
||||
}
|
||||
|
||||
fn plan_file(plan: [String], i: Int) -> String {
|
||||
return native_list_get(plan, i * 3)
|
||||
}
|
||||
|
||||
fn plan_salience(plan: [String], i: Int) -> Int {
|
||||
return str_to_int(native_list_get(plan, i * 3 + 1))
|
||||
}
|
||||
|
||||
fn plan_text(plan: [String], i: Int) -> String {
|
||||
return native_list_get(plan, i * 3 + 2)
|
||||
}
|
||||
|
||||
// ── Manifest ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// {"utterance": "...", "segments": [{"file":..., "salience":0.9, "text":"..."}]}
|
||||
// Salience arrives as a 0..1 float in the manifest and is converted once, here,
|
||||
// at the edge — the same discipline the runtime uses for wire encodings.
|
||||
|
||||
fn conv_salience_pm(raw: String) -> Int {
|
||||
// "0.85" -> 850. Parsed by hand rather than through a float so a manifest
|
||||
// typo degrades to a visible number instead of a silent 0.0.
|
||||
let dot: Int = str_index_of(raw, ".")
|
||||
if dot < 0 {
|
||||
let whole: Int = str_to_int(raw)
|
||||
return whole * 1000
|
||||
}
|
||||
let ip: Int = str_to_int(str_slice(raw, 0, dot))
|
||||
let frac: String = str_slice(raw, dot + 1, str_len(raw))
|
||||
let pm: Int = 0
|
||||
let scale: Int = 100
|
||||
let i: Int = 0
|
||||
while i < 3 {
|
||||
let d: Int = 0
|
||||
if i < str_len(frac) {
|
||||
let c: Int = str_char_code(frac, i)
|
||||
if c >= 48 {
|
||||
if c <= 57 {
|
||||
d = c - 48
|
||||
}
|
||||
}
|
||||
}
|
||||
pm = pm + d * scale
|
||||
scale = scale / 10
|
||||
i = i + 1
|
||||
}
|
||||
return ip * 1000 + pm
|
||||
}
|
||||
|
||||
fn conv_load_manifest(path: String) -> [String] {
|
||||
let plan: [String] = plan_new()
|
||||
let raw: String = fs_read(path)
|
||||
if str_eq(raw, "") {
|
||||
organ_disclose("CONVERSE: cannot read manifest " + path)
|
||||
return plan
|
||||
}
|
||||
let segs: String = json_get_raw(raw, "segments")
|
||||
let n: Int = json_array_len(segs)
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let seg: String = json_array_get(segs, i)
|
||||
let file: String = json_get_string(seg, "file")
|
||||
let text: String = json_get_string(seg, "text")
|
||||
let sal: String = json_get_raw(seg, "salience")
|
||||
let pm: Int = conv_salience_pm(sal)
|
||||
if pm <= 0 {
|
||||
pm = 500
|
||||
}
|
||||
plan = plan_add(plan, file, pm, text)
|
||||
i = i + 1
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
fn conv_utterance(path: String) -> String {
|
||||
let raw: String = fs_read(path)
|
||||
return json_get_string(raw, "utterance")
|
||||
}
|
||||
|
||||
// ── The decision ─────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Returns: 0 = backchannel, carry on seamlessly
|
||||
// 1 = hold the floor ("hang on, let me finish this thought")
|
||||
// 2 = yield (stop, let them in)
|
||||
//
|
||||
// All arguments are per-mille integers. Holding requires BOTH that the material
|
||||
// is worth finishing AND that the interrupter is not high-authority — either
|
||||
// condition alone is not enough, because "what I'm saying is important" is
|
||||
// exactly the reasoning that produces a system nobody can get a word in against.
|
||||
fn conv_decide(salience_pm: Int, progress_pm: Int, authority_pm: Int, is_backchannel: Bool) -> Int {
|
||||
if is_backchannel {
|
||||
return 0
|
||||
}
|
||||
let hold_score: Int = (salience_pm * 6 + progress_pm * 4) / 10
|
||||
if hold_score >= 600 {
|
||||
if authority_pm < 800 {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
// ── Resume ───────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The remaining plan, written where a later run can find it. This is what turns
|
||||
// an interruption into a pause rather than a loss.
|
||||
|
||||
fn conv_resume_path() -> String {
|
||||
let home: String = env("PERIPH_HOME")
|
||||
if str_eq(home, "") {
|
||||
return "peripheral/.resume.json"
|
||||
}
|
||||
return home + "/.resume.json"
|
||||
}
|
||||
|
||||
// Minimal JSON string escaping. Written here rather than reached for from the
|
||||
// runtime because the organ needs exactly two escapes and no dependency: a
|
||||
// segment label containing a quote or a backslash must not be able to produce a
|
||||
// resume file that fails to parse and silently loses the thread.
|
||||
fn conv_escape(s: String) -> String {
|
||||
let n: Int = str_len(s)
|
||||
let out: String = ""
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let c: Int = str_char_code(s, i)
|
||||
if c == 34 {
|
||||
out = out + "\\\""
|
||||
} else {
|
||||
if c == 92 {
|
||||
out = out + "\\\\"
|
||||
} else {
|
||||
if c >= 32 {
|
||||
out = out + str_slice(s, i, i + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
fn conv_persist_resume(plan: [String], start_at: Int, reason: String) -> Bool {
|
||||
let n: Int = plan_count(plan)
|
||||
let body: String = "{\"resume_from\": " + int_to_str(start_at) + ", \"reason\": \"" + reason + "\", \"segments\": ["
|
||||
let i: Int = start_at
|
||||
let first: Bool = true
|
||||
while i < n {
|
||||
if first == false {
|
||||
body = body + ", "
|
||||
}
|
||||
body = body + "{\"file\": \"" + plan_file(plan, i) + "\", \"salience\": " + int_to_str(plan_salience(plan, i)) + ", \"text\": \"" + conv_escape(plan_text(plan, i)) + "\"}"
|
||||
first = false
|
||||
i = i + 1
|
||||
}
|
||||
body = body + "]}\n"
|
||||
let ok: Bool = fs_write(conv_resume_path(), body)
|
||||
organ_disclose("CONVERSE: meaning-plan persisted (" + int_to_str(n - start_at) + " segments remain) — Neuron can resume the thread.")
|
||||
return ok
|
||||
}
|
||||
|
||||
fn conv_clear_resume() -> Bool {
|
||||
return fs_write(conv_resume_path(), "")
|
||||
}
|
||||
|
||||
// Read a persisted plan back. Salience is already per-mille here (we wrote it),
|
||||
// so it is NOT re-scaled — the manifest and the resume file are different
|
||||
// formats on purpose, and conflating them silently divides every salience by a
|
||||
// thousand.
|
||||
fn conv_load_resume() -> [String] {
|
||||
let plan: [String] = plan_new()
|
||||
let raw: String = fs_read(conv_resume_path())
|
||||
if str_eq(raw, "") {
|
||||
return plan
|
||||
}
|
||||
let segs: String = json_get_raw(raw, "segments")
|
||||
let n: Int = json_array_len(segs)
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let seg: String = json_array_get(segs, i)
|
||||
plan = plan_add(plan, json_get_string(seg, "file"), json_get_int(seg, "salience"), json_get_string(seg, "text"))
|
||||
i = i + 1
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
// ── The loop ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// live_mic : open the microphone with AEC and let real speech drive barge-in.
|
||||
// barge_ms : if >= 0, inject a barge event at that offset into the utterance
|
||||
// instead. Deterministic, so the decision paths can be exercised
|
||||
// without a room and a person — the same reason periph.swift has it.
|
||||
// kind : "backchannel" or "bargein", for the injected case.
|
||||
// authority : interrupter authority, per-mille.
|
||||
//
|
||||
// Returns the index the utterance stopped at, or -1 if it completed.
|
||||
|
||||
fn conv_run(plan: [String], authority_pm: Int, barge_ms: Int, kind: String, live_mic: Bool) -> Int {
|
||||
let n: Int = plan_count(plan)
|
||||
if n <= 0 {
|
||||
organ_disclose("CONVERSE: nothing to say.")
|
||||
return 0 - 1
|
||||
}
|
||||
if speaker_available() == 0 {
|
||||
organ_disclose("CONVERSE: no speaker on this build — cannot hold a conversation.")
|
||||
return 0 - 1
|
||||
}
|
||||
|
||||
let mic_live: Bool = false
|
||||
if live_mic {
|
||||
if organ_may_listen() {
|
||||
let m: Int = mic_monitor_start()
|
||||
if m == 1 {
|
||||
organ_disclose("CONVERSE: full-duplex — mic listening WHILE speaking, AEC on (won't self-interrupt).")
|
||||
mic_live = true
|
||||
}
|
||||
if m == 2 {
|
||||
organ_disclose("CONVERSE: full-duplex — mic listening, but AEC UNAVAILABLE; raising the VAD floor so we do not barge in on ourselves.")
|
||||
mic_live = true
|
||||
}
|
||||
if m == 0 {
|
||||
organ_disclose("CONVERSE: could not open the mic monitor — falling back to injected events.")
|
||||
}
|
||||
}
|
||||
}
|
||||
if mic_live == false {
|
||||
organ_disclose("CONVERSE: deterministic mode (live mic off).")
|
||||
}
|
||||
|
||||
// Without AEC the mic hears the speaker, so the threshold has to sit above
|
||||
// our own output. This is a mitigation and not a fix: the honest note is
|
||||
// that barge-in is markedly less sensitive in this mode.
|
||||
let vad_pm: Int = 20
|
||||
if mic_live {
|
||||
if mic_monitor_start() == 2 {
|
||||
vad_pm = 60
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed_ms: Int = 0
|
||||
let prior_ms: Int = 0
|
||||
let handled: Bool = false
|
||||
// An injected barge is ONE event, not a condition that stays true. Without
|
||||
// this the deadline re-fires on every poll after a backchannel resume, and
|
||||
// the utterance live-locks: paused, resumed, paused again, forever.
|
||||
let injected_fired: Bool = false
|
||||
let i: Int = 0
|
||||
|
||||
while i < n {
|
||||
let file: String = plan_file(plan, i)
|
||||
let sal: Int = plan_salience(plan, i)
|
||||
let frames: Int = wav_frames(file)
|
||||
let rate: Int = wav_rate(file)
|
||||
if frames <= 0 {
|
||||
organ_disclose("CONVERSE: missing or unreadable segment '" + file + "', skipping.")
|
||||
i = i + 1
|
||||
} else {
|
||||
let dur_ms: Int = frames * 1000 / rate
|
||||
organ_disclose("CONVERSE: speaking segment " + int_to_str(i + 1) + "/" + int_to_str(n) + " (salience " + int_to_str(sal) + "/1000) — \"" + plan_text(plan, i) + "\"")
|
||||
let started: Int = speaker_play_wav_async(file)
|
||||
if started == 0 {
|
||||
organ_disclose("CONVERSE: could not start playback for '" + file + "'.")
|
||||
i = i + 1
|
||||
} else {
|
||||
let seg_ms: Int = 0
|
||||
let done: Bool = false
|
||||
let interrupted: Bool = false
|
||||
let speech_ticks: Int = 0
|
||||
|
||||
while done == false {
|
||||
sleep_ms(10)
|
||||
seg_ms = seg_ms + 10
|
||||
|
||||
if speaker_playing() == 0 {
|
||||
done = true
|
||||
} else {
|
||||
// The tick counter is an approximation — each pass costs
|
||||
// more than the sleep it asked for. The DAC position is
|
||||
// the truth, so drive the injected deadline off THAT and
|
||||
// an injected barge lands where it was asked to land.
|
||||
let pos_ms: Int = speaker_played_frames() * 1000 / rate
|
||||
elapsed_ms = prior_ms + pos_ms
|
||||
// --- onset detection: real speech, or an injected event ---
|
||||
let onset: Bool = false
|
||||
if mic_live {
|
||||
let rms: Float = mic_monitor_rms()
|
||||
let rms_pm: Int = float_to_int(rms * 1000.0)
|
||||
if rms_pm > vad_pm {
|
||||
speech_ticks = speech_ticks + 1
|
||||
} else {
|
||||
speech_ticks = 0
|
||||
}
|
||||
// ~60ms of continuous voice: short enough to feel
|
||||
// instant, long enough that a door closing is not a turn.
|
||||
if speech_ticks >= 3 {
|
||||
if handled == false {
|
||||
onset = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if barge_ms >= 0 {
|
||||
if injected_fired == false {
|
||||
if elapsed_ms >= barge_ms {
|
||||
onset = true
|
||||
injected_fired = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if onset {
|
||||
handled = true
|
||||
// (1) BARGE-IN — pause on the spot.
|
||||
speaker_pause()
|
||||
let played: Int = speaker_played_frames()
|
||||
let at_ms: Int = played * 1000 / rate
|
||||
let progress_pm: Int = at_ms * 1000 / dur_ms
|
||||
if progress_pm > 1000 {
|
||||
progress_pm = 1000
|
||||
}
|
||||
organ_disclose("CONVERSE: << user speech at " + int_to_str(at_ms) + "ms into segment " + int_to_str(i + 1) + " — PAUSED instantly >>")
|
||||
|
||||
// (2) backchannel or real barge-in?
|
||||
let is_bc: Bool = false
|
||||
if barge_ms >= 0 {
|
||||
if str_eq(kind, "backchannel") {
|
||||
is_bc = true
|
||||
}
|
||||
} else {
|
||||
// Live: look again ~250ms after onset. If the
|
||||
// energy has already collapsed it was "mm-hm".
|
||||
sleep_ms(250)
|
||||
let r2: Float = mic_monitor_rms()
|
||||
if float_to_int(r2 * 1000.0) < 15 {
|
||||
is_bc = true
|
||||
}
|
||||
}
|
||||
|
||||
// (3) yield, hold, or carry on
|
||||
let d: Int = conv_decide(sal, progress_pm, authority_pm, is_bc)
|
||||
if d == 0 {
|
||||
organ_disclose("CONVERSE: read as BACKCHANNEL (\"mm-hm\") — keep going, resume seamlessly.")
|
||||
handled = false
|
||||
speech_ticks = 0
|
||||
speaker_resume()
|
||||
}
|
||||
if d == 1 {
|
||||
organ_disclose("CONVERSE: HOLD the floor — \"hang on, let me finish this thought.\" (salience " + int_to_str(sal) + ", progress " + int_to_str(progress_pm) + ")")
|
||||
speaker_resume()
|
||||
// Finish THIS segment, then yield the remainder:
|
||||
// holding is a request for a moment, not a claim
|
||||
// on the rest of the conversation.
|
||||
while speaker_playing() == 1 {
|
||||
sleep_ms(20)
|
||||
}
|
||||
speaker_stop()
|
||||
conv_persist_resume(plan, i + 1, "held-then-yield")
|
||||
if mic_live {
|
||||
mic_monitor_stop()
|
||||
}
|
||||
return i + 1
|
||||
}
|
||||
if d == 2 {
|
||||
organ_disclose("CONVERSE: YIELD — stop, let them in. Remembering where I was (resumable).")
|
||||
speaker_stop()
|
||||
conv_persist_resume(plan, i, "yield")
|
||||
if mic_live {
|
||||
mic_monitor_stop()
|
||||
}
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if interrupted == false {
|
||||
prior_ms = prior_ms + dur_ms
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
conv_clear_resume()
|
||||
organ_disclose("CONVERSE: utterance complete (uninterrupted).")
|
||||
if mic_live {
|
||||
mic_monitor_stop()
|
||||
}
|
||||
return 0 - 1
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user