diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2b63ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ + +# organ: local device state and its own engram store — never production's +peripheral/.consent.json +peripheral/.resume.json +peripheral/.engram/ +peripheral/organ diff --git a/lang/runtime/el_audio_darwin.m b/lang/runtime/el_audio_darwin.m new file mode 100644 index 0000000..da43a64 --- /dev/null +++ b/lang/runtime/el_audio_darwin.m @@ -0,0 +1,525 @@ +/* el_audio_darwin.m — the SPEAKER realizer. El's native audio output on Darwin. + * + * WHY THIS FILE EXISTS. + * + * Neuron could already turn meaning into samples — the render path in + * elp/src/speech.el superposes formant resonances over a glottal source and + * produces PCM. What it could not do was make a sound. Every path from those + * samples to the air ran outside the language: a 939-line Swift program + * (peripheral/src/periph.swift) that shelled out to /usr/bin/afplay. So the + * voice was not a capability of El or of Neuron. It was a separate binary + * standing next to them, and "speak" meant "ask that binary to speak." + * + * A speaker is not a language feature the way a string is, but it is exactly + * the kind of thing a runtime owns: a device. El already owns the filesystem, + * the network, the clock, and a graph. It should own the one output device that + * makes it audible. After this file, `speak` is an El operation. + * + * WHY IT IS A REALIZER AND NOT PURE EL. + * + * This is the boundary the whole design turns on. Everything ABOVE the sample + * buffer is arithmetic and belongs in El: formant geometry, superposition, + * envelopes, WAV framing, the voice signature. Everything in this file is the + * part that cannot be arithmetic — handing a buffer to CoreAudio and waiting + * for the hardware to drain it. There is no way to express "the DAC has now + * played these samples" in El, and there should not be. So the split is: El + * computes the sound, the realizer emits it, and the realizer is as thin as it + * can possibly be — it makes no decisions about content, it has no opinion + * about audio, and it cannot synthesize anything. + * + * The precedent is eg_cosine_batch_strategy_metal_hand.m: a platform-bound + * capability compiled as its OWN translation unit, declared in el_runtime.h, + * and linked in where the platform supports it. Deliberately NOT a patch to + * el_runtime.c — adding a device to El must not mean editing the core runtime, + * for the same reason adding a modality must not (see el_runtime.c's realizer + * registry: a realizer is resolved by name, so new organs never touch the + * middle of the language). el_audio_null.c is the same two entry points for + * every platform that is not Darwin, so El code that speaks still links + * everywhere and simply reports that it has no speaker. + * + * WHY AudioQueue AND NOT afplay. + * + * afplay is a process. Using it means the sound Neuron makes is a file it wrote + * and asked something else to open — which forces every utterance through the + * disk, cannot start until the whole utterance exists, and puts a fork/exec + * between the intent to speak and the sound. AudioQueue takes the samples + * directly out of memory. Nothing is written, nothing is spawned, and a caller + * that wants to stream can push buffers as it renders them. + * + * AudioToolbox ships with macOS, so this stays own-core: no cloud, no library + * to install, no model. The output is the local speaker and nothing leaves the + * machine — there is no network path in this file at all, by construction. + */ + +#import +#import +#include +#include +#include +#include +#include "el_runtime.h" + +/* Three buffers is the standard AudioQueue depth: one being played by the + * hardware, one queued behind it, one being refilled. Fewer risks a gap on a + * busy machine; more only adds latency before the first sound. */ +#define EL_AQ_NBUF 3 +#define EL_AQ_FRAMES 8192 + +typedef struct { + const int16_t* pcm; + int64_t frames; + int64_t pos; + volatile int inflight; /* buffers CoreAudio still owns */ + volatile int drained; /* set once the last buffer has been played */ +} ElAqState; + +/* Called on an AudioQueue-internal thread each time a buffer finishes playing. + * Refills and re-enqueues while samples remain; when the source is exhausted it + * lets the buffer die and counts it out. `drained` flips only when the queue is + * holding nothing, which is what makes the play call synchronous without + * clipping the tail — the same reason periph.swift used .dataPlayedBack rather + * than treating "consumed" as "heard". */ +static void el_aq_callback(void* userData, AudioQueueRef q, AudioQueueBufferRef buf) { + ElAqState* st = (ElAqState*)userData; + int64_t remain = st->frames - st->pos; + if (remain <= 0) { + if (--st->inflight <= 0) st->drained = 1; + return; + } + int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(buf->mAudioData, st->pcm + st->pos, (size_t)n * sizeof(int16_t)); + buf->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); + st->pos += n; + if (AudioQueueEnqueueBuffer(q, buf, 0, NULL) != noErr) { + if (--st->inflight <= 0) st->drained = 1; + } +} + +/* Play a 16-bit mono PCM buffer out the default output device, blocking until + * the hardware has actually finished. Returns 1 on success, 0 on any failure — + * never throws, never hangs indefinitely. */ +static int el_audio_play_raw(const int16_t* pcm, int64_t frames, int32_t sample_rate) { + if (!pcm || frames <= 0 || sample_rate <= 0) return 0; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sample_rate; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + ElAqState st; + memset(&st, 0, sizeof(st)); + st.pcm = pcm; + st.frames = frames; + + AudioQueueRef q = NULL; + /* NULL run loop => callbacks arrive on an AudioQueue-internal thread, so + * this function can simply wait rather than having to pump a run loop it + * does not own. El programs are not required to have one. */ + if (AudioQueueNewOutput(&fmt, el_aq_callback, &st, NULL, NULL, 0, &q) != noErr || !q) { + return 0; + } + + AudioQueueBufferRef bufs[EL_AQ_NBUF]; + int prepared = 0; + for (int i = 0; i < EL_AQ_NBUF; i++) { + if (AudioQueueAllocateBuffer(q, EL_AQ_FRAMES * sizeof(int16_t), &bufs[i]) != noErr) break; + prepared++; + } + if (prepared == 0) { AudioQueueDispose(q, true); return 0; } + + /* Prime: fill what we can before starting, so playback begins immediately + * rather than after the first underrun. */ + for (int i = 0; i < prepared; i++) { + int64_t remain = st.frames - st.pos; + if (remain <= 0) break; + int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(bufs[i]->mAudioData, st.pcm + st.pos, (size_t)n * sizeof(int16_t)); + bufs[i]->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); + st.pos += n; + if (AudioQueueEnqueueBuffer(q, bufs[i], 0, NULL) != noErr) break; + st.inflight++; + } + if (st.inflight == 0) { AudioQueueDispose(q, true); return 0; } + + if (AudioQueueStart(q, NULL) != noErr) { AudioQueueDispose(q, true); return 0; } + + /* Bound the wait by the material's own duration plus a margin. A speaker + * that wedges a program is worse than a speaker that gives up. */ + double seconds = (double)frames / (double)sample_rate; + int64_t max_us = (int64_t)((seconds + 5.0) * 1000000.0); + int64_t waited = 0; + const int64_t tick = 5000; /* 5 ms */ + while (!st.drained && waited < max_us) { + usleep((useconds_t)tick); + waited += tick; + } + + AudioQueueStop(q, true); + AudioQueueDispose(q, true); + return st.drained ? 1 : 0; +} + +/* ── El entry points ──────────────────────────────────────────────────────── + * Declared in el_runtime.h; see there for the El-facing contract. */ + +/* 1 when this build has a real speaker behind it. El code should ask before + * speaking so the no-speaker case is a reported condition, not a silence that + * looks like success. */ +el_val_t speaker_available(void) { + return (el_val_t)1; +} + +el_val_t speaker_name(void) { + return EL_STR("coreaudio-audioqueue"); +} + +/* Play an El [Int] of 16-bit samples. Values are clamped, not wrapped: a + * render that overshoots should distort at the rails the way real clipping + * does, rather than invert phase and produce a sound nothing in the signal + * chain intended. */ +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { + int64_t n = (int64_t)el_list_len(samples); + int32_t sr = (int32_t)sample_rate; + if (n <= 0 || sr <= 0) return (el_val_t)0; + + int16_t* pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); + if (!pcm) return (el_val_t)0; + + for (int64_t i = 0; i < n; i++) { + int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); + if (v > 32767) v = 32767; + if (v < -32768) v = -32768; + pcm[i] = (int16_t)v; + } + + int ok = el_audio_play_raw(pcm, n, sr); + free(pcm); + return (el_val_t)(ok ? 1 : 0); +} + +/* ── Asynchronous playback ─────────────────────────────────────────────────── + * + * converse needs this and a blocking play cannot give it. Barge-in means + * stopping ON THE SPOT when the user starts talking — not at the end of the + * current buffer, and certainly not at the end of the utterance. So the async + * path keeps one queue alive, reports how far the hardware actually got, and + * can be halted mid-buffer. + * + * `played_frames` is what makes an interrupted utterance resumable at the + * sample rather than at the segment: it is the position the DAC reached, not + * the position we enqueued to, and those differ by up to the full queue depth. + * + * One utterance at a time. A second async play stops the first — a mouth that + * can say two things at once is not a feature. */ + +static AudioQueueRef g_aq = NULL; +static ElAqState* g_aq_state = NULL; +static int16_t* g_aq_pcm = NULL; +static int32_t g_aq_sr = 0; + +static void el_audio_teardown(void) { + if (g_aq) { + AudioQueueStop(g_aq, true); + AudioQueueDispose(g_aq, true); + g_aq = NULL; + } + free(g_aq_pcm); g_aq_pcm = NULL; + free(g_aq_state); g_aq_state = NULL; + g_aq_sr = 0; +} + +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { + el_audio_teardown(); + + int64_t n = (int64_t)el_list_len(samples); + int32_t sr = (int32_t)sample_rate; + if (n <= 0 || sr <= 0) return (el_val_t)0; + + g_aq_pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); + if (!g_aq_pcm) return (el_val_t)0; + for (int64_t i = 0; i < n; i++) { + int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); + if (v > 32767) v = 32767; + if (v < -32768) v = -32768; + g_aq_pcm[i] = (int16_t)v; + } + + g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); + if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } + g_aq_state->pcm = g_aq_pcm; + g_aq_state->frames = n; + g_aq_sr = sr; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sr; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { + el_audio_teardown(); + return (el_val_t)0; + } + + for (int i = 0; i < EL_AQ_NBUF; i++) { + int64_t remain = g_aq_state->frames - g_aq_state->pos; + if (remain <= 0) break; + AudioQueueBufferRef b = NULL; + if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; + int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); + b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); + g_aq_state->pos += k; + if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; + g_aq_state->inflight++; + } + if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } + + if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } + return (el_val_t)1; +} + +el_val_t speaker_playing(void) { + if (!g_aq || !g_aq_state) return (el_val_t)0; + return (el_val_t)(g_aq_state->drained ? 0 : 1); +} + +/* Frames the DAC has actually rendered. AudioQueueGetCurrentTime's mSampleTime + * is relative to queue start, which is exactly the "where was I really" figure + * a resumable utterance needs. Falls back to the enqueued position if the + * timeline is unavailable (it is, briefly, right after start). */ +el_val_t speaker_played_frames(void) { + if (!g_aq || !g_aq_state) return (el_val_t)0; + AudioTimeStamp ts; + memset(&ts, 0, sizeof(ts)); + Boolean discontinuity = false; + if (AudioQueueGetCurrentTime(g_aq, NULL, &ts, &discontinuity) == noErr && + (ts.mFlags & kAudioTimeStampSampleTimeValid)) { + int64_t played = (int64_t)ts.mSampleTime; + if (played < 0) played = 0; + if (played > g_aq_state->frames) played = g_aq_state->frames; + return (el_val_t)played; + } + return (el_val_t)g_aq_state->pos; +} + +/* Pause where we are, keeping the queue and its position intact. + * + * This is the difference between barge-in and "finish the buffer". The moment + * the microphone hears speech, output must stop AT THAT SAMPLE — a listener + * experiences even 200ms of continued talking as being talked over. Pause + * rather than stop because the interruption might turn out to be a backchannel + * ("mm-hm"), and the right response to a backchannel is to carry on as though + * nothing happened, which requires the queue to still be exactly where it was. + * A stop-and-restart would re-attack the buffer and be audible as a stutter. */ +el_val_t speaker_pause(void) { + if (!g_aq) return (el_val_t)0; + return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0); +} + +el_val_t speaker_resume(void) { + if (!g_aq) return (el_val_t)0; + return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0); +} + +el_val_t speaker_stop(void) { + if (!g_aq) return (el_val_t)0; + /* immediate: do NOT let the queue finish what it is holding */ + AudioQueueStop(g_aq, true); + el_audio_teardown(); + return (el_val_t)1; +} + +/* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer. + * Returns frames, or 0 on any failure; *out is set only on success. Shared by + * the blocking and async WAV paths. */ +static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) { + if (!path || !out) return 0; + FILE* f = fopen(path, "rb"); + if (!f) return 0; + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; } + long size = ftell(f); + if (size <= 44) { fclose(f); return 0; } + rewind(f); + unsigned char* d = (unsigned char*)malloc((size_t)size); + if (!d) { fclose(f); return 0; } + size_t got = fread(d, 1, (size_t)size, f); + fclose(f); + if (got != (size_t)size) { free(d); return 0; } + if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; } + + int32_t sr = 0, channels = 0, bits = 0; + long dataOff = -1, dataLen = 0, o = 12; + /* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders + * routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads + * padding as audio. */ + while (o + 8 <= size) { + long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); + if (sz < 0) break; + if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { + channels = (int32_t)(d[o+10] | (d[o+11] << 8)); + sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); + bits = (int32_t)(d[o+22] | (d[o+23] << 8)); + } else if (memcmp(d + o, "data", 4) == 0) { + dataOff = o + 8; + dataLen = sz; + if (dataOff + dataLen > size) dataLen = size - dataOff; + } + o += 8 + sz + (sz & 1); + } + if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; } + + long frames = dataLen / (2 * channels); + int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); + if (!pcm) { free(d); return 0; } + for (long i = 0; i < frames; i++) { + long b = dataOff + i * 2 * channels; + pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); + } + free(d); + *out = pcm; + if (out_sr) *out_sr = sr; + return (int64_t)frames; +} + +/* Async WAV playback. converse speaks PRE-RENDERED segments and must keep + * listening while it does, so it needs the file on the queue without blocking + * and needs to be able to stop it mid-buffer. Going through the file rather + * than an El [Int] also avoids marshalling a million-element list per segment + * for audio the caller never intends to look at. */ +el_val_t speaker_play_wav_async(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p) return (el_val_t)0; + + el_audio_teardown(); + + int32_t sr = 0; + int16_t* pcm = NULL; + int64_t frames = el_wav_load(p, &pcm, &sr); + if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; } + + g_aq_pcm = pcm; + g_aq_sr = sr; + g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); + if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } + g_aq_state->pcm = g_aq_pcm; + g_aq_state->frames = frames; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sr; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { + el_audio_teardown(); + return (el_val_t)0; + } + for (int i = 0; i < EL_AQ_NBUF; i++) { + int64_t remain = g_aq_state->frames - g_aq_state->pos; + if (remain <= 0) break; + AudioQueueBufferRef b = NULL; + if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; + int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); + b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); + g_aq_state->pos += k; + if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; + g_aq_state->inflight++; + } + if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } + if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } + return (el_val_t)1; +} + +/* Total frames and sample rate of a WAV, without playing it — wav-info, and the + * duration converse needs to compute progress through a segment. */ +el_val_t wav_frames(el_val_t path) { + const char* p = EL_CSTR(path); + int16_t* pcm = NULL; int32_t sr = 0; + int64_t n = el_wav_load(p, &pcm, &sr); + free(pcm); + return (el_val_t)n; +} + +el_val_t wav_rate(el_val_t path) { + const char* p = EL_CSTR(path); + int16_t* pcm = NULL; int32_t sr = 0; + int64_t n = el_wav_load(p, &pcm, &sr); + free(pcm); + return (el_val_t)(n > 0 ? sr : 0); +} + +/* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows + * how to write a WAV and a caller may reasonably want to hear one back without + * re-rendering it; the parse is deliberately minimal and chunk-walking, so the + * JUNK/FLLR padding that recorders emit does not defeat it. */ +el_val_t speaker_play_wav(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p) return (el_val_t)0; + FILE* f = fopen(p, "rb"); + if (!f) return (el_val_t)0; + + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return (el_val_t)0; } + long size = ftell(f); + if (size <= 44) { fclose(f); return (el_val_t)0; } + rewind(f); + + unsigned char* d = (unsigned char*)malloc((size_t)size); + if (!d) { fclose(f); return (el_val_t)0; } + size_t got = fread(d, 1, (size_t)size, f); + fclose(f); + if (got != (size_t)size) { free(d); return (el_val_t)0; } + + if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return (el_val_t)0; } + + int32_t sr = 0, channels = 0, bits = 0; + long dataOff = -1, dataLen = 0; + long o = 12; + while (o + 8 <= size) { + long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); + if (sz < 0) break; + if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { + channels = (int32_t)(d[o+10] | (d[o+11] << 8)); + sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); + bits = (int32_t)(d[o+22] | (d[o+23] << 8)); + } else if (memcmp(d + o, "data", 4) == 0) { + dataOff = o + 8; + dataLen = sz; + if (dataOff + dataLen > size) dataLen = size - dataOff; + } + o += 8 + sz + (sz & 1); + } + if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return (el_val_t)0; } + + long frames = dataLen / (2 * channels); + int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); + if (!pcm) { free(d); return (el_val_t)0; } + /* Take channel 0; the organ is mono by design and downmixing would be an + * opinion about content this layer is not entitled to have. */ + for (long i = 0; i < frames; i++) { + long b = dataOff + i * 2 * channels; + pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); + } + free(d); + + int ok = el_audio_play_raw(pcm, frames, sr); + free(pcm); + return (el_val_t)(ok ? 1 : 0); +} diff --git a/lang/runtime/el_capture_darwin.m b/lang/runtime/el_capture_darwin.m new file mode 100644 index 0000000..3651408 --- /dev/null +++ b/lang/runtime/el_capture_darwin.m @@ -0,0 +1,841 @@ +/* el_capture_darwin.m — the MICROPHONE and CAMERA realizers. El's afferent + * organ on Darwin: the two entry points through which the world gets in. + * + * WHY THIS FILE EXISTS, AND WHY IT IS A REALIZER RATHER THAN PURE EL. + * + * el_audio_darwin.m argued the efferent half of this: El can compute a sound + * but it cannot make one, because "the DAC has now played these samples" is not + * a fact any amount of arithmetic can produce. This file is the same argument + * run backwards. El can compute *about* a sound — it can window it, take its + * autocorrelation, run Levinson-Durbin over that, find the formant peaks in the + * resulting all-pole envelope, and hand back a voiceprint — but it cannot ASK. + * There is no expression in El, and there must not be, whose value is "the next + * 1024 frames the microphone hears" or "what the camera is pointed at right + * now." Those are not computed; they are *requested*, from an operating system + * that owns the device, mediates consent for it, and delivers the answer on a + * thread of its choosing whenever it feels like it. Asking is the one primitive + * operation here. Everything else in this file is bookkeeping around the ask. + * + * So the line is drawn exactly where el_audio_darwin.m drew it, at the sample + * buffer, and it is drawn on purpose: + * + * BELOW the line (here): open the device, honour the OS permission gate, + * install a tap or a frame delegate, convert whatever the hardware happens to + * emit into the one shape El asked for, and hand it up. No opinions about + * content. No analysis. No decisions. + * + * ABOVE the line (El): energy, zero-crossing rate, spectral centroid, F0 by + * autocorrelation, LPC, formants F1-F5, the compact descriptors, the + * scene-geometry grid, the yield-or-hold turn-taking decision. All of it is + * arithmetic over a buffer, all of it belongs in El, and none of it appears + * below. The reference this file ports — peripheral/src/periph.swift — held + * both halves, and that was the problem worth fixing: the descriptors were + * trapped in a 939-line binary standing next to the language instead of being + * written in it. Porting the *whole* of periph.swift down here would have + * reproduced that mistake in C. Only the ask came down. + * + * The precedent for the file's SHAPE is eg_cosine_batch_strategy_metal_hand.m: + * a platform-bound capability compiled as its own translation unit, declared in + * el_runtime.h, linked in where the platform supports it, and deliberately NOT + * a patch to the middle of el_runtime.c. Acquiring a device must not mean + * editing the language, for the same reason acquiring a modality must not (see + * the realizer registry: organs are resolved by name). el_peripheral_null.c is + * the same entry points everywhere else, so El code that listens still links on + * every platform and merely reports having no ear. + * + * FAIL CLOSED, ALWAYS. + * + * A capture path that returns plausible-looking zeros when it was denied is + * worse than one that returns nothing, because the caller cannot tell the + * difference between a silent room and a refused microphone. Every entry point + * here checks AVCaptureDevice's authorization status BEFORE touching hardware + * and returns the empty value — an empty list, a 0 map — on anything short of + * .authorized. mic_available() and camera_available() report that state WITHOUT + * prompting, so El can ask "may I?" without the act of asking being a prompt. + * + * NEVER HANG. + * + * Every wait in this file is bounded: 30s on a permission prompt (the user has + * to walk to the dialog), seconds+5 on a capture of `seconds`, 10s on a camera + * frame. An organ that wedges the program holding it is not an organ, it is a + * fault. Every path also tears the device down on the way out, including the + * failure paths, so a timed-out capture does not leave the mic light on. + * + * OWN-CORE AND LOCAL BY CONSTRUCTION. + * + * AVFoundation, CoreVideo, CoreGraphics and ImageIO ship with macOS. There is + * no third-party library here, no model, and — the part that matters — no + * network path of any kind. Samples and pixels move from local hardware into an + * El value and stop. periph.swift had a URLSession in it; this file has no + * socket, no URL, and nothing that could grow one without being obvious in + * review. Consent is enforced above this layer in El and below it by the OS; + * this layer's whole contribution to that is refusing to proceed. + * + * DISCLOSURE. + * + * Every actual device touch writes one line to stderr and flushes it, before + * the device opens. stderr and not stdout: a program that announces "I am about + * to open the microphone" on stdout has corrupted its own output, and the + * caller must be able to separate the answer from how it was obtained. One line + * per touch, no more — a disclosure rail that spams is a rail people learn to + * ignore. + */ + +#if defined(__APPLE__) + +#import +#import +#import +#import +#import +#import + +#include +#include +#include +#include +#include +#include + +#include "el_runtime.h" + +/* ── Disclosure ────────────────────────────────────────────────────────────── + * One flushed line per real device touch, on stderr. Flushed rather than + * buffered so the line reaches the terminal BEFORE the mic light comes on + * rather than whenever the buffer happens to drain. */ +static void el_cap_disclose(const char* what) { + fprintf(stderr, " [peripheral] %s\n", what); + fflush(stderr); +} + +/* ── Permission ────────────────────────────────────────────────────────────── + * authorizationStatus is a pure read of the TCC database: it never prompts and + * never blocks, which is what lets mic_available()/camera_available() answer + * honestly without the question itself becoming an event. requestAccess DOES + * prompt, so it lives behind its own entry point and nothing calls it + * implicitly. */ + +static int el_cap_authorized(AVMediaType media) { + @try { + return [AVCaptureDevice authorizationStatusForMediaType:media] + == AVAuthorizationStatusAuthorized ? 1 : 0; + } @catch (NSException* e) { + (void)e; + return 0; + } +} + +static int el_cap_device_present(AVMediaType media) { + @try { + return [AVCaptureDevice defaultDeviceWithMediaType:media] != nil ? 1 : 0; + } @catch (NSException* e) { + (void)e; + return 0; + } +} + +/* Prompt once and wait, bounded. 30 seconds is the same budget periph.swift + * used: long enough for a human to notice a dialog and decide, short enough + * that an unattended run fails rather than parks forever. A timeout is reported + * as "not granted", which is the safe reading — we genuinely do not know that + * it was. */ +static int el_cap_request(AVMediaType media) { + __block int granted = 0; + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + @try { + [AVCaptureDevice requestAccessForMediaType:media + completionHandler:^(BOOL ok) { + granted = ok ? 1 : 0; + dispatch_semaphore_signal(sem); + }]; + } @catch (NSException* e) { + (void)e; + return 0; + } + if (dispatch_semaphore_wait(sem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(30 * NSEC_PER_SEC))) != 0) { + return 0; /* timed out — treat as refused */ + } + return granted; +} + +/* ════════════════════════════════════════════════════════════════════════════ + * MICROPHONE — one-shot capture + * ══════════════════════════════════════════════════════════════════════════ */ + +/* The sink the tap block writes into. An object rather than a static so two + * captures can never share state, and so ARC keeps it alive for exactly as long + * as the block that captured it. The lock is real, not decorative: the tap runs + * on an AVAudioEngine-internal thread and the waiter runs on the caller's. */ +@interface ElCapMicSink : NSObject +@property (nonatomic, strong) NSMutableData* pcm; +@property (nonatomic, strong) NSLock* lock; +@end + +@implementation ElCapMicSink +- (instancetype)init { + self = [super init]; + if (self) { + _pcm = [NSMutableData data]; + _lock = [[NSLock alloc] init]; + } + return self; +} +@end + +/* Capture `seconds` of mono 16-bit PCM at `sample_rate`. + * + * The hardware format is NOT assumed. A built-in mic will typically hand back + * float32 at 44.1 or 48 kHz, an aggregate device may be 8 channels at 96 kHz, + * and a caller asking for 16 kHz mono (which is what the formant path wants) + * gets 16 kHz mono either way. AVAudioConverter does the rate conversion and + * the downmix; doing it by hand would mean writing a resampler in the one file + * that is supposed to contain no arithmetic. + * + * The converter is built ONCE, outside the tap, because a sample-rate converter + * carries filter state across buffers — rebuilding it per callback would put a + * discontinuity at every buffer boundary, which is audible and which would then + * show up in El's spectral descriptors as energy that was never in the room. */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { + el_val_t empty = el_list_empty(); + + if (!el_cap_authorized(AVMediaTypeAudio)) return empty; + + int64_t secs = (int64_t)seconds; + int64_t sr = (int64_t)sample_rate; + if (secs <= 0 || sr <= 0) return empty; + /* Bound the ask. A caller that asks for a year of audio has made a mistake, + * and honouring it would mean an unkillable capture and an OOM. */ + if (secs > 300) secs = 300; + if (sr > 384000) sr = 384000; + + __block AVAudioEngine* engine = nil; + AVAudioInputNode* input = nil; + AVAudioFormat* hwFmt = nil; + int tapped = 0; + + @try { + engine = [[AVAudioEngine alloc] init]; + input = [engine inputNode]; + hwFmt = [input inputFormatForBus:0]; + } @catch (NSException* e) { + (void)e; + return empty; + } + if (!input || !hwFmt || hwFmt.sampleRate <= 0 || hwFmt.channelCount == 0) { + return empty; + } + + /* Preferred target: mono int16 at the requested rate. If the converter + * refuses that pairing (some exotic input layouts will not downmix), fall + * back to keeping the hardware's channel count and taking channel 0 on the + * way out — the organ is mono by design and inventing a downmix here would + * be an opinion about content this layer is not entitled to have. */ + AVAudioFormat* outFmt = + [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:(double)sr + channels:1 + interleaved:YES]; + AVAudioConverter* conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt + toFormat:outFmt] : nil; + AVAudioChannelCount outCh = 1; + if (!conv) { + outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:(double)sr + channels:hwFmt.channelCount + interleaved:YES]; + conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil; + outCh = hwFmt.channelCount; + } + if (!conv || !outFmt) return empty; + + ElCapMicSink* sink = [[ElCapMicSink alloc] init]; + const int64_t want = secs * sr; /* frames we are waiting for */ + + { + char msg[192]; + snprintf(msg, sizeof(msg), + "MIC: opening the microphone for %llds -> %lld Hz mono PCM " + "(local, never egresses).", (long long)secs, (long long)sr); + el_cap_disclose(msg); + } + + const double ratio = (double)sr / hwFmt.sampleRate; + + @try { + [input installTapOnBus:0 + bufferSize:4096 + format:hwFmt + block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { + (void)when; + if (!buf || buf.frameLength == 0) return; + + AVAudioFrameCount cap = + (AVAudioFrameCount)((double)buf.frameLength * ratio) + 1024; + AVAudioPCMBuffer* out = + [[AVAudioPCMBuffer alloc] initWithPCMFormat:outFmt frameCapacity:cap]; + if (!out) return; + + __block BOOL fed = NO; + AVAudioConverterInputBlock feed = + ^AVAudioBuffer* _Nullable (AVAudioPacketCount need, + AVAudioConverterInputStatus* _Nonnull status) { + (void)need; + if (fed) { *status = AVAudioConverterInputStatus_NoDataNow; return nil; } + fed = YES; + *status = AVAudioConverterInputStatus_HaveData; + return buf; + }; + + NSError* err = nil; + AVAudioConverterOutputStatus st = + [conv convertToBuffer:out error:&err withInputFromBlock:feed]; + if (st == AVAudioConverterOutputStatus_Error || out.frameLength == 0) return; + + const int16_t* src = out.int16ChannelData ? out.int16ChannelData[0] : NULL; + if (!src) return; + + NSUInteger n = (NSUInteger)out.frameLength; + [sink.lock lock]; + if (outCh == 1) { + [sink.pcm appendBytes:src length:n * sizeof(int16_t)]; + } else { + /* Interleaved: stride to channel 0. */ + for (NSUInteger i = 0; i < n; i++) { + int16_t v = src[i * outCh]; + [sink.pcm appendBytes:&v length:sizeof(int16_t)]; + } + } + [sink.lock unlock]; + }]; + tapped = 1; + + [engine prepare]; + NSError* startErr = nil; + if (![engine startAndReturnError:&startErr]) { + [input removeTapOnBus:0]; + return empty; + } + } @catch (NSException* e) { + (void)e; + @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } + @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } + return empty; + } + + /* Wait for `want` frames, bounded by the material's own duration plus a + * margin. A device that stops producing must not become a hang. */ + const int64_t deadline_us = (secs + 5) * 1000000; + int64_t waited_us = 0; + const int64_t tick_us = 5000; + for (;;) { + [sink.lock lock]; + int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); + [sink.lock unlock]; + if (have >= want || waited_us >= deadline_us) break; + usleep((useconds_t)tick_us); + waited_us += tick_us; + } + + @try { [input removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } + @try { [engine stop]; } @catch (NSException* e) { (void)e; } + engine = nil; + + /* Hand up exactly what was asked for, or everything we got if the device + * came up short. Never padded: silence we invented is indistinguishable + * from silence we heard, and El has no way to tell them apart afterwards. */ + [sink.lock lock]; + int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); + int64_t n = have < want ? have : want; + const int16_t* pcm = (const int16_t*)[sink.pcm bytes]; + el_val_t list = empty; + for (int64_t i = 0; i < n; i++) { + list = el_list_append(list, (el_val_t)(int64_t)pcm[i]); + } + [sink.lock unlock]; + + return list; +} + +el_val_t mic_available(void) { + if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; + return (el_val_t)(el_cap_device_present(AVMediaTypeAudio) ? 1 : 0); +} + +el_val_t mic_request_access(void) { + return (el_val_t)(el_cap_request(AVMediaTypeAudio) ? 1 : 0); +} + +/* ════════════════════════════════════════════════════════════════════════════ + * MICROPHONE — live monitor (the full-duplex ear) + * + * converse needs to keep listening WHILE it speaks, which means the mic is open + * at the same time as the speaker. In a real room that is a feedback path: + * without cancellation Neuron hears its own voice, decides someone is talking, + * and barges in on itself. setVoiceProcessingEnabled: hands the input node to + * the OS voice-processing unit, which subtracts the known output signal from + * the input — the single thing that makes barge-in work outside a headset. + * + * It is not always available (some aggregate and virtual devices refuse it), so + * failure to enable it is reported as a DISTINCT return value (2) rather than + * folded into success. The caller needs to know, because the correct response + * is to raise the VAD floor, and a caller that thinks AEC is on will set that + * floor far too low. + * + * The tap keeps only a running short-window RMS in a static behind a mutex. + * Deliberately not a queue of samples: this path is polled at ~50 Hz by a loop + * that only ever asks "is someone talking", and buffering audio nobody reads + * would be an unbounded allocation in the middle of a conversation. + * ══════════════════════════════════════════════════════════════════════════ */ + +static AVAudioEngine* g_mon_engine = nil; +static int g_mon_running = 0; +static int g_mon_code = 0; /* what the successful start reported */ +static double g_mon_rms = 0.0; +static pthread_mutex_t g_mon_lock = PTHREAD_MUTEX_INITIALIZER; + +el_val_t mic_monitor_start(void) { + /* Idempotent, and it re-reports the ORIGINAL code rather than a bare 1: a + * caller that starts twice must not be told AEC is on when the first start + * already discovered it was not. */ + if (g_mon_running) return (el_val_t)g_mon_code; + if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; + + AVAudioEngine* engine = nil; + AVAudioInputNode* input = nil; + AVAudioFormat* fmt = nil; + int aec = 0; + int tapped = 0; + + @try { + engine = [[AVAudioEngine alloc] init]; + input = [engine inputNode]; + } @catch (NSException* e) { + (void)e; + return (el_val_t)0; + } + if (!input) return (el_val_t)0; + + /* Enable AEC BEFORE reading the format: the voice-processing unit imposes + * its own input format, and a tap installed with the pre-VP format would be + * rejected at start. */ + @try { + NSError* vpErr = nil; + if ([input respondsToSelector:@selector(setVoiceProcessingEnabled:error:)]) { + aec = [input setVoiceProcessingEnabled:YES error:&vpErr] ? 1 : 0; + } + } @catch (NSException* e) { + (void)e; + aec = 0; + } + + @try { + fmt = [input inputFormatForBus:0]; + } @catch (NSException* e) { + (void)e; + return (el_val_t)0; + } + if (!fmt || fmt.sampleRate <= 0 || fmt.channelCount == 0) return (el_val_t)0; + + el_cap_disclose(aec + ? "MIC: opening the microphone for live monitoring, echo-cancelled (local)." + : "MIC: opening the microphone for live monitoring, NO echo cancellation (local)."); + + @try { + [input installTapOnBus:0 + bufferSize:1024 + format:fmt + block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { + (void)when; + if (!buf) return; + AVAudioFrameCount n = buf.frameLength; + if (n == 0) return; + + double sum = 0.0; + /* Whatever the VP unit hands back — float32 is the norm, int16 and + * int32 are possible on odd hardware — normalise to -1..1 so the + * Float El sees means the same thing on every device. */ + if (buf.floatChannelData) { + const float* ch = buf.floatChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) sum += (double)ch[i] * (double)ch[i]; + } else if (buf.int16ChannelData) { + const int16_t* ch = buf.int16ChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) { + double v = (double)ch[i] / 32768.0; + sum += v * v; + } + } else if (buf.int32ChannelData) { + const int32_t* ch = buf.int32ChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) { + double v = (double)ch[i] / 2147483648.0; + sum += v * v; + } + } else { + return; + } + + double rms = sqrt(sum / (double)n); + if (rms < 0.0) rms = 0.0; + if (rms > 1.0) rms = 1.0; + + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = rms; + pthread_mutex_unlock(&g_mon_lock); + }]; + tapped = 1; + + [engine prepare]; + NSError* startErr = nil; + if (![engine startAndReturnError:&startErr]) { + [input removeTapOnBus:0]; + return (el_val_t)0; + } + } @catch (NSException* e) { + (void)e; + @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } + @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } + return (el_val_t)0; + } + + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = 0.0; + pthread_mutex_unlock(&g_mon_lock); + + g_mon_engine = engine; + g_mon_running = 1; + g_mon_code = aec ? 1 : 2; + return (el_val_t)g_mon_code; +} + +/* Float in 0..1. Reads the last window the tap computed; never blocks on the + * audio thread beyond the mutex, because this is polled inside a turn-taking + * loop where a stall IS a missed barge-in. */ +el_val_t mic_monitor_rms(void) { + double rms = 0.0; + pthread_mutex_lock(&g_mon_lock); + rms = g_mon_rms; + pthread_mutex_unlock(&g_mon_lock); + return el_from_float(rms); +} + +el_val_t mic_monitor_stop(void) { + AVAudioEngine* engine = g_mon_engine; + g_mon_engine = nil; + g_mon_running = 0; + g_mon_code = 0; + + if (engine) { + @try { [[engine inputNode] removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } + @try { [engine stop]; } @catch (NSException* e) { (void)e; } + } + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = 0.0; + pthread_mutex_unlock(&g_mon_lock); + return (el_val_t)1; +} + +/* ════════════════════════════════════════════════════════════════════════════ + * CAMERA + * ══════════════════════════════════════════════════════════════════════════ */ + +static void el_cap_free_bitmap(void* info, const void* data, size_t size) { + (void)info; (void)size; + free((void*)data); +} + +/* CVPixelBuffer -> CGImage, own-core, no CoreImage. + * + * The output is pinned to 32BGRA at the AVCaptureVideoDataOutput (see below) + * precisely so this conversion can be a memcpy and a CGImageCreate. The + * alternative — accepting the camera's native 2vuy/420v and colour-converting + * here — would mean either pulling in CoreImage or writing a YUV->RGB matrix in + * the file that is supposed to contain no arithmetic. Asking the capture output + * for BGRA moves that work into AVFoundation, where it is already written and + * already hardware-accelerated. + * + * The rows are copied out rather than aliased because the CVPixelBuffer is + * recycled by the capture session the moment the delegate returns; a CGImage + * pointing at it would be pointing at the NEXT frame by the time anyone looked. */ +static CGImageRef el_cap_cgimage_from_pixelbuffer(CVPixelBufferRef pb) { + if (!pb) return NULL; + if (CVPixelBufferGetPixelFormatType(pb) != kCVPixelFormatType_32BGRA) return NULL; + if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return NULL; + + size_t w = CVPixelBufferGetWidth(pb); + size_t h = CVPixelBufferGetHeight(pb); + size_t src_bpr = CVPixelBufferGetBytesPerRow(pb); + const uint8_t* base = (const uint8_t*)CVPixelBufferGetBaseAddress(pb); + + CGImageRef img = NULL; + if (base && w > 0 && h > 0 && src_bpr >= w * 4) { + size_t dst_bpr = w * 4; + uint8_t* copy = (uint8_t*)malloc(dst_bpr * h); + if (copy) { + for (size_t y = 0; y < h; y++) { + memcpy(copy + y * dst_bpr, base + y * src_bpr, dst_bpr); + } + CGDataProviderRef dp = + CGDataProviderCreateWithData(NULL, copy, dst_bpr * h, el_cap_free_bitmap); + if (dp) { + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + if (cs) { + img = CGImageCreate(w, h, 8, 32, dst_bpr, cs, + (CGBitmapInfo)(kCGBitmapByteOrder32Little | + kCGImageAlphaNoneSkipFirst), + dp, NULL, false, kCGRenderingIntentDefault); + CGColorSpaceRelease(cs); + } + CGDataProviderRelease(dp); /* provider owns `copy` from here */ + } else { + free(copy); + } + } + } + + CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + return img; +} + +/* The frame delegate. AVCaptureVideoDataOutput is used rather than + * AVCapturePhotoOutput for the same reason periph.swift used it: the photo path + * wants KVO and a session owned by an app object, and this runs in a plain CLI + * process with no run loop it can assume. A data output just calls back. + * + * The first frames are dropped on purpose. A camera that has just been powered + * on is still converging exposure and white balance, and the first frame is + * reliably darker and greener than the room. El's scene-geometry descriptors + * are brightness and mean-colour statistics, so handing up an unsettled frame + * would not produce a slightly worse answer, it would produce a confidently + * wrong one. */ +@interface ElCapFrameGrabber : NSObject { + CGImageRef _img; + int _seen; + dispatch_semaphore_t _sem; +} +- (dispatch_semaphore_t)sem; +- (CGImageRef)takeImage; /* transfers ownership to the caller */ +@end + +@implementation ElCapFrameGrabber + +- (instancetype)init { + self = [super init]; + if (self) { + _img = NULL; + _seen = 0; + _sem = dispatch_semaphore_create(0); + } + return self; +} + +- (dispatch_semaphore_t)sem { return _sem; } + +- (CGImageRef)takeImage { + CGImageRef out = _img; + _img = NULL; + return out; +} + +- (void)dealloc { + if (_img) { CGImageRelease(_img); _img = NULL; } +} + +/* Runs on the serial delegate queue, so no lock is needed among callbacks; the + * waiter only reads _img after the semaphore has been signalled AND the session + * has been stopped, which orders it after the last callback. */ +- (void)captureOutput:(AVCaptureOutput*)output +didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection*)connection { + (void)output; (void)connection; + _seen++; + if (_img != NULL || _seen < 5) return; /* let exposure settle */ + + CVImageBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pb) return; + CGImageRef img = el_cap_cgimage_from_pixelbuffer(pb); + if (!img) return; + _img = img; + dispatch_semaphore_signal(_sem); +} + +@end + +/* Bring the camera up, take exactly one settled frame, put it back down. + * Returns a +1 CGImageRef the caller releases, or NULL. Bounded at 10s: a + * camera held by another process, or one whose TCC grant was revoked between + * the check and the open, must fail rather than park. */ +static CGImageRef el_cap_grab_frame(void) { + AVCaptureSession* session = nil; + AVCaptureVideoDataOutput* output = nil; + ElCapFrameGrabber* grabber = nil; + CGImageRef img = NULL; + + @try { + AVCaptureDevice* dev = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; + if (!dev) return NULL; + + NSError* err = nil; + AVCaptureDeviceInput* in = [AVCaptureDeviceInput deviceInputWithDevice:dev error:&err]; + if (!in) return NULL; + + session = [[AVCaptureSession alloc] init]; + session.sessionPreset = AVCaptureSessionPresetPhoto; + if (![session canAddInput:in]) return NULL; + [session addInput:in]; + + output = [[AVCaptureVideoDataOutput alloc] init]; + output.alwaysDiscardsLateVideoFrames = YES; + /* Pin the pixel format so the CGImage conversion above stays a memcpy. + * Every macOS capture device advertises 32BGRA. */ + output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey : + @(kCVPixelFormatType_32BGRA) }; + + grabber = [[ElCapFrameGrabber alloc] init]; + dispatch_queue_t q = dispatch_queue_create("el.capture.camera", DISPATCH_QUEUE_SERIAL); + [output setSampleBufferDelegate:grabber queue:q]; + + if (![session canAddOutput:output]) return NULL; + [session addOutput:output]; + + el_cap_disclose("CAMERA: opening the camera for one frame (local, never egresses)."); + [session startRunning]; + } @catch (NSException* e) { + (void)e; + @try { [session stopRunning]; } @catch (NSException* e2) { (void)e2; } + return NULL; + } + + long timed_out = dispatch_semaphore_wait([grabber sem], + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC))); + + /* Stop first, then detach the delegate, then read. In that order the last + * callback has already returned by the time anyone touches the image. */ + @try { [session stopRunning]; } @catch (NSException* e) { (void)e; } + @try { [output setSampleBufferDelegate:nil queue:NULL]; } @catch (NSException* e) { (void)e; } + + if (timed_out == 0) img = [grabber takeImage]; + return img; +} + +el_val_t camera_available(void) { + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + return (el_val_t)(el_cap_device_present(AVMediaTypeVideo) ? 1 : 0); +} + +el_val_t camera_request_access(void) { + return (el_val_t)(el_cap_request(AVMediaTypeVideo) ? 1 : 0); +} + +/* Longest edge of the grid handed to El. 64 is not a resolution, it is a budget: + * a 1920x1080 frame is 6.2 MILLION packed RGB ints, and building that as an El + * list would cost more time and memory than everything El then does with it. + * The descriptors El computes over this — mean colour, brightness, a 3x3 + * luminance grid — are region statistics, and region statistics do not get + * meaningfully better above a 64-wide grid. The TRUE frame dimensions are + * reported separately so nothing downstream has to guess what was thrown away. */ +#define EL_CAP_GRID_MAX 64 + +/* One frame as Map{width, height, grid_w, grid_h, pixels:[Int]}. + * + * "pixels" is packed R,G,B with NO alpha — three ints per grid cell, row-major + * from the TOP-LEFT. (CGBitmapContext lays its buffer out top row first and + * CGContextDrawImage does the flip, so row 0 here is the top of the frame, the + * same convention periph.swift's grid indexing assumed.) Alpha is dropped + * because a camera frame has none worth carrying and it would inflate the list + * by a third to say "opaque" six thousand times. */ +el_val_t camera_capture_rgb(void) { + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + + CGImageRef img = el_cap_grab_frame(); + if (!img) return (el_val_t)0; + + size_t w = CGImageGetWidth(img); + size_t h = CGImageGetHeight(img); + if (w == 0 || h == 0) { CGImageRelease(img); return (el_val_t)0; } + + /* Preserve aspect ratio, longest edge capped. */ + size_t gw = w, gh = h; + size_t longest = w > h ? w : h; + if (longest > EL_CAP_GRID_MAX) { + double s = (double)EL_CAP_GRID_MAX / (double)longest; + gw = (size_t)((double)w * s + 0.5); + gh = (size_t)((double)h * s + 0.5); + if (gw == 0) gw = 1; + if (gh == 0) gh = 1; + } + + size_t bpr = gw * 4; + uint8_t* buf = (uint8_t*)calloc(1, bpr * gh); + if (!buf) { CGImageRelease(img); return (el_val_t)0; } + + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + CGContextRef ctx = cs ? CGBitmapContextCreate(buf, gw, gh, 8, bpr, cs, + (CGBitmapInfo)kCGImageAlphaPremultipliedLast) + : NULL; + if (cs) CGColorSpaceRelease(cs); + if (!ctx) { free(buf); CGImageRelease(img); return (el_val_t)0; } + + /* Nearest-neighbour. This is a decimation for statistics, not a thumbnail + * for a human to look at; smoothing would only cost time and blur the very + * region boundaries the grid exists to measure. */ + CGContextSetInterpolationQuality(ctx, kCGInterpolationNone); + CGContextDrawImage(ctx, CGRectMake(0, 0, (CGFloat)gw, (CGFloat)gh), img); + CGContextRelease(ctx); + CGImageRelease(img); + + el_val_t pixels = el_list_empty(); + for (size_t y = 0; y < gh; y++) { + const uint8_t* row = buf + y * bpr; + for (size_t x = 0; x < gw; x++) { + const uint8_t* p = row + x * 4; /* RGBA8, premultiplied-last */ + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[0]); + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[1]); + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[2]); + } + } + free(buf); + + el_val_t m = el_map_new((el_val_t)0); + if (!m) return (el_val_t)0; + m = el_map_set(m, EL_STR("width"), (el_val_t)(int64_t)w); + m = el_map_set(m, EL_STR("height"), (el_val_t)(int64_t)h); + m = el_map_set(m, EL_STR("grid_w"), (el_val_t)(int64_t)gw); + m = el_map_set(m, EL_STR("grid_h"), (el_val_t)(int64_t)gh); + m = el_map_set(m, EL_STR("pixels"), pixels); + return m; +} + +/* One frame to disk as JPEG, at FULL resolution — the opposite budget from + * camera_capture_rgb, and for the opposite reason. A file is not being walked + * element-by-element by an interpreter; it costs one ImageIO call and it is the + * artefact a human or a later pass will actually look at. The encoder is + * ImageIO's because a JPEG encoder is a codec, and re-implementing one in El + * would be a large amount of arithmetic that buys nothing: the point of keeping + * work in El is the reasoning, not the entropy coding. */ +el_val_t camera_capture_jpeg(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p || !*p) return (el_val_t)0; + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + + CGImageRef img = el_cap_grab_frame(); + if (!img) return (el_val_t)0; + + int ok = 0; + @autoreleasepool { + NSString* ns = [NSString stringWithUTF8String:p]; + NSURL* url = ns ? [NSURL fileURLWithPath:ns] : nil; + if (url) { + CGImageDestinationRef dst = + CGImageDestinationCreateWithURL((__bridge CFURLRef)url, CFSTR("public.jpeg"), 1, NULL); + if (dst) { + CGImageDestinationAddImage(dst, img, NULL); + ok = CGImageDestinationFinalize(dst) ? 1 : 0; + CFRelease(dst); + } + } + } + CGImageRelease(img); + return (el_val_t)ok; +} + +#endif /* __APPLE__ */ diff --git a/lang/runtime/el_peripheral_null.c b/lang/runtime/el_peripheral_null.c new file mode 100644 index 0000000..6e333c8 --- /dev/null +++ b/lang/runtime/el_peripheral_null.c @@ -0,0 +1,89 @@ +/* el_peripheral_null.c — the no-device build of El's I/O organ. + * + * Every entry point declared in el_runtime.h's "Peripheral" block, implemented + * as an honest refusal. This is what a platform without an El audio/capture + * realizer links instead of el_audio_darwin.m + el_capture_darwin.m, so an El + * program that speaks or listens still COMPILES AND LINKS everywhere. + * + * The distinction that matters: these do not pretend. speaker_available() and + * mic_available() return 0, and every operation returns its failure sentinel. + * A program asking "can I speak here?" gets a truthful no, rather than a + * silence it would have to infer something from. Silent success is the failure + * mode this whole change exists to eliminate — El spent this entire codebase's + * history writing WAV files full of zeros and reporting ok=true, and nobody + * caught it because nothing ever said "there is no sound here". + * + * Compiled INSTEAD OF the Darwin realizers, never alongside them — the symbols + * are the same by design, which is the point: the El side never branches on + * platform, it branches on speaker_available(). + */ + +#include "el_runtime.h" + +#if !defined(__APPLE__) + +/* ── Speaker ─────────────────────────────────────────────────────────────── */ + +el_val_t speaker_available(void) { return (el_val_t)0; } +el_val_t speaker_name(void) { return EL_STR("none"); } + +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { + (void)samples; (void)sample_rate; + return (el_val_t)0; +} + +el_val_t speaker_play_wav(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { + (void)samples; (void)sample_rate; + return (el_val_t)0; +} + +el_val_t speaker_play_wav_async(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +el_val_t speaker_pause(void) { return (el_val_t)0; } +el_val_t speaker_resume(void) { return (el_val_t)0; } +el_val_t speaker_playing(void) { return (el_val_t)0; } +el_val_t speaker_stop(void) { return (el_val_t)0; } +el_val_t speaker_played_frames(void) { return (el_val_t)0; } + +/* WAV geometry is pure parsing and would work fine here, but reporting a + * duration for audio this build cannot play would invite a caller to sequence + * around a silence. Refuse consistently with the rest of the file. */ +el_val_t wav_frames(el_val_t path) { (void)path; return (el_val_t)0; } +el_val_t wav_rate(el_val_t path) { (void)path; return (el_val_t)0; } + +/* ── Microphone ──────────────────────────────────────────────────────────── */ + +el_val_t mic_available(void) { return (el_val_t)0; } +el_val_t mic_request_access(void) { return (el_val_t)0; } + +/* Empty list, not 0: the contract says capture returns samples, and a caller + * iterating the result must find nothing rather than dereference a non-list. */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { + (void)seconds; (void)sample_rate; + return el_list_empty(); +} + +el_val_t mic_monitor_start(void) { return (el_val_t)0; } +el_val_t mic_monitor_rms(void) { return el_from_float(0.0); } +el_val_t mic_monitor_stop(void) { return (el_val_t)0; } + +/* ── Camera ──────────────────────────────────────────────────────────────── */ + +el_val_t camera_available(void) { return (el_val_t)0; } +el_val_t camera_request_access(void) { return (el_val_t)0; } +el_val_t camera_capture_rgb(void) { return (el_val_t)0; } + +el_val_t camera_capture_jpeg(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +#endif /* !__APPLE__ */ diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index cfae8e8..b0a41c3 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -80,6 +80,92 @@ void println(el_val_t s); void print(el_val_t s); el_val_t readline(void); +/* stderr counterpart of println (defined in el_seed.c). El could write to + * stdout and nowhere else, which is right for a program's RESULT and wrong for + * everything about how that result was produced. Disclosure especially has to + * leave on a stream the caller can separate from the answer: a program that + * announces "I am about to open the microphone" on stdout has corrupted its own + * output. Flushed on every call, so a disclosure reaches the terminal BEFORE + * the device it describes is touched rather than whenever the buffer drains. */ +void eprintln(el_val_t s); + +/* ── Peripheral: the speaker, the microphone, the camera ───────────────────── + * + * El's I/O organ. Implemented per platform in its OWN translation unit — + * el_audio_darwin.m / el_capture_darwin.m on Darwin, el_peripheral_null.c + * everywhere else — so El code that speaks or listens links on every platform + * and merely reports having no device where there isn't one. Declared here and + * deliberately NOT implemented in el_runtime.c: acquiring a device must not + * mean editing the middle of the language, the same rule the realizer registry + * follows for modalities. + * + * These are the ONLY parts of the organ that are not El. Everything above the + * sample buffer — WAV encode/decode, LPC autocorrelation, Levinson-Durbin, + * formant extraction, source-filter resynthesis, the compact descriptors, the + * converse decision loop — is arithmetic, and arithmetic belongs in El. What + * remains here is what El cannot express: handing a buffer to the DAC and + * waiting for it to drain, and asking the OS for frames off a capture device. + * + * Local by construction: none of these entry points has a network path. Samples + * and pixels go to and from local hardware and nowhere else. Consent is + * enforced ABOVE this layer in El (peripheral/src/organ.el) for the Neuron-level + * grant, and BELOW it by the OS for TCC; capture fails closed on either. */ + +/* Speaker (efferent). speaker_play_pcm16 BLOCKS until the audio has actually + * been played rather than merely queued, so a caller can sequence utterances + * without guessing durations and without clipping each tail. */ +el_val_t speaker_available(void); /* 1 if a real speaker backs this build */ +el_val_t speaker_name(void); /* backend id, e.g. "coreaudio-audioqueue" */ +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate); /* [Int] 16-bit mono; 1 ok */ +el_val_t speaker_play_wav(el_val_t path); /* 16-bit mono RIFF/WAVE; 1 ok */ + +/* Asynchronous playback — required by converse, which must keep listening while + * it speaks and must be able to stop ON THE SPOT mid-buffer. A blocking play + * cannot be interrupted, and "finish the current buffer" is not barge-in. + * speaker_stop() halts output immediately; speaker_playing() reports whether + * the hardware is still going; speaker_played_frames() is how far it actually + * got, which is what makes an interrupted utterance resumable at the sample. */ +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate); +el_val_t speaker_play_wav_async(el_val_t path); +el_val_t speaker_pause(void); /* stop AT THIS SAMPLE, keep position */ +el_val_t speaker_resume(void); /* carry on from exactly there */ +el_val_t speaker_playing(void); +el_val_t speaker_stop(void); +el_val_t speaker_played_frames(void); + +/* WAV geometry without playing — wav-info, and the segment duration converse + * needs to turn elapsed time into progress. */ +el_val_t wav_frames(el_val_t path); +el_val_t wav_rate(el_val_t path); + +/* Microphone (afferent). Fails CLOSED: returns 0 unless the OS has granted + * capture access. mic_capture_pcm16 blocks for `seconds` and returns an [Int] + * of 16-bit mono samples at `sample_rate` — the raw stream is handed to El and + * never written anywhere by this layer. mic_available() reports device + + * permission state without prompting. */ +el_val_t mic_available(void); /* 1 device present AND OS-authorized */ +el_val_t mic_request_access(void); /* prompt once; 1 if granted */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate); /* [Int], empty on refusal */ + +/* Live monitoring for full-duplex converse. mic_monitor_start enables the OS + * voice-processing unit (acoustic echo cancellation) so the microphone does not + * hear the speaker — without AEC, Neuron barges in on its own voice and + * turn-taking is unusable in a real room. mic_monitor_rms returns the current + * short-window RMS as a Float in 0..1. */ +el_val_t mic_monitor_start(void); /* 1 ok; 2 = started but AEC unavailable */ +el_val_t mic_monitor_rms(void); /* Float */ +el_val_t mic_monitor_stop(void); + +/* Camera (afferent). Fails CLOSED like the microphone. camera_capture_rgb + * returns a Map with width/height and the frame as an [Int] of packed RGB + * bytes, so the descriptor arithmetic can happen in El rather than here. + * camera_capture_jpeg writes an encoded frame via ImageIO, which is a codec and + * not something El should re-implement. */ +el_val_t camera_available(void); +el_val_t camera_request_access(void); +el_val_t camera_capture_rgb(void); /* Map{width,height,pixels:[Int]} or 0 */ +el_val_t camera_capture_jpeg(el_val_t path); /* 1 ok */ + /* ── String builtins ─────────────────────────────────────────────────────── */ el_val_t el_str_concat(el_val_t a, el_val_t b); diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index 8313183..15b5ba6 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -154,9 +154,18 @@ static void seed_request_start(void) { * file still links on its own. */ __attribute__((weak)) void el_str_cache_flush(void); +/* Byte-buffer capacity registry (defined below, next to the string + * primitives). The arena frees the pointers it tracked, so any capacity + * entry for those addresses must go with them — otherwise a later malloc + * reusing the address would inherit a stale width. */ +static void seed_cap_drop(const char* p); + static void seed_request_end(void) { _seed_arena_on = 0; - for (size_t i = 0; i < _seed_arena.count; i++) free(_seed_arena.ptrs[i]); + for (size_t i = 0; i < _seed_arena.count; i++) { + seed_cap_drop(_seed_arena.ptrs[i]); + free(_seed_arena.ptrs[i]); + } _seed_arena.count = 0; if (el_str_cache_flush) el_str_cache_flush(); /* freed pointers may be reused */ } @@ -188,6 +197,114 @@ static char* seed_strbuf(size_t n) { static el_val_t seed_wrap_str(char* s) { return EL_STR(s); } +/* ── Byte-buffer capacity registry ──────────────────────────────────────────── + * A String produced by __str_alloc is a fixed-size BYTE BUFFER, not text. Its + * length is the capacity it was asked for; strlen() is meaningless on it, + * because the buffer is zero-filled and binary content (PCM audio, RIFF + * headers, image rasters) contains NUL bytes by nature. + * + * Before this registry existed, __str_set_char bounds-checked the write index + * against strlen(p). For a freshly __str_alloc'd buffer strlen(p) == 0, so the + * check `idx >= len` rejected EVERY index and the function was a total no-op: + * every El program that built bytes this way wrote a file of pure zeros and + * still saw a success return. That is why El's own-core WAV writer emitted + * 55,244 silent bytes with a correct-looking header length and no header. + * + * The fix cannot be "trust the index", because that removes the bound. It also + * cannot be a length header stored behind the pointer, because __str_set_char + * accepts any String — including a string literal in .rodata, where reading the + * bytes preceding the pointer is undefined and may fault. So capacity is kept + * in a side table keyed by the pointer itself: allocation registers, the arena + * sweep unregisters, and anything not registered keeps the exact strlen + * behaviour it had before. Text semantics are unchanged; byte buffers gain the + * bound they always should have had. */ + +typedef struct { + char* ptr; /* NULL = empty slot, (char*)1 = tombstone */ + size_t cap; +} SeedCapEntry; + +#define SEED_CAP_TOMB ((char*)1) + +static _Thread_local SeedCapEntry* _seed_cap = NULL; +static _Thread_local size_t _seed_cap_mask = 0; /* table size - 1 */ +static _Thread_local size_t _seed_cap_used = 0; /* live + tombstoned */ + +static size_t seed_cap_hash(const char* p) { + uintptr_t h = (uintptr_t)p >> 4; /* malloc alignment: low bits are dead */ + h *= (uintptr_t)0x9E3779B97F4A7C15ull; + return (size_t)(h >> 32); +} + +static void seed_cap_put(char* p, size_t cap); + +static void seed_cap_grow(void) { + size_t old_size = _seed_cap_mask ? _seed_cap_mask + 1 : 0; + SeedCapEntry* old = _seed_cap; + size_t new_size = old_size ? old_size * 2 : 256; + SeedCapEntry* fresh = calloc(new_size, sizeof(SeedCapEntry)); + if (!fresh) return; /* out of memory: keep old table */ + _seed_cap = fresh; + _seed_cap_mask = new_size - 1; + _seed_cap_used = 0; + for (size_t i = 0; i < old_size; i++) { + if (old[i].ptr && old[i].ptr != SEED_CAP_TOMB) seed_cap_put(old[i].ptr, old[i].cap); + } + free(old); +} + +static void seed_cap_put(char* p, size_t cap) { + if (!p) return; + if (!_seed_cap || (_seed_cap_used + 1) * 4 >= (_seed_cap_mask + 1) * 3) { + seed_cap_grow(); + if (!_seed_cap) return; + } + size_t i = seed_cap_hash(p) & _seed_cap_mask; + size_t first_free = (size_t)-1; + for (;;) { + char* e = _seed_cap[i].ptr; + if (e == p) { _seed_cap[i].cap = cap; return; } /* address reused */ + if (e == SEED_CAP_TOMB && first_free == (size_t)-1) first_free = i; + if (!e) { + if (first_free != (size_t)-1) i = first_free; else _seed_cap_used++; + _seed_cap[i].ptr = p; + _seed_cap[i].cap = cap; + return; + } + i = (i + 1) & _seed_cap_mask; + } +} + +/* Capacity of a registered byte buffer, or -1 when the pointer is not one. */ +static int64_t seed_cap_get(const char* p) { + if (!p || !_seed_cap) return -1; + size_t i = seed_cap_hash(p) & _seed_cap_mask; + for (;;) { + char* e = _seed_cap[i].ptr; + if (!e) return -1; + if (e == (char*)p) return (int64_t)_seed_cap[i].cap; + i = (i + 1) & _seed_cap_mask; + } +} + +static void seed_cap_drop(const char* p) { + if (!p || !_seed_cap) return; + size_t i = seed_cap_hash(p) & _seed_cap_mask; + for (;;) { + char* e = _seed_cap[i].ptr; + if (!e) return; + if (e == (char*)p) { _seed_cap[i].ptr = SEED_CAP_TOMB; return; } + i = (i + 1) & _seed_cap_mask; + } +} + +/* Effective addressable length of a String: its buffer capacity when it is a + * byte buffer, otherwise strlen. */ +static int64_t seed_addressable_len(const char* p) { + int64_t cap = seed_cap_get(p); + return cap >= 0 ? cap : (int64_t)strlen(p); +} + /* ── String primitives ───────────────────────────────────────────────────── */ el_val_t __str_len(el_val_t s) { @@ -199,7 +316,7 @@ el_val_t __str_len(el_val_t s) { el_val_t __str_char_at(el_val_t s, el_val_t i) { const char* p = EL_CSTR(s); if (!p) return 0; - int64_t len = (int64_t)strlen(p); + int64_t len = seed_addressable_len(p); /* capacity for byte buffers */ int64_t idx = (int64_t)i; if (idx < 0 || idx >= len) return 0; return (el_val_t)(unsigned char)p[idx]; @@ -210,13 +327,14 @@ el_val_t __str_alloc(el_val_t n) { if (sz < 0) sz = 0; char* buf = seed_strbuf((size_t)sz); memset(buf, 0, (size_t)sz + 1); + seed_cap_put(buf, (size_t)sz); /* this is a byte buffer of width sz */ return seed_wrap_str(buf); } el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c) { char* p = (char*)(uintptr_t)s; if (!p) return s; - int64_t len = (int64_t)strlen(p); + int64_t len = seed_addressable_len(p); /* capacity for byte buffers */ int64_t idx = (int64_t)i; if (idx < 0 || idx >= len) return s; p[idx] = (char)(unsigned char)(int64_t)c; @@ -406,6 +524,15 @@ el_val_t __fs_mkdir(el_val_t path) { return 1; } +/* stderr counterpart of println. Flushed immediately: a disclosure line is only + * worth anything if it lands before the thing it discloses happens. */ +void eprintln(el_val_t s) { + const char* p = EL_CSTR(s); + fputs(p ? p : "", stderr); + fputc('\n', stderr); + fflush(stderr); +} + el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n) { const char* p = EL_CSTR(path); const char* b = EL_CSTR(bytes); diff --git a/peripheral/README.md b/peripheral/README.md index 765fd35..5f1d0b4 100644 --- a/peripheral/README.md +++ b/peripheral/README.md @@ -1,80 +1,184 @@ -# peripheral — Neuron's I/O organ (own-core, local, consent-gated) +# peripheral — Neuron's I/O organ, in El -The interface made physical. Two afferent senses in, one efferent voice out — -all reached the way the agentic surface reaches any tool. +**El speaks.** The engram stores geometry and does not speak; the speaking +belongs to the language and its runtime. + +Until this landed, the organ was a 939-line Swift program (`src/periph.swift`) +that shelled out to `afplay`. Neuron's mouth and ears were a separate binary +standing next to the language, and "speak" meant "ask that binary to speak." +That program is now **reference material, not the implementation.** ``` -MIC (hear) afferent device -> capture -> descriptor -> ingest -> geometry -CAMERA (see) afferent device -> capture -> descriptor -> ingest -> scene-geometry -SPEAKER(speak) efferent render WAV -> PLAY ALOUD out the speaker +SPEAKER (speak) efferent samples ──────────────► CoreAudio ──► the room +MIC (hear) afferent device ──► samples ──► descriptor ──► engram +CAMERA (see) afferent device ──► frame ──► descriptor ──► engram ``` -Closes the conversational loop: **hear (mic) -> understand (engram) -> speak (speaker)**. +## The split, and why it falls where it does + +Exactly **two** things here are not El, and they are the two things El cannot +express as arithmetic: + +| Not El (realizers) | Why | +|---|---| +| `lang/runtime/el_audio_darwin.m` | Handing a buffer to the DAC and waiting for it to drain. There is no way to say "the hardware has now played these samples" in El, and there should not be. | +| `lang/runtime/el_capture_darwin.m` | Asking the OS for samples off a microphone or frames off a camera, plus the TCC permission dance. | + +**Everything else is El**, because everything else is arithmetic: + +| In El | Where | +|---|---| +| WAV encode / decode (chunk-walking, JUNK/FLLR tolerant) | `src/organ_dsp.el`, `elp/src/speech.el` | +| LPC autocorrelation + Levinson-Durbin (order 16 @ 16 kHz) | `src/organ_dsp.el` | +| Formant extraction off the all-pole spectral envelope | `src/organ_dsp.el` | +| Source-filter resynthesis (glottal impulse train through the filter) | `src/organ_dsp.el` | +| Audio descriptor `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` | `src/organ_dsp.el` | +| Voice descriptor `[F0, F1..F5, bandwidths]` | `src/organ_dsp.el` | +| Scene descriptor `[w, h, meanRGB, brightness, 3×3 luminance grid]` | `src/organ.el` | +| Consent, disclosure, the voice-from-engram fetch | `src/organ.el` | +| Barge-in, yield-or-hold, backchannel, resume | `src/organ_converse.el` | +| The command surface | `src/organ_cli.el` | + +Both realizers are their **own translation units**, declared in +`lang/runtime/el_runtime.h`, and deliberately **not** patches to +`el_runtime.c`. Acquiring a device must not mean editing the middle of the +language — the same rule the realizer registry follows for modalities. +`lang/runtime/el_peripheral_null.c` provides the identical entry points +everywhere else, so El that speaks links on any platform and truthfully reports +having no speaker rather than going quietly silent. + +## The voice comes from the engram + +A voice is **geometry in the engram**, not a JSON file next to the code and +certainly not constants in a source file. The organ fetches it the way anything +retrieves a memory — it asks: + +```el +let g: [Int] = organ_voice_fetch("will") +// [peripheral] VOICE: fetched 'will' FROM THE ENGRAM — +// f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531 +``` + +`organ_voice_fetch` issues an engram query and reads the geometry off the node +that comes back. Nothing opens a file. If the region is not in the graph it +returns **empty**, not a plausible default — a caller has to be able to tell +"this is how they sound" from "I never heard them." + +The reverse direction is `ingest-voice`: an LPC voiceprint becomes a node, and +from then on the voice is a memory rather than a measurement someone wrote down. + +## What the organ never does + +**It never learns a word.** Pronunciation, vocabulary and phonemes belong to the +language faculty and are already built as ingested geometry — *the engram knows +how to pronounce*. The seam is `synth_codes(codes, voice, pmap)`: the codes and +the phoneme map arrive from the language side as geometry, and the organ's whole +job is turning them into samples and getting the samples out the speaker, plus +the same trip in reverse for the senses. There is no lexicon here and no +grapheme-to-phoneme rule, by design. ## Rails -- **Own-core.** macOS-native only: AVFoundation (camera/mic), CoreAudio voice- - processing (AEC), afplay (speaker), ImageIO/CoreGraphics (frames), hand-rolled - DSP (WAV, LPC, formant synthesis). No cloud, no heavy deps. -- **Local-only.** Raw streams are written to `out/` and never egress. `.gitignore` - keeps captured media out of git. -- **Consent-gated (two locks).** A Neuron-level grant (`grant`/`revoke`) *and* the - OS TCC permission. Sensitive senses (camera/mic) fail closed without both. -- **Disclosed.** Every device touch prints a `[peripheral]` line on stderr. + +- **Own-core.** CoreAudio / AVFoundation / ImageIO — all ship with macOS. No + cloud, no model, no heavy dependency. There is **no network code in the organ + at all**, by construction. +- **Local-only.** Raw streams stay on the machine. What leaves a capture is a + descriptor of a few dozen numbers. A 1920×1080 frame becomes 15 integers + (~414,000× smaller); three seconds of audio becomes 8. +- **Consent, two locks.** A Neuron-level grant **and** the OS TCC permission. + Camera and mic **fail closed** without both. The speaker is disclosed but not + gated — you cannot secretly speak aloud, and gating it would mean Neuron needs + permission to answer. +- **Disclosed.** Every device touch prints a `[peripheral]` line on **stderr** + (via `eprintln`, flushed immediately), so a disclosure lands before the device + is touched and never contaminates the program's stdout. ## Build + +```bash +./peripheral/build.sh /tmp/organ ``` -swiftc -O -o bin/periph src/periph.swift \ - -framework AVFoundation -framework CoreMedia -framework Foundation \ - -framework CoreGraphics -framework ImageIO -framework CoreImage -``` + +Concatenates the El modules, compiles with `elc`, links the two realizers. +Run it **from the repo root** or the `.psv` phoneme data will not resolve. ## Commands + ``` -periph grant|revoke # Neuron-level consent -periph status -periph speak # SPEAK ALOUD (efferent) -periph tone [hz] [sec] # own-core WAV synth -periph listen # MIC capture (afferent), 16k mono -periph see # CAMERA one frame (afferent) -periph feat-audio | feat-image # capture -> compact descriptor -periph ingest-audio|ingest-image # descriptor -> engram node (geometry) -periph voiceprint # extract F0 + formants F1-F5 -periph imitate # speak back in that voice (LPC resynthesis) -periph hear-imitate # MIC -> signature -> imitate -> SPEAK ALOUD -periph converse [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] [--live-mic] +organ grant|revoke Neuron-level consent +organ status consent + device state +organ speak play a WAV aloud (efferent) +organ tone [hz] [ms] synthesize and play — no file at all +organ say [CODE...] fetch voice FROM THE ENGRAM, render, speak +organ listen mic capture 16k mono (afferent) +organ see one camera frame (afferent) +organ wav-info WAV geometry +organ feat-audio compact audio descriptor (8 numbers) +organ feat-image compact scene-geometry from the camera +organ voiceprint F0 + formants F1-F5 (LPC) +organ imitate LPC analysis-resynthesis +organ hear-imitate mic -> signature -> imitate -> speak aloud +organ ingest-audio descriptor -> engram node (geometry) +organ ingest-voice voiceprint -> engram voice region +organ converse [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume] ``` -## The afferent metabolism -A capture is never shipped raw. It becomes a **compact descriptor** — the afferent -twin of the music instrument-signature: -- audio -> `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` (~2400-6000x smaller) -- image -> `[w, h, meanRGB, brightness, 3x3 luminance grid]` (~400000x smaller) -- voice -> `[F0, F1..F5, bandwidths]` (11 numbers) +## Interruptibility -That descriptor is what the ingest organ (engram `POST /api/nodes`) turns into an -embedded node = geometry. +`converse` speaks an ordered, salience-tagged **meaning-plan** while listening: -## Voice by imitation -`voiceprint`/`imitate` are own-core LPC (autocorrelation + Levinson-Durbin, order -16 @ 16 kHz), formant extraction from the LPC spectral envelope, and source-filter -resynthesis (glottal impulse train at F0 through the all-pole formant filter). A -voice is grabbed by ear as ~a dozen numbers and spoken back — **no training, no -stolen voice.** Measured fidelity on real speech: resynthesized formants match the -source within 2-3%. The full phoneme->formant path for *novel* sentences is the -speech faculty's seam (`elp` audio surface profile); this engine provides the -formant synthesis primitive it renders through. +- **barge-in** — output stops at the sample, not at the end of the buffer. The + realizer exposes `pause`/`resume` and reports `played_frames` (the real DAC + position) precisely so this is possible. +- **yield-or-hold** — a decision, not a rule: `hold = salience·0.6 + + progress·0.4`, and holding also requires that the interrupter not be + high-authority. Otherwise yield, because the polite default is the right one. +- **backchannel** — "mm-hm" is brief and low-energy; resume seamlessly. +- **resumable** — on yield the remaining plan persists to `.resume.json`; + `--resume` picks the thread back up. An interruption should cost a turn, not + the content. -## Interruptibility (native turn-taking) -`converse` plays the utterance as an ordered, salience-tagged **meaning-plan** -while the mic listens (full-duplex, AEC on so it never barges in on its own voice): -- **barge-in**: user speech -> pause on the spot (sample-accurate), not "finish the buffer." -- **yield-or-hold**: a decision grounded in the current segment's salience + progress - + the interrupter's authority — YIELD (stop) or HOLD ("hang on, let me finish"). -- **backchannel** ("mm-hm"): brief/low -> keep going, resume seamlessly. -- **resumable**: on yield the remaining plan persists (`.resume.json`); `--resume` - picks the thread back up ("as I was saying"). +Live full-duplex uses `--live-mic` with the OS voice-processing unit (AEC) so +Neuron does not barge in on its own voice. `--barge-at` injects the event +deterministically for testing. -Live full-duplex uses `--live-mic` (OS AEC). Injected `--barge-at` drives the -decision loop deterministically for testing. -``` -``` +## Measured against the Swift original + +Same input (`out/mic_room.wav`, 16 kHz mono, 48121 samples), Swift `periph` +vs the El organ: + +| | Swift | El | +|---|---|---| +| seconds | 3.0075625 | 3.0076 | +| rms | 0.0047766496761 | 0.004777 | +| peak | 0.01806640625 | 0.018066 | +| zcr_hz | 416.28395087 | 416.2840 | +| centroid_hz | 727.60529169 | 727.6053 | +| f0_hz | 400 | 400.0000 | +| formants F1–F5 | 1734.375 / 3343.75 / 3875 / 4359.375 / 4468.75 | identical | +| bandwidths B1–B5 | 2000 / 2968.75 / 4203.125 / 4687.5 / 5000 | identical | + +Agreement to every printed digit. `imitate` cannot match bit-for-bit because the +Swift excites unvoiced frames with `Double.random` — two Swift runs correlate +0.957 with **each other**; El correlates **0.958** with Swift. The port is as +close to the original as the original is to itself, and the deterministic prefix +is bit-identical. + +## Honest status + +- **Works:** speaker (CoreAudio, no `afplay`, no subprocess — verified: zero + `afplay`/Swift strings in the binary, no child process during playback), mic + capture, camera capture, all descriptors, LPC voiceprint, imitate, + hear-imitate, voice fetch/ingest against the engram, converse (yield, hold, + yield-to-authority, backchannel, resume — all exercised with real audio). +- **Coarse, and labelled so:** a fetched voice is one formant triple with no + coarticulation and no prosody. It is an impression, explicitly **not a + clone**, and `prov=COARSE` says so on the node. +- **Not verified here:** live `--live-mic` barge-in in a real room with a real + interrupter. The AEC path is implemented and the deterministic path is proven; + the acoustic behaviour is not something a headless run can establish. +- **Not in the engram yet:** the structured `Voice` / `VowelTarget` geometry + nodes live in the organ's own store and in snapshot files from earlier work, + but the **production engram does not carry them**. Getting them there is an + ingest, not a code change. +- `src/periph.swift` is kept as the reference the port was measured against. diff --git a/peripheral/build.sh b/peripheral/build.sh new file mode 100755 index 0000000..4613088 --- /dev/null +++ b/peripheral/build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# build.sh — build the El organ. +# +# El has no import system on this path, so the modules are concatenated in +# dependency order (the same thing elp/tests/run.sh does) and handed to elc as +# one unit. The two device realizers are then linked in. +# +# MUST be run from the repo root, or the .psv phoneme geometry will not resolve +# and the render silently degrades. +set -uo pipefail + +OUT="${1:-./peripheral/organ}" +REPO="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# Dependency order. The elp modules supply the render (synth_codes) and the +# phoneme-geometry read; the organ supplies everything else. +cat elp/src/voice-profile.el \ + elp/src/accent.el \ + elp/src/voice-ingest.el \ + elp/src/speech-ingest.el \ + elp/src/speech.el \ + peripheral/src/organ.el \ + peripheral/src/organ_dsp.el \ + peripheral/src/organ_converse.el \ + peripheral/src/organ_cli.el \ + | grep -v '^import ' > "$WORK/organ.el" + +cd "$REPO/lang" +./dist/platform/elc "$WORK/organ.el" > "$WORK/organ.c" || { echo "elc failed" >&2; exit 1; } + +SSL_PREFIX="$(brew --prefix openssl@3 2>/dev/null || echo /usr/local)" + +# The peripheral realizers are per-platform: Darwin gets the real devices, +# anything else gets el_peripheral_null.c and honestly reports having none. +case "$(uname)" in + Darwin) + # The Objective-C realizers are compiled SEPARATELY, with -fobjc-arc. The + # capture realizer is written against ARC (it holds AVFoundation objects); + # compiling it MRR silently changes its memory semantics, which on a device + # path shows up as a use-after-free under load rather than as an error here. + cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_audio_darwin.m -o "$WORK/el_audio.o" || exit 1 + cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_capture_darwin.m -o "$WORK/el_capture.o" || exit 1 + PERIPH_SRC="$WORK/el_audio.o $WORK/el_capture.o" + PERIPH_LIBS="-framework AudioToolbox -framework AVFoundation -framework CoreMedia + -framework CoreVideo -framework CoreGraphics -framework ImageIO + -framework Foundation" + ;; + *) + PERIPH_SRC="runtime/el_peripheral_null.c" + PERIPH_LIBS="" + ;; +esac + +cc -O1 -I runtime -I"$SSL_PREFIX/include" -L"$SSL_PREFIX/lib" \ + -o "$OUT" "$WORK/organ.c" \ + runtime/el_runtime.c runtime/el_seed.c \ + runtime/engram_cognition.c runtime/engram_geometry.c runtime/engram_reason.c \ + runtime/engram_store.c runtime/engram_verify.c runtime/engram_vindex.c \ + runtime/eg_cosine_batch.c runtime/eg_cosine_batch_strategy_cpu.c \ + $PERIPH_SRC $PERIPH_LIBS \ + -lcurl -lssl -lcrypto -lpthread -lm || { echo "link failed" >&2; exit 1; } + +echo "built: $OUT" diff --git a/peripheral/src/organ.el b/peripheral/src/organ.el new file mode 100644 index 0000000..cf4af38 --- /dev/null +++ b/peripheral/src/organ.el @@ -0,0 +1,570 @@ +// organ.el — Neuron's I/O organ, in El. +// +// THE PRINCIPLE. El speaks. The engram stores geometry and does not speak. +// Before this file the organ was a 939-line Swift program standing next to the +// language (peripheral/src/periph.swift): Neuron's mouth and ears were a +// separate binary, and "speak" meant "shell out to that binary, which shells +// out to afplay." That is not a voice, it is a subprocess. The voice belongs to +// the language and its runtime. +// +// THE SPLIT. Exactly two things here are not El, and they are the two things El +// cannot express as arithmetic: +// +// the speaker — handing a buffer to the DAC and waiting for it to drain +// the capture — asking the OS for samples off a mic or frames off a camera +// +// Those live in lang/runtime/el_audio_darwin.m and el_capture_darwin.m, as +// their own translation units, declared in el_runtime.h. Everything ELSE that +// the Swift did — WAV encode and decode, LPC autocorrelation, Levinson-Durbin, +// formant extraction off the all-pole envelope, source-filter resynthesis, the +// compact descriptors, the converse yield-or-hold decision — is arithmetic, and +// arithmetic is El's. See organ_dsp.el for that half. +// +// WHERE THE VOICE COMES FROM. Not from this file, and not from a JSON manifest +// on disk. A voice is GEOMETRY IN THE ENGRAM, and the organ goes and gets it by +// asking the engram, the same way anything else asks the engram for anything: +// a query against the graph, then read the numbers off the node that comes +// back. organ_voice_fetch is that. The previous path, load_voice("...json"), +// parsed a file — which quietly made the voice a build artifact instead of a +// memory. If the region is not in the graph, the honest answer is an empty +// result, not a default voice. +// +// WHAT THE ORGAN NEVER DOES. It never learns a word. Pronunciation, vocabulary +// and phonemes are the language faculty's, already built as ingested geometry — +// "the engram knows how to pronounce." The seam is synth_codes(codes, voice, +// pmap): the codes and the phoneme map arrive from the language side as +// geometry, and the organ's whole job is turning them into samples and getting +// the samples out the speaker, plus the same trip in reverse for the senses. +// +// RAILS, all non-negotiable: +// own-core — CoreAudio / AVFoundation / ImageIO, all shipped with the OS. +// No cloud, no model, no heavy dependency. There is no network +// call anywhere in the organ, by construction. +// local-only — raw streams stay on the machine. What leaves a capture is a +// DESCRIPTOR of a few dozen numbers, never the stream. +// consent — two locks on the sensitive senses: a Neuron-level grant AND +// the OS TCC permission. Camera and mic FAIL CLOSED without +// both. The speaker is disclosed but not gated (see below). +// disclosed — every device touch prints a [peripheral] line on stderr. +// Nothing here is ever silent about being a device. + +// ── Disclosure ─────────────────────────────────────────────────────────────── +// +// stderr, not stdout: a program that announces "I am opening the microphone" on +// stdout has corrupted its own output. And flushed immediately, so the line is +// on the terminal BEFORE the device is touched — a disclosure that arrives +// after the fact is a log, not a disclosure. + +fn organ_disclose(msg: String) -> Bool { + eprintln(" [peripheral] " + msg) + return true +} + +// ── Consent, the Neuron-level lock ─────────────────────────────────────────── +// +// The OS has its own lock (TCC) and it is not enough on its own: TCC grants the +// TERMINAL access to the microphone, once, more or less forever. That says the +// user trusts the app. It does not say the user consents to THIS program +// listening THIS time. So Neuron keeps its own grant, revocable, on the same +// footing — and both must be open for a sensitive sense to work. +// +// Stored next to the organ rather than in the engram deliberately: consent must +// be inspectable and revocable without a running graph, and a permission that +// can only be revoked by the system it governs is not a permission. + +fn organ_consent_path() -> String { + let home: String = env("PERIPH_HOME") + if str_eq(home, "") { + return "peripheral/.consent.json" + } + return home + "/.consent.json" +} + +fn organ_consent_granted(device: String) -> Bool { + let raw: String = fs_read(organ_consent_path()) + if str_eq(raw, "") { + return false + } + // A device is granted only on an explicit true. Anything unparseable, + // missing or malformed reads as NOT granted — the failure direction for a + // permission file is always closed. + let key: String = "\"" + device + "\"" + let at: Int = str_index_of(raw, key) + if at < 0 { + return false + } + let tail: String = str_slice(raw, at, str_len(raw)) + let t: Int = str_index_of(tail, "true") + let f: Int = str_index_of(tail, "false") + if t < 0 { + return false + } + if f < 0 { + return true + } + // whichever token appears first after the key is this device's value + if t < f { + return true + } + return false +} + +fn organ_consent_write(camera: Bool, mic: Bool) -> Bool { + let c: String = "false" + if camera { + c = "true" + } + let m: String = "false" + if mic { + m = "true" + } + return fs_write(organ_consent_path(), "{\"camera\": " + c + ", \"mic\": " + m + "}\n") +} + +fn organ_grant(device: String) -> Bool { + let cam: Bool = organ_consent_granted("camera") + let mic: Bool = organ_consent_granted("mic") + if str_eq(device, "camera") { + cam = true + } + if str_eq(device, "mic") { + mic = true + } + let ok: Bool = organ_consent_write(cam, mic) + organ_disclose("granted '" + device + "' (Neuron-level) — raw stream stays local, never egresses.") + return ok +} + +fn organ_revoke(device: String) -> Bool { + let cam: Bool = organ_consent_granted("camera") + let mic: Bool = organ_consent_granted("mic") + if str_eq(device, "camera") { + cam = false + } + if str_eq(device, "mic") { + mic = false + } + let ok: Bool = organ_consent_write(cam, mic) + organ_disclose("revoked '" + device + "' (Neuron-level).") + return ok +} + +fn organ_consent_status() -> String { + let cam: String = "denied" + if organ_consent_granted("camera") { + cam = "granted" + } + let mic: String = "denied" + if organ_consent_granted("mic") { + mic = "granted" + } + return "camera=" + cam + " mic=" + mic +} + +// Both locks, in order, with a disclosure for each outcome. Returns false and +// says exactly which lock is shut — a refusal that does not say why is +// indistinguishable from a bug. +fn organ_may_listen() -> Bool { + if organ_consent_granted("mic") == false { + organ_disclose("CONSENT DENIED for 'mic' (Neuron-level). Run: organ grant mic") + return false + } + if mic_available() == 0 { + organ_disclose("CONSENT DENIED for 'mic' (OS/TCC), or no input device. Grant microphone access to this terminal in System Settings > Privacy.") + return false + } + organ_disclose("consent OK (Neuron + OS) for 'mic' — local only, never egresses.") + return true +} + +fn organ_may_see() -> Bool { + if organ_consent_granted("camera") == false { + organ_disclose("CONSENT DENIED for 'camera' (Neuron-level). Run: organ grant camera") + return false + } + if camera_available() == 0 { + organ_disclose("CONSENT DENIED for 'camera' (OS/TCC), or no capture device. Grant camera access to this terminal in System Settings > Privacy.") + return false + } + organ_disclose("consent OK (Neuron + OS) for 'camera' — local only, never egresses.") + return true +} + +// ── SPEAKER (efferent) ─────────────────────────────────────────────────────── +// +// Not consent-gated, and that is a deliberate asymmetry rather than an +// oversight. The microphone and camera take information OFF the user without +// them necessarily knowing; the speaker puts information INTO a room the user +// is in, audibly, which is self-disclosing by its nature — you cannot secretly +// speak aloud. So the speaker is DISCLOSED (every utterance announces itself on +// stderr) but not gated. Gating it would mean Neuron needs permission to answer. + +fn organ_speak_samples(samples: [Int], sr: Int) -> Bool { + let n: Int = native_list_len(samples) + if n <= 0 { + organ_disclose("SPEAKER: nothing to say (0 samples) — not touching the device.") + return false + } + if speaker_available() == 0 { + organ_disclose("SPEAKER: no audio output on this build (" + speaker_name() + ") — cannot speak.") + return false + } + let secs: Int = n * 1000 / sr + organ_disclose("SPEAKER: playing " + int_to_str(n) + " samples (" + int_to_str(secs) + " ms @ " + int_to_str(sr) + " Hz) ALOUD via " + speaker_name() + " (efferent).") + let ok: Int = speaker_play_pcm16(samples, sr) + if ok == 1 { + organ_disclose("SPEAKER: done — Neuron spoke aloud.") + return true + } + organ_disclose("SPEAKER: playback FAILED.") + return false +} + +fn organ_speak_wav(path: String) -> Bool { + if speaker_available() == 0 { + organ_disclose("SPEAKER: no audio output on this build — cannot speak.") + return false + } + if fs_exists(path) == false { + organ_disclose("SPEAKER: no such file: " + path) + return false + } + organ_disclose("SPEAKER: playing '" + path + "' ALOUD via " + speaker_name() + " (efferent).") + let ok: Int = speaker_play_wav(path) + if ok == 1 { + organ_disclose("SPEAKER: done — Neuron spoke aloud.") + return true + } + organ_disclose("SPEAKER: playback FAILED.") + return false +} + +// ── The voice, fetched FROM THE ENGRAM ─────────────────────────────────────── +// +// This is the part that matters most and is easiest to get subtly wrong. A +// voice is not a constant in code and it is not a JSON file next to the code — +// it is a region of the graph, put there by having heard someone, and the organ +// retrieves it the way anything retrieves a memory: by asking. +// +// The node content is the geometry, in the engram's own flat key=value form: +// voice will | f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531 ... +// so the read is: query the graph, take the returned node, pull the numbers off +// it. Nothing here opens a file. +// +// Returns [f0, f0_end, kf, f1, f2, f3], or an EMPTY list when the region is not +// in the graph. Empty is the honest answer — a caller that gets no voice must +// not be handed a plausible default and left unable to tell the difference +// between "this is how they sound" and "I never heard them." + +// Read an unsigned integer that follows `key` in `s`. Stops at the first +// non-digit, returns 0 when the key is absent. +fn organ_int_after(s: String, key: String) -> Int { + let at: Int = str_index_of(s, key) + if at < 0 { + return 0 + } + let i: Int = at + str_len(key) + let n: Int = str_len(s) + let v: Int = 0 + let seen: Int = 0 + while i < n { + let c: Int = str_char_code(s, i) + if c < 48 { + i = n + } else { + if c > 57 { + i = n + } else { + v = v * 10 + (c - 48) + seen = seen + 1 + i = i + 1 + } + } + } + if seen == 0 { + return 0 + } + return v +} + +// Ask the engram for a named voice region and read its geometry back. +fn organ_voice_fetch(name: String) -> [Int] { + let out: [Int] = native_list_empty() + let marker: String = "voice " + name + " |" + // The graph is asked by MEANING, not by id or by path. + let hits: String = engram_search_json("voice " + name + " f0 formants", 12) + let at: Int = str_index_of(hits, marker) + if at < 0 { + // Fall back to a scan of the resident graph before giving up: search is + // geometric and a small graph may not rank the region first. + let scan: String = engram_scan_nodes_json(500, 0) + at = str_index_of(scan, marker) + if at < 0 { + organ_disclose("VOICE: no region for '" + name + "' in the engram — nothing to speak with.") + return out + } + hits = scan + } + let win: String = str_slice(hits, at, at + 240) + out = native_list_append(out, organ_int_after(win, "f0=")) + out = native_list_append(out, organ_int_after(win, "f0_end=")) + out = native_list_append(out, organ_int_after(win, "kf=")) + out = native_list_append(out, organ_int_after(win, "f1=")) + out = native_list_append(out, organ_int_after(win, "f2=")) + out = native_list_append(out, organ_int_after(win, "f3=")) + organ_disclose("VOICE: fetched '" + name + "' FROM THE ENGRAM — f0=" + int_to_str(native_list_get(out, 0)) + " f0_end=" + int_to_str(native_list_get(out, 1)) + " kf=" + int_to_str(native_list_get(out, 2)) + " f1=" + int_to_str(native_list_get(out, 3)) + " f2=" + int_to_str(native_list_get(out, 4)) + " f3=" + int_to_str(native_list_get(out, 5))) + return out +} + +// Put a measured voice INTO the engram as geometry. This is the afferent end of +// the same wire: a voiceprint (organ_dsp.el's LPC analysis) becomes a node, and +// from then on the voice is a memory rather than a measurement someone happened +// to write down. `prov` carries the honesty: COARSE means one formant triple, no +// coarticulation, no prosody — an impression, explicitly not a clone. +fn organ_voice_ingest(name: String, f0: Int, f0_end: Int, kf: Int, f1: Int, f2: Int, f3: Int, src: String, prov: String) -> String { + let hub: String = engram_node("voice-signature-set " + name + " grounding=measured src=" + src, "VoiceSet", 90) + let body: String = "voice " + name + " | f0=" + int_to_str(f0) + " f0_end=" + int_to_str(f0_end) + " kf=" + int_to_str(kf) + " f1=" + int_to_str(f1) + " f2=" + int_to_str(f2) + " f3=" + int_to_str(f3) + " grounding=measured src=" + src + " prov=" + prov + let vid: String = engram_node(body, "Voice", 90) + engram_connect(hub, vid, 90, "has-signature") + organ_disclose("VOICE: ingested '" + name + "' into the engram as geometry (node " + vid + ").") + return vid +} + +// Turn the fetched geometry into the voice slot-map the render consumes. Kept +// separate from the fetch so the organ never invents a voice: if the fetch came +// back empty this returns empty too, and the caller has to deal with it. +// +// The slot-map is built here rather than by calling the render's own +// constructor, so the organ carries NO dependency on the language faculty's +// modules — it only has to agree with them about a wire format, which is the +// looser and more honest coupling. (The layout is the same key/value [String] +// convention lang_get / surface_get / voice_get all read.) +fn organ_voice_profile(name: String, g: [Int]) -> [String] { + let r: [String] = native_list_empty() + if native_list_len(g) < 6 { + return r + } + r = native_list_append(r, "name") + r = native_list_append(r, name) + r = native_list_append(r, "f0") + r = native_list_append(r, int_to_str(native_list_get(g, 0))) + r = native_list_append(r, "f0_end") + r = native_list_append(r, int_to_str(native_list_get(g, 1))) + r = native_list_append(r, "kf") + r = native_list_append(r, int_to_str(native_list_get(g, 2))) + r = native_list_append(r, "dur") + r = native_list_append(r, "1000") + r = native_list_append(r, "tilt") + r = native_list_append(r, "1000") + r = native_list_append(r, "breath") + r = native_list_append(r, "8") + return r +} + +// ── Scene geometry (afferent, camera) ──────────────────────────────────────── +// +// The image half of the afferent metabolism, and the same principle as the +// audio descriptor: a frame is never handed on raw. The realizer returns a +// small pixel grid; THIS computes the descriptor, in El, because averaging +// pixels is arithmetic and arithmetic is not a device concern. +// +// Returns 15 numbers — [w, h, meanR, meanG, meanB, brightness_pm, and a 3x3 +// luminance grid] — standing in for a multi-megapixel frame. The 3x3 grid is +// the smallest thing that still says WHERE the light is, which is most of what +// makes a scene comparable to another scene; a single brightness average would +// make a lamp on the left indistinguishable from a lamp on the right. +// +// Luminance is Rec. 601 (0.299R + 0.587G + 0.114B), in integer per-mille, so +// the descriptor is reproducible rather than subject to float drift. +fn organ_image_descriptor() -> [Int] { + let out: [Int] = native_list_empty() + let frame: Any = camera_capture_rgb() + if frame == 0 { + return out + } + let w: Int = el_map_get(frame, "width") + let h: Int = el_map_get(frame, "height") + let gw: Int = el_map_get(frame, "grid_w") + let gh: Int = el_map_get(frame, "grid_h") + let px: [Int] = el_map_get(frame, "pixels") + let np: Int = native_list_len(px) + if np < 3 { + return out + } + let count: Int = np / 3 + let rsum: Int = 0 + let gsum: Int = 0 + let bsum: Int = 0 + // 3x3 accumulators, row-major + let cell: [Int] = native_list_empty() + let cn: [Int] = native_list_empty() + let z: Int = 0 + while z < 9 { + cell = native_list_append(cell, 0) + cn = native_list_append(cn, 0) + z = z + 1 + } + // El has no list-set, so the cells are summed into parallel scalars and + // reassembled — nine explicit accumulators would be worse to read than one + // pass per cell over a grid this small. + let c0: Int = 0 + let c1: Int = 0 + let c2: Int = 0 + let c3: Int = 0 + let c4: Int = 0 + let c5: Int = 0 + let c6: Int = 0 + let c7: Int = 0 + let c8: Int = 0 + let n0: Int = 0 + let n1: Int = 0 + let n2: Int = 0 + let n3: Int = 0 + let n4: Int = 0 + let n5: Int = 0 + let n6: Int = 0 + let n7: Int = 0 + let n8: Int = 0 + let i: Int = 0 + while i < count { + let r: Int = native_list_get(px, i * 3) + let g: Int = native_list_get(px, i * 3 + 1) + let b: Int = native_list_get(px, i * 3 + 2) + rsum = rsum + r + gsum = gsum + g + bsum = bsum + b + let lum: Int = (299 * r + 587 * g + 114 * b) / 1000 + let x: Int = i - (i / gw) * gw + let y: Int = i / gw + let cx: Int = x * 3 / gw + let cy: Int = y * 3 / gh + if cx > 2 { + cx = 2 + } + if cy > 2 { + cy = 2 + } + let idx: Int = cy * 3 + cx + if idx == 0 { + c0 = c0 + lum + n0 = n0 + 1 + } + if idx == 1 { + c1 = c1 + lum + n1 = n1 + 1 + } + if idx == 2 { + c2 = c2 + lum + n2 = n2 + 1 + } + if idx == 3 { + c3 = c3 + lum + n3 = n3 + 1 + } + if idx == 4 { + c4 = c4 + lum + n4 = n4 + 1 + } + if idx == 5 { + c5 = c5 + lum + n5 = n5 + 1 + } + if idx == 6 { + c6 = c6 + lum + n6 = n6 + 1 + } + if idx == 7 { + c7 = c7 + lum + n7 = n7 + 1 + } + if idx == 8 { + c8 = c8 + lum + n8 = n8 + 1 + } + i = i + 1 + } + let rA: Int = rsum / count + let gA: Int = gsum / count + let bA: Int = bsum / count + let bright: Int = (299 * rA + 587 * gA + 114 * bA) / 255 + out = native_list_append(out, w) + out = native_list_append(out, h) + out = native_list_append(out, rA) + out = native_list_append(out, gA) + out = native_list_append(out, bA) + out = native_list_append(out, bright) + if n0 < 1 { + n0 = 1 + } + if n1 < 1 { + n1 = 1 + } + if n2 < 1 { + n2 = 1 + } + if n3 < 1 { + n3 = 1 + } + if n4 < 1 { + n4 = 1 + } + if n5 < 1 { + n5 = 1 + } + if n6 < 1 { + n6 = 1 + } + if n7 < 1 { + n7 = 1 + } + if n8 < 1 { + n8 = 1 + } + out = native_list_append(out, c0 / n0) + out = native_list_append(out, c1 / n1) + out = native_list_append(out, c2 / n2) + out = native_list_append(out, c3 / n3) + out = native_list_append(out, c4 / n4) + out = native_list_append(out, c5 / n5) + out = native_list_append(out, c6 / n6) + out = native_list_append(out, c7 / n7) + out = native_list_append(out, c8 / n8) + organ_disclose("FEAT(image): 15-number scene-geometry vs " + int_to_str(w * h * 3) + " pixel-channels — the descriptor travels, the frame does not.") + return out +} + +// ── Own-core tone ──────────────────────────────────────────────────────────── +// +// The smallest possible proof that the organ owns its medium end to end: a sine +// with a gentle attack and release, computed here, played by us, no file and no +// library anywhere in the path. +fn organ_tone(hz: Int, ms: Int, sr: Int) -> [Int] { + let n: Int = sr * ms / 1000 + let out: [Int] = native_list_empty() + let two_pi: Float = 6.283185307 + let srf: Float = int_to_float(sr) + let hzf: Float = int_to_float(hz) + let i: Int = 0 + // 20 ms of ramp at each end; a square-edged tone clicks, and a click is the + // organ announcing that it does not understand envelopes. + let ramp: Int = sr / 50 + if ramp < 1 { + ramp = 1 + } + while i < n { + let t: Float = int_to_float(i) / srf + let s: Float = math_sin(two_pi * hzf * t) + let env: Int = 32767 + if i < ramp { + env = 32767 * i / ramp + } + let tail: Int = n - i + if tail < ramp { + env = 32767 * tail / ramp + } + let v: Int = float_to_int(s * 9000.0) * env / 32767 + out = native_list_append(out, v) + i = i + 1 + } + return out +} diff --git a/peripheral/src/organ_cli.el b/peripheral/src/organ_cli.el new file mode 100644 index 0000000..35fd7ac --- /dev/null +++ b/peripheral/src/organ_cli.el @@ -0,0 +1,459 @@ +// organ_cli.el — the organ's command surface. main() lives here. +// +// One binary, the same verbs the Swift program had, and nothing behind them +// except El and two thin device realizers. This file is the proof surface: if +// `organ speak` makes a sound and no Swift binary is in the process tree, the +// claim in organ.el's header is true. +// +// Verbs, and what each one demonstrates: +// +// grant/revoke/status the Neuron-level consent lock, inspectable +// speak efferent — audio out of El's own speaker +// tone own-core synthesis: computed in El, played by El, +// never touching the disk +// say fetch a VOICE FROM THE ENGRAM and render through it +// listen afferent — mic capture, consent-gated, fails closed +// see afferent — one camera frame, same two locks +// wav-info WAV geometry, parsed in El +// feat-audio capture -> compact descriptor (8 numbers) +// feat-image frame -> compact scene-geometry +// voiceprint F0 + formants F1-F5 by LPC, in El +// imitate LPC analysis-resynthesis, in El +// hear-imitate the closed loop: hear a voice, take its signature, +// speak back in it +// ingest-audio descriptor -> engram node (the capture becomes geometry) +// ingest-voice voiceprint -> engram voice region (how a voice is learned) +// converse full-duplex interruptible utterance +// +// The descriptors are the point of the afferent half. A capture is NEVER handed +// on raw: a three-second recording is ~48,000 samples and what leaves this +// process is eight numbers. That is both the privacy rail (the stream stays +// local because only its shape travels) and the reason the engram can hold a +// perception at all — geometry is storable, a waveform is not. + +fn cli_usage() -> Bool { + println("organ — Neuron's I/O organ, native El (own-core, local, consent-gated)") + println(" grant|revoke Neuron-level consent") + println(" status consent + device state") + println(" speak play a WAV aloud (efferent)") + println(" tone [hz] [ms] synthesize and play, no file at all") + println(" say [CODE...] fetch voice FROM THE ENGRAM, render, speak") + println(" listen mic capture 16k mono (afferent)") + println(" see one camera frame (afferent)") + println(" wav-info WAV geometry") + println(" feat-audio compact audio descriptor (8 numbers)") + println(" feat-image compact scene-geometry from the camera") + println(" voiceprint F0 + formants F1-F5 (LPC)") + println(" imitate LPC analysis-resynthesis") + println(" hear-imitate mic -> signature -> imitate -> speak aloud") + println(" ingest-audio descriptor -> engram node (geometry)") + println(" ingest-voice voiceprint -> engram voice region") + println(" converse [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume]") + return true +} + +// The engram the organ reads and writes. Its own store, never production's. +fn cli_engram_dir() -> String { + let d: String = env("ORGAN_ENGRAM") + if str_eq(d, "") { + return "peripheral/.engram" + } + return d +} + +fn cli_open_engram() -> Bool { + let dir: String = cli_engram_dir() + fs_mkdir(dir) + let ok: Int = engram_store_boot(dir) + if ok == 1 { + return true + } + return false +} + +// ── formatting helpers ─────────────────────────────────────────────────────── + +fn cli_f(v: Float, dec: Int) -> String { + return format_float(v, dec) +} + +// ── the descriptor, printed and ingested ───────────────────────────────────── +// +// [seconds, sr, ch, rms, peak, zcr, centroid, f0] — the same eight numbers the +// Swift produced, computed in El, and the compression ratio is the headline: +// a few dozen bytes standing in for a few hundred kilobytes. +fn cli_audio_descriptor_text(v: [Float], path: String) -> String { + let secs: Float = native_list_get(v, 0) + let sr: Float = native_list_get(v, 1) + let ch: Float = native_list_get(v, 2) + let rms: Float = native_list_get(v, 3) + let peak: Float = native_list_get(v, 4) + let zcr: Float = native_list_get(v, 5) + let cen: Float = native_list_get(v, 6) + let f0: Float = native_list_get(v, 7) + return "Heard sound (afferent, mic): " + cli_f(secs, 2) + "s at " + cli_f(sr, 0) + "Hz. RMS energy " + cli_f(rms, 4) + ", peak " + cli_f(peak, 4) + ", zero-crossing rate " + cli_f(zcr, 0) + "Hz, spectral centroid " + cli_f(cen, 0) + "Hz, estimated voice pitch F0 " + cli_f(f0, 0) + "Hz. Compact voice/sound signature (8 numbers) — phonetic geometry seed." +} + +fn cli_feat_audio(path: String) -> Bool { + let v: [Float] = dsp_compute_audio(path) + if native_list_len(v) < 8 { + println("{\"ok\": false, \"op\": \"feat-audio\", \"error\": \"cannot read PCM\"}") + return false + } + organ_disclose("FEAT(audio): 8-number signature vs " + int_to_str(float_to_int(native_list_get(v, 0) * native_list_get(v, 1))) + " raw samples.") + println("{\"ok\": true, \"op\": \"feat-audio\", \"file\": \"" + path + "\", \"seconds\": " + cli_f(native_list_get(v, 0), 4) + ", \"sample_rate\": " + cli_f(native_list_get(v, 1), 0) + ", \"channels\": " + cli_f(native_list_get(v, 2), 0) + ", \"rms\": " + cli_f(native_list_get(v, 3), 6) + ", \"peak\": " + cli_f(native_list_get(v, 4), 6) + ", \"zcr_hz\": " + cli_f(native_list_get(v, 5), 4) + ", \"centroid_hz\": " + cli_f(native_list_get(v, 6), 4) + ", \"f0_hz\": " + cli_f(native_list_get(v, 7), 4) + "}") + return true +} + +fn cli_voiceprint(path: String) -> Bool { + let v: [Float] = dsp_voiceprint(path) + if native_list_len(v) < 4 { + println("{\"ok\": false, \"op\": \"voiceprint\", \"error\": \"cannot read speech\"}") + return false + } + let nf: Int = float_to_int(native_list_get(v, 3)) + let fs: String = "" + let bs: String = "" + let i: Int = 0 + while i < nf { + if i > 0 { + fs = fs + ", " + bs = bs + ", " + } + fs = fs + cli_f(native_list_get(v, 4 + i * 2), 3) + bs = bs + cli_f(native_list_get(v, 5 + i * 2), 3) + i = i + 1 + } + println("{\"ok\": true, \"op\": \"voiceprint\", \"file\": \"" + path + "\", \"f0_hz\": " + cli_f(native_list_get(v, 0), 4) + ", \"f0_range\": [" + cli_f(native_list_get(v, 1), 4) + ", " + cli_f(native_list_get(v, 2), 4) + "], \"formants_hz\": [" + fs + "], \"bandwidths_hz\": [" + bs + "]}") + return true +} + +// ── main ───────────────────────────────────────────────────────────────────── + +fn main() { + let a: [String] = args() + let n: Int = native_list_len(a) + if n < 1 { + cli_usage() + return + } + let cmd: String = native_list_get(a, 0) + + // ---- consent ----------------------------------------------------------- + if str_eq(cmd, "grant") { + if n < 2 { + println("grant needs a device") + return + } + organ_grant(native_list_get(a, 1)) + println("{\"ok\": true, \"op\": \"grant\", \"consent\": \"" + organ_consent_status() + "\"}") + return + } + if str_eq(cmd, "revoke") { + if n < 2 { + println("revoke needs a device") + return + } + organ_revoke(native_list_get(a, 1)) + println("{\"ok\": true, \"op\": \"revoke\", \"consent\": \"" + organ_consent_status() + "\"}") + return + } + if str_eq(cmd, "status") { + println("{\"ok\": true, \"op\": \"status\", \"consent\": \"" + organ_consent_status() + "\", \"speaker\": \"" + speaker_name() + "\", \"speaker_available\": " + int_to_str(speaker_available()) + ", \"mic_os_authorized\": " + int_to_str(mic_available()) + ", \"camera_os_authorized\": " + int_to_str(camera_available()) + "}") + return + } + + // ---- efferent ---------------------------------------------------------- + if str_eq(cmd, "speak") { + if n < 2 { + println("speak needs a wav") + return + } + let ok: Bool = organ_speak_wav(native_list_get(a, 1)) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"speak\", \"played_aloud\": " + bool_to_str(ok) + "}") + return + } + if str_eq(cmd, "tone") { + let hz: Int = 220 + let ms: Int = 1000 + if n >= 2 { + hz = str_to_int(native_list_get(a, 1)) + } + if n >= 3 { + ms = str_to_int(native_list_get(a, 2)) + } + let s: [Int] = organ_tone(hz, ms, 16000) + let ok: Bool = organ_speak_samples(s, 16000) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"tone\", \"hz\": " + int_to_str(hz) + ", \"ms\": " + int_to_str(ms) + ", \"samples\": " + int_to_str(native_list_len(s)) + ", \"file\": null}") + return + } + + // ---- the voice, from the engram ---------------------------------------- + if str_eq(cmd, "say") { + if n < 3 { + println("say needs [CODE...]") + return + } + cli_open_engram() + let vname: String = native_list_get(a, 1) + let g: [Int] = organ_voice_fetch(vname) + if native_list_len(g) < 6 { + println("{\"ok\": false, \"op\": \"say\", \"error\": \"no voice region '" + vname + "' in the engram\"}") + return + } + // Codes and the phoneme map come from the LANGUAGE side. The organ does + // not know what a word is and never looks one up. + let pmap: [String] = ingest_phonetics("elp/data/phonetics.psv") + let codes: [String] = native_list_empty() + let i: Int = 2 + while i < n { + codes = native_list_append(codes, native_list_get(a, i)) + i = i + 1 + } + let voice: [String] = organ_voice_profile(vname, g) + let s: [Int] = synth_codes(codes, voice, pmap) + let ok: Bool = organ_speak_samples(s, 16000) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"say\", \"voice\": \"" + vname + "\", \"f0\": " + int_to_str(native_list_get(g, 0)) + ", \"kf\": " + int_to_str(native_list_get(g, 2)) + ", \"codes\": " + int_to_str(native_list_len(codes)) + ", \"samples\": " + int_to_str(native_list_len(s)) + "}") + return + } + + // ---- afferent ---------------------------------------------------------- + if str_eq(cmd, "listen") { + if n < 3 { + println("listen needs ") + return + } + let secs: Int = str_to_int(native_list_get(a, 1)) + let out: String = native_list_get(a, 2) + if organ_may_listen() == false { + println("{\"ok\": false, \"op\": \"listen\", \"error\": \"consent denied (fails closed)\"}") + return + } + organ_disclose("MIC: capturing " + int_to_str(secs) + "s (16 kHz mono, LOCAL, never egresses).") + let s: [Int] = mic_capture_pcm16(secs, 16000) + let got: Int = native_list_len(s) + if got <= 0 { + println("{\"ok\": false, \"op\": \"listen\", \"error\": \"capture returned nothing\"}") + return + } + let ok: Bool = write_wav(s, 16000, out) + organ_disclose("MIC: captured " + int_to_str(got) + " frames — ready to hand to the ingest organ.") + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"listen\", \"file\": \"" + out + "\", \"frames\": " + int_to_str(got) + ", \"sample_rate\": 16000}") + return + } + if str_eq(cmd, "see") { + if n < 2 { + println("see needs an out path") + return + } + if organ_may_see() == false { + println("{\"ok\": false, \"op\": \"see\", \"error\": \"consent denied (fails closed)\"}") + return + } + organ_disclose("CAMERA: capturing one frame (LOCAL, never egresses).") + let ok: Int = camera_capture_jpeg(native_list_get(a, 1)) + println("{\"ok\": " + int_to_str(ok) + ", \"op\": \"see\", \"file\": \"" + native_list_get(a, 1) + "\"}") + return + } + + // ---- descriptors ------------------------------------------------------- + if str_eq(cmd, "wav-info") { + if n < 2 { + println("wav-info needs a wav") + return + } + let p: String = native_list_get(a, 1) + let w: [Float] = dsp_read_wav(p) + if dsp_wav_n(w) <= 0 { + println("{\"ok\": false, \"op\": \"wav-info\"}") + return + } + println("{\"ok\": true, \"op\": \"wav-info\", \"sample_rate\": " + int_to_str(dsp_wav_sr(w)) + ", \"channels\": " + int_to_str(dsp_wav_ch(w)) + ", \"frames\": " + int_to_str(dsp_wav_n(w)) + "}") + return + } + if str_eq(cmd, "feat-audio") { + if n < 2 { + println("feat-audio needs a wav") + return + } + cli_feat_audio(native_list_get(a, 1)) + return + } + if str_eq(cmd, "feat-image") { + if organ_may_see() == false { + println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"consent denied (fails closed)\"}") + return + } + let f: [Int] = organ_image_descriptor() + if native_list_len(f) < 15 { + println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"no frame\"}") + return + } + let grid: String = "" + let i: Int = 6 + while i < 15 { + if i > 6 { + grid = grid + ", " + } + grid = grid + int_to_str(native_list_get(f, i)) + i = i + 1 + } + println("{\"ok\": true, \"op\": \"feat-image\", \"width\": " + int_to_str(native_list_get(f, 0)) + ", \"height\": " + int_to_str(native_list_get(f, 1)) + ", \"mean_rgb\": [" + int_to_str(native_list_get(f, 2)) + ", " + int_to_str(native_list_get(f, 3)) + ", " + int_to_str(native_list_get(f, 4)) + "], \"brightness_pm\": " + int_to_str(native_list_get(f, 5)) + ", \"luma_grid\": [" + grid + "]}") + return + } + if str_eq(cmd, "voiceprint") { + if n < 2 { + println("voiceprint needs a wav") + return + } + cli_voiceprint(native_list_get(a, 1)) + return + } + if str_eq(cmd, "imitate") { + if n < 3 { + println("imitate needs ") + return + } + let s: [Int] = dsp_imitate(native_list_get(a, 1)) + if native_list_len(s) <= 0 { + println("{\"ok\": false, \"op\": \"imitate\"}") + return + } + let ok: Bool = write_wav(s, 16000, native_list_get(a, 2)) + organ_disclose("IMITATE: rebuilt the voice from its own LPC signature (own-core, no training, no stolen voice).") + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"imitate\", \"out\": \"" + native_list_get(a, 2) + "\", \"samples\": " + int_to_str(native_list_len(s)) + ", \"method\": \"LPC analysis-resynthesis\"}") + return + } + if str_eq(cmd, "hear-imitate") { + if n < 3 { + println("hear-imitate needs ") + return + } + let secs: Int = str_to_int(native_list_get(a, 1)) + let out: String = native_list_get(a, 2) + if organ_may_listen() == false { + println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"consent denied (fails closed)\"}") + return + } + let heard: String = out + ".heard.wav" + organ_disclose("HEAR-IMITATE: open the ear, listen " + int_to_str(secs) + "s, take the voice, speak it back.") + let s: [Int] = mic_capture_pcm16(secs, 16000) + if native_list_len(s) <= 0 { + println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"capture returned nothing\"}") + return + } + write_wav(s, 16000, heard) + let re: [Int] = dsp_imitate(heard) + if native_list_len(re) <= 0 { + println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"could not model the voice\"}") + return + } + write_wav(re, 16000, out) + let ok: Bool = organ_speak_samples(re, 16000) + println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"hear-imitate\", \"heard\": \"" + heard + "\", \"out\": \"" + out + "\", \"spoke_aloud\": " + bool_to_str(ok) + "}") + return + } + + // ---- the afferent wire: descriptor -> geometry -------------------------- + if str_eq(cmd, "ingest-audio") { + if n < 2 { + println("ingest-audio needs a wav") + return + } + let p: String = native_list_get(a, 1) + let v: [Float] = dsp_compute_audio(p) + if native_list_len(v) < 8 { + println("{\"ok\": false, \"op\": \"ingest-audio\"}") + return + } + cli_open_engram() + let content: String = cli_audio_descriptor_text(v, p) + let id: String = engram_node(content, "Observation", 70) + engram_store_checkpoint() + organ_disclose("INGEST: the capture is now GEOMETRY in the engram (node " + id + ") — the descriptor travelled, the stream did not.") + println("{\"ok\": true, \"op\": \"ingest-audio\", \"node_id\": \"" + id + "\", \"content\": \"" + content + "\"}") + return + } + if str_eq(cmd, "ingest-voice") { + if n < 3 { + println("ingest-voice needs ") + return + } + let p: String = native_list_get(a, 1) + let name: String = native_list_get(a, 2) + let v: [Float] = dsp_voiceprint(p) + if native_list_len(v) < 10 { + println("{\"ok\": false, \"op\": \"ingest-voice\", \"error\": \"no voiced frames\"}") + return + } + cli_open_engram() + let f0: Int = float_to_int(native_list_get(v, 0)) + let f1: Int = float_to_int(native_list_get(v, 4)) + let f2: Int = float_to_int(native_list_get(v, 6)) + let f3: Int = float_to_int(native_list_get(v, 8)) + // kf is the vocal-tract scale: this speaker's F1 against the nominal + // /AA/ F1 of 730 Hz. One number standing for a tract length. + let kf: Int = 1000 * f1 / 730 + let f0e: Int = f0 * 85 / 100 + let id: String = organ_voice_ingest(name, f0, f0e, kf, f1, f2, f3, "el-organ-lpc-voiceprint", "COARSE") + engram_store_checkpoint() + println("{\"ok\": true, \"op\": \"ingest-voice\", \"node_id\": \"" + id + "\", \"name\": \"" + name + "\", \"f0\": " + int_to_str(f0) + ", \"kf\": " + int_to_str(kf) + ", \"f1\": " + int_to_str(f1) + ", \"f2\": " + int_to_str(f2) + ", \"f3\": " + int_to_str(f3) + "}") + return + } + + // ---- converse ---------------------------------------------------------- + if str_eq(cmd, "converse") { + if n < 2 { + println("converse needs a manifest") + return + } + let mf: String = native_list_get(a, 1) + let authority: Int = 500 + let barge: Int = 0 - 1 + let kind: String = "bargein" + let live: Bool = false + let resume: Bool = false + let i: Int = 2 + while i < n { + let f: String = native_list_get(a, i) + if str_eq(f, "--authority") { + if i + 1 < n { + authority = str_to_int(native_list_get(a, i + 1)) + i = i + 1 + } + } + if str_eq(f, "--barge-at") { + if i + 1 < n { + let spec: String = native_list_get(a, i + 1) + let c: Int = str_index_of(spec, ":") + if c < 0 { + barge = str_to_int(spec) + } else { + barge = str_to_int(str_slice(spec, 0, c)) + kind = str_slice(spec, c + 1, str_len(spec)) + } + i = i + 1 + } + } + if str_eq(f, "--live-mic") { + live = true + } + if str_eq(f, "--resume") { + resume = true + } + i = i + 1 + } + let plan: [String] = conv_load_manifest(mf) + if resume { + plan = conv_load_resume() + organ_disclose("CONVERSE: resuming — \"as I was saying...\" (" + int_to_str(plan_count(plan)) + " segments left).") + } else { + organ_disclose("CONVERSE: utterance = \"" + conv_utterance(mf) + "\" (" + int_to_str(plan_count(plan)) + " segments).") + } + let stopped: Int = conv_run(plan, authority, barge, kind, live) + println("{\"ok\": true, \"op\": \"converse\", \"stopped_at\": " + int_to_str(stopped) + ", \"complete\": " + bool_to_str(stopped < 0) + "}") + return + } + + cli_usage() +} diff --git a/peripheral/src/organ_converse.el b/peripheral/src/organ_converse.el new file mode 100644 index 0000000..058dad9 --- /dev/null +++ b/peripheral/src/organ_converse.el @@ -0,0 +1,454 @@ +// organ_converse.el — full-duplex, interruptible speech. The turn-taking organ. +// +// WHAT THIS IS FOR. A system that plays an utterance to completion and only +// then listens is not conversational, it is a loudspeaker with a queue. Being +// interruptible is not a feature bolted onto speech; it is most of what makes +// speech social. So the utterance is not a blob of audio — it is an ordered, +// SALIENCE-TAGGED MEANING-PLAN, and the organ speaks it while listening, decides +// what to do when interrupted, and can pick the thread back up afterwards. +// +// THREE THINGS HAVE TO BE TRUE, and each one is a place naive implementations +// go wrong: +// +// Barge-in is AT THE SAMPLE. When the mic hears speech, output stops on the +// spot — not at the end of the current buffer, not at the end of the segment. +// A listener experiences even a fifth of a second of continued talking as +// being talked over. This is why the speaker realizer has pause/resume and +// reports played_frames: "finish the buffer" is not barge-in. +// +// Yield-or-hold is a DECISION, not a rule. Stopping every time anyone makes a +// noise is its own failure — it means Neuron can never finish a sentence that +// matters. So the choice is grounded: how salient is what I am mid-saying, +// how close am I to done, and how much authority does the interrupter have. +// Holding the floor is justified when what I am saying matters AND finishing +// is cheap AND the interrupter is not high-priority. Otherwise yield, because +// the polite default is the right default. +// +// A backchannel is NOT an interruption. "mm-hm" means keep going. Treating it +// as a barge-in makes the system stop every three seconds during ordinary +// listening behaviour, which is worse than not listening at all. It is +// distinguished by being brief and low-energy: sample again shortly after +// onset, and if the speech already died away it was a backchannel. +// +// AND THE UTTERANCE SURVIVES. On yield, the remaining plan is persisted, so +// Neuron can resume — "as I was saying" — instead of losing the thought. An +// interruption should cost a turn, not the content. +// +// The AEC rail: the microphone runs with the OS voice-processing unit enabled +// so it does not hear our own speaker. Without it Neuron barges in on its own +// voice on the first syllable and the whole loop is unusable in a real room. +// +// Note what is NOT here: nothing about words. A segment carries a `text` field +// purely as a label for disclosure. The organ speaks pre-rendered audio and +// never inspects language — that is the language faculty's, and the seam holds. + +// ── The meaning-plan ───────────────────────────────────────────────────────── +// +// Stored as a flat [String] with stride 3 — file, salience-per-mille, text — +// because El has no record type and parallel lists drift out of step under +// editing. Salience is an integer per-mille rather than a Float so the decision +// arithmetic stays exact and reproducible; a turn-taking decision that varies +// with floating-point rounding is not one you can debug. + +fn plan_new() -> [String] { + return native_list_empty() +} + +fn plan_add(plan: [String], file: String, salience_pm: Int, text: String) -> [String] { + let p: [String] = plan + p = native_list_append(p, file) + p = native_list_append(p, int_to_str(salience_pm)) + p = native_list_append(p, text) + return p +} + +fn plan_count(plan: [String]) -> Int { + return native_list_len(plan) / 3 +} + +fn plan_file(plan: [String], i: Int) -> String { + return native_list_get(plan, i * 3) +} + +fn plan_salience(plan: [String], i: Int) -> Int { + return str_to_int(native_list_get(plan, i * 3 + 1)) +} + +fn plan_text(plan: [String], i: Int) -> String { + return native_list_get(plan, i * 3 + 2) +} + +// ── Manifest ───────────────────────────────────────────────────────────────── +// +// {"utterance": "...", "segments": [{"file":..., "salience":0.9, "text":"..."}]} +// Salience arrives as a 0..1 float in the manifest and is converted once, here, +// at the edge — the same discipline the runtime uses for wire encodings. + +fn conv_salience_pm(raw: String) -> Int { + // "0.85" -> 850. Parsed by hand rather than through a float so a manifest + // typo degrades to a visible number instead of a silent 0.0. + let dot: Int = str_index_of(raw, ".") + if dot < 0 { + let whole: Int = str_to_int(raw) + return whole * 1000 + } + let ip: Int = str_to_int(str_slice(raw, 0, dot)) + let frac: String = str_slice(raw, dot + 1, str_len(raw)) + let pm: Int = 0 + let scale: Int = 100 + let i: Int = 0 + while i < 3 { + let d: Int = 0 + if i < str_len(frac) { + let c: Int = str_char_code(frac, i) + if c >= 48 { + if c <= 57 { + d = c - 48 + } + } + } + pm = pm + d * scale + scale = scale / 10 + i = i + 1 + } + return ip * 1000 + pm +} + +fn conv_load_manifest(path: String) -> [String] { + let plan: [String] = plan_new() + let raw: String = fs_read(path) + if str_eq(raw, "") { + organ_disclose("CONVERSE: cannot read manifest " + path) + return plan + } + let segs: String = json_get_raw(raw, "segments") + let n: Int = json_array_len(segs) + let i: Int = 0 + while i < n { + let seg: String = json_array_get(segs, i) + let file: String = json_get_string(seg, "file") + let text: String = json_get_string(seg, "text") + let sal: String = json_get_raw(seg, "salience") + let pm: Int = conv_salience_pm(sal) + if pm <= 0 { + pm = 500 + } + plan = plan_add(plan, file, pm, text) + i = i + 1 + } + return plan +} + +fn conv_utterance(path: String) -> String { + let raw: String = fs_read(path) + return json_get_string(raw, "utterance") +} + +// ── The decision ───────────────────────────────────────────────────────────── +// +// Returns: 0 = backchannel, carry on seamlessly +// 1 = hold the floor ("hang on, let me finish this thought") +// 2 = yield (stop, let them in) +// +// All arguments are per-mille integers. Holding requires BOTH that the material +// is worth finishing AND that the interrupter is not high-authority — either +// condition alone is not enough, because "what I'm saying is important" is +// exactly the reasoning that produces a system nobody can get a word in against. +fn conv_decide(salience_pm: Int, progress_pm: Int, authority_pm: Int, is_backchannel: Bool) -> Int { + if is_backchannel { + return 0 + } + let hold_score: Int = (salience_pm * 6 + progress_pm * 4) / 10 + if hold_score >= 600 { + if authority_pm < 800 { + return 1 + } + } + return 2 +} + +// ── Resume ─────────────────────────────────────────────────────────────────── +// +// The remaining plan, written where a later run can find it. This is what turns +// an interruption into a pause rather than a loss. + +fn conv_resume_path() -> String { + let home: String = env("PERIPH_HOME") + if str_eq(home, "") { + return "peripheral/.resume.json" + } + return home + "/.resume.json" +} + +// Minimal JSON string escaping. Written here rather than reached for from the +// runtime because the organ needs exactly two escapes and no dependency: a +// segment label containing a quote or a backslash must not be able to produce a +// resume file that fails to parse and silently loses the thread. +fn conv_escape(s: String) -> String { + let n: Int = str_len(s) + let out: String = "" + let i: Int = 0 + while i < n { + let c: Int = str_char_code(s, i) + if c == 34 { + out = out + "\\\"" + } else { + if c == 92 { + out = out + "\\\\" + } else { + if c >= 32 { + out = out + str_slice(s, i, i + 1) + } + } + } + i = i + 1 + } + return out +} + +fn conv_persist_resume(plan: [String], start_at: Int, reason: String) -> Bool { + let n: Int = plan_count(plan) + let body: String = "{\"resume_from\": " + int_to_str(start_at) + ", \"reason\": \"" + reason + "\", \"segments\": [" + let i: Int = start_at + let first: Bool = true + while i < n { + if first == false { + body = body + ", " + } + body = body + "{\"file\": \"" + plan_file(plan, i) + "\", \"salience\": " + int_to_str(plan_salience(plan, i)) + ", \"text\": \"" + conv_escape(plan_text(plan, i)) + "\"}" + first = false + i = i + 1 + } + body = body + "]}\n" + let ok: Bool = fs_write(conv_resume_path(), body) + organ_disclose("CONVERSE: meaning-plan persisted (" + int_to_str(n - start_at) + " segments remain) — Neuron can resume the thread.") + return ok +} + +fn conv_clear_resume() -> Bool { + return fs_write(conv_resume_path(), "") +} + +// Read a persisted plan back. Salience is already per-mille here (we wrote it), +// so it is NOT re-scaled — the manifest and the resume file are different +// formats on purpose, and conflating them silently divides every salience by a +// thousand. +fn conv_load_resume() -> [String] { + let plan: [String] = plan_new() + let raw: String = fs_read(conv_resume_path()) + if str_eq(raw, "") { + return plan + } + let segs: String = json_get_raw(raw, "segments") + let n: Int = json_array_len(segs) + let i: Int = 0 + while i < n { + let seg: String = json_array_get(segs, i) + plan = plan_add(plan, json_get_string(seg, "file"), json_get_int(seg, "salience"), json_get_string(seg, "text")) + i = i + 1 + } + return plan +} + +// ── The loop ───────────────────────────────────────────────────────────────── +// +// live_mic : open the microphone with AEC and let real speech drive barge-in. +// barge_ms : if >= 0, inject a barge event at that offset into the utterance +// instead. Deterministic, so the decision paths can be exercised +// without a room and a person — the same reason periph.swift has it. +// kind : "backchannel" or "bargein", for the injected case. +// authority : interrupter authority, per-mille. +// +// Returns the index the utterance stopped at, or -1 if it completed. + +fn conv_run(plan: [String], authority_pm: Int, barge_ms: Int, kind: String, live_mic: Bool) -> Int { + let n: Int = plan_count(plan) + if n <= 0 { + organ_disclose("CONVERSE: nothing to say.") + return 0 - 1 + } + if speaker_available() == 0 { + organ_disclose("CONVERSE: no speaker on this build — cannot hold a conversation.") + return 0 - 1 + } + + let mic_live: Bool = false + if live_mic { + if organ_may_listen() { + let m: Int = mic_monitor_start() + if m == 1 { + organ_disclose("CONVERSE: full-duplex — mic listening WHILE speaking, AEC on (won't self-interrupt).") + mic_live = true + } + if m == 2 { + organ_disclose("CONVERSE: full-duplex — mic listening, but AEC UNAVAILABLE; raising the VAD floor so we do not barge in on ourselves.") + mic_live = true + } + if m == 0 { + organ_disclose("CONVERSE: could not open the mic monitor — falling back to injected events.") + } + } + } + if mic_live == false { + organ_disclose("CONVERSE: deterministic mode (live mic off).") + } + + // Without AEC the mic hears the speaker, so the threshold has to sit above + // our own output. This is a mitigation and not a fix: the honest note is + // that barge-in is markedly less sensitive in this mode. + let vad_pm: Int = 20 + if mic_live { + if mic_monitor_start() == 2 { + vad_pm = 60 + } + } + + let elapsed_ms: Int = 0 + let prior_ms: Int = 0 + let handled: Bool = false + // An injected barge is ONE event, not a condition that stays true. Without + // this the deadline re-fires on every poll after a backchannel resume, and + // the utterance live-locks: paused, resumed, paused again, forever. + let injected_fired: Bool = false + let i: Int = 0 + + while i < n { + let file: String = plan_file(plan, i) + let sal: Int = plan_salience(plan, i) + let frames: Int = wav_frames(file) + let rate: Int = wav_rate(file) + if frames <= 0 { + organ_disclose("CONVERSE: missing or unreadable segment '" + file + "', skipping.") + i = i + 1 + } else { + let dur_ms: Int = frames * 1000 / rate + organ_disclose("CONVERSE: speaking segment " + int_to_str(i + 1) + "/" + int_to_str(n) + " (salience " + int_to_str(sal) + "/1000) — \"" + plan_text(plan, i) + "\"") + let started: Int = speaker_play_wav_async(file) + if started == 0 { + organ_disclose("CONVERSE: could not start playback for '" + file + "'.") + i = i + 1 + } else { + let seg_ms: Int = 0 + let done: Bool = false + let interrupted: Bool = false + let speech_ticks: Int = 0 + + while done == false { + sleep_ms(10) + seg_ms = seg_ms + 10 + + if speaker_playing() == 0 { + done = true + } else { + // The tick counter is an approximation — each pass costs + // more than the sleep it asked for. The DAC position is + // the truth, so drive the injected deadline off THAT and + // an injected barge lands where it was asked to land. + let pos_ms: Int = speaker_played_frames() * 1000 / rate + elapsed_ms = prior_ms + pos_ms + // --- onset detection: real speech, or an injected event --- + let onset: Bool = false + if mic_live { + let rms: Float = mic_monitor_rms() + let rms_pm: Int = float_to_int(rms * 1000.0) + if rms_pm > vad_pm { + speech_ticks = speech_ticks + 1 + } else { + speech_ticks = 0 + } + // ~60ms of continuous voice: short enough to feel + // instant, long enough that a door closing is not a turn. + if speech_ticks >= 3 { + if handled == false { + onset = true + } + } + } + if barge_ms >= 0 { + if injected_fired == false { + if elapsed_ms >= barge_ms { + onset = true + injected_fired = true + } + } + } + + if onset { + handled = true + // (1) BARGE-IN — pause on the spot. + speaker_pause() + let played: Int = speaker_played_frames() + let at_ms: Int = played * 1000 / rate + let progress_pm: Int = at_ms * 1000 / dur_ms + if progress_pm > 1000 { + progress_pm = 1000 + } + organ_disclose("CONVERSE: << user speech at " + int_to_str(at_ms) + "ms into segment " + int_to_str(i + 1) + " — PAUSED instantly >>") + + // (2) backchannel or real barge-in? + let is_bc: Bool = false + if barge_ms >= 0 { + if str_eq(kind, "backchannel") { + is_bc = true + } + } else { + // Live: look again ~250ms after onset. If the + // energy has already collapsed it was "mm-hm". + sleep_ms(250) + let r2: Float = mic_monitor_rms() + if float_to_int(r2 * 1000.0) < 15 { + is_bc = true + } + } + + // (3) yield, hold, or carry on + let d: Int = conv_decide(sal, progress_pm, authority_pm, is_bc) + if d == 0 { + organ_disclose("CONVERSE: read as BACKCHANNEL (\"mm-hm\") — keep going, resume seamlessly.") + handled = false + speech_ticks = 0 + speaker_resume() + } + if d == 1 { + organ_disclose("CONVERSE: HOLD the floor — \"hang on, let me finish this thought.\" (salience " + int_to_str(sal) + ", progress " + int_to_str(progress_pm) + ")") + speaker_resume() + // Finish THIS segment, then yield the remainder: + // holding is a request for a moment, not a claim + // on the rest of the conversation. + while speaker_playing() == 1 { + sleep_ms(20) + } + speaker_stop() + conv_persist_resume(plan, i + 1, "held-then-yield") + if mic_live { + mic_monitor_stop() + } + return i + 1 + } + if d == 2 { + organ_disclose("CONVERSE: YIELD — stop, let them in. Remembering where I was (resumable).") + speaker_stop() + conv_persist_resume(plan, i, "yield") + if mic_live { + mic_monitor_stop() + } + return i + } + } + } + } + if interrupted == false { + prior_ms = prior_ms + dur_ms + i = i + 1 + } + } + } + } + + conv_clear_resume() + organ_disclose("CONVERSE: utterance complete (uninterrupted).") + if mic_live { + mic_monitor_stop() + } + return 0 - 1 +} diff --git a/peripheral/src/organ_dsp.el b/peripheral/src/organ_dsp.el new file mode 100644 index 0000000..87068f0 --- /dev/null +++ b/peripheral/src/organ_dsp.el @@ -0,0 +1,1019 @@ +// organ_dsp.el — the AFFERENT DSP organ, own-core, ported from peripheral/src/periph.swift. +// +// The mirror of speech.el: where speech.el RENDERS meaning out through a voice +// (efferent), this module HEARS — it takes a raw 16-bit PCM RIFF/WAVE capture +// and metabolizes it into GEOMETRY: a compact 8-number audio descriptor, a +// voice-signature (F0 + formants F1-F5 by LPC), and an LPC analysis-resynthesis +// that speaks the heard voice back from its own signature. +// +// Everything here is float physics (math_sin/math_cos/math_sqrt), not the +// fixed-point integer path speech.el uses for synthesis — the analysis side +// needs the dynamic range that autocorrelation and Levinson-Durbin demand. +// +// Binary I/O note: an El String truncates at the first NUL, so a WAV can never +// be read through fs_read(). We read it through fs_read_b64_chunk(), which +// hands back plain-ASCII base64 of a byte window, and decode that base64 HERE, +// in El, into a [Int] of byte values. Chunks are a multiple of 3 bytes so each +// base64 window decodes cleanly with no padding except the final one. +// +// Every function is prefixed dsp_ so nothing here can collide with speech.el +// (write_wav / wav_le16 / wav_le32 / sp_* / voice_* all live there and are NOT +// redefined). imitate returns [Int] samples — hand them to speech.el's write_wav. + +// --------------------------------------------------------------------------- +// Base64 -> bytes (own-core; the only way binary reaches El intact) +// --------------------------------------------------------------------------- + +// Standard RFC 4648 alphabet A-Za-z0-9+/ -> 0..63. Padding '=' and any other +// character -> -1 (a sentinel; there is no exception handling in El). +fn dsp_b64_val(c: Int) -> Int { + if c >= 65 { + if c <= 90 { + return c - 65 + } + } + if c >= 97 { + if c <= 122 { + return c - 71 + } + } + if c >= 48 { + if c <= 57 { + return c + 4 + } + } + if c == 43 { + return 62 + } + if c == 47 { + return 63 + } + return 0 - 1 +} + +// Read a whole file as a list of byte values 0..255. Empty list on failure. +fn dsp_read_bytes(path: String) -> [Int] { + let bytes: [Int] = native_list_empty() + let size: Int = fs_size(path) + if size <= 0 { + return bytes + } + let chunk: Int = 60000 // multiple of 3 -> no interior padding + let off: Int = 0 + while off < size { + let s: String = fs_read_b64_chunk(path, off, chunk) + let sl: Int = str_len(s) + if sl < 4 { + return bytes + } + let i: Int = 0 + while i + 3 < sl { + let c0: Int = dsp_b64_val(str_char_code(s, i)) + let c1: Int = dsp_b64_val(str_char_code(s, i + 1)) + let c2: Int = dsp_b64_val(str_char_code(s, i + 2)) + let c3: Int = dsp_b64_val(str_char_code(s, i + 3)) + if c0 < 0 { + return bytes + } + if c1 < 0 { + return bytes + } + let b0: Int = c0 * 4 + c1 / 16 + bytes = native_list_append(bytes, b0) + if c2 >= 0 { + let lo1: Int = c1 - (c1 / 16) * 16 + let b1: Int = lo1 * 16 + c2 / 4 + bytes = native_list_append(bytes, b1) + if c3 >= 0 { + let lo2: Int = c2 - (c2 / 4) * 4 + let b2: Int = lo2 * 64 + c3 + bytes = native_list_append(bytes, b2) + } + } + i = i + 4 + } + off = off + chunk + } + return bytes +} + +fn dsp_rd16(b: [Int], o: Int) -> Int { + let b0: Int = native_list_get(b, o) + let b1: Int = native_list_get(b, o + 1) + return b0 + b1 * 256 +} + +fn dsp_rd32(b: [Int], o: Int) -> Int { + let b0: Int = native_list_get(b, o) + let b1: Int = native_list_get(b, o + 1) + let b2: Int = native_list_get(b, o + 2) + let b3: Int = native_list_get(b, o + 3) + return b0 + b1 * 256 + b2 * 65536 + b3 * 16777216 +} + +// Four bytes at o compared against a 4-char ASCII chunk id. +fn dsp_chunk_is(b: [Int], o: Int, id: String) -> Bool { + let k: Int = 0 + while k < 4 { + let got: Int = native_list_get(b, o + k) + let want: Int = str_char_code(id, k) + if got != want { + return false + } + k = k + 1 + } + return true +} + +// --------------------------------------------------------------------------- +// readWavSamples — 16-bit PCM RIFF/WAVE -> normalized samples. +// Walks chunks to find 'fmt ' and 'data', so JUNK/FLLR padding (which +// AVAudioRecorder emits) is stepped over rather than mistaken for audio. +// Channel 0 only if stereo. +// +// The returned list is PACKED: [ sr, ch, n, s0, s1, ... s(n-1) ] with the three +// header numbers carried as Floats (El has no tuples). Use dsp_wav_sr / +// dsp_wav_ch / dsp_wav_n / dsp_wav_pcm to open it. Empty list on failure. +// --------------------------------------------------------------------------- +fn dsp_read_wav(path: String) -> [Float] { + let out: [Float] = native_list_empty() + let d: [Int] = dsp_read_bytes(path) + let dn: Int = native_list_len(d) + if dn <= 44 { + return out + } + let ch: Int = 0 + let sr: Int = 0 + let bits: Int = 0 + let o: Int = 12 + while o + 8 <= dn { + let sz: Int = dsp_rd32(d, o + 4) + if dsp_chunk_is(d, o, "fmt ") { + if o + 24 <= dn { + ch = dsp_rd16(d, o + 10) + sr = dsp_rd32(d, o + 12) + bits = dsp_rd16(d, o + 22) + } + } + if dsp_chunk_is(d, o, "data") { + if bits != 16 { + return out + } + if ch <= 0 { + return out + } + // Faithful to periph.swift, including its `d.count - 1` bound and + // the `while i + 1 < end` test — the last sample of a file whose + // data chunk runs to EOF is dropped there, so it is dropped here. + let start: Int = o + 8 + let end: Int = start + sz + if end > dn - 1 { + end = dn - 1 + } + let samples: [Float] = native_list_empty() + let step: Int = 2 * ch + let i: Int = start + let count: Int = 0 + while i + 1 < end { + let v: Int = dsp_rd16(d, i) + if v >= 32768 { + v = v - 65536 + } + let f: Float = int_to_float(v) / 32768.0 + samples = native_list_append(samples, f) + count = count + 1 + i = i + step + } + let srf: Float = int_to_float(sr) + let chf: Float = int_to_float(ch) + let nf: Float = int_to_float(count) + out = native_list_append(out, srf) + out = native_list_append(out, chf) + out = native_list_append(out, nf) + let j: Int = 0 + while j < count { + let sv: Float = native_list_get(samples, j) + out = native_list_append(out, sv) + j = j + 1 + } + return out + } + let adv: Int = 8 + sz + (sz - (sz / 2) * 2) + if adv <= 0 { + return out + } + o = o + adv + } + return out +} + +fn dsp_wav_sr(w: [Float]) -> Int { + if native_list_len(w) < 3 { + return 0 + } + let v: Float = native_list_get(w, 0) + return float_to_int(v) +} + +fn dsp_wav_ch(w: [Float]) -> Int { + if native_list_len(w) < 3 { + return 0 + } + let v: Float = native_list_get(w, 1) + return float_to_int(v) +} + +fn dsp_wav_n(w: [Float]) -> Int { + if native_list_len(w) < 3 { + return 0 + } + let v: Float = native_list_get(w, 2) + return float_to_int(v) +} + +// The bare sample list, unpacked from the header-prefixed form. +fn dsp_wav_pcm(w: [Float]) -> [Float] { + let out: [Float] = native_list_empty() + let n: Int = dsp_wav_n(w) + let i: Int = 0 + while i < n { + let v: Float = native_list_get(w, i + 3) + out = native_list_append(out, v) + i = i + 1 + } + return out +} + +// --------------------------------------------------------------------------- +// small float helpers (El has no unary minus on Float in every position, and +// no min/max builtin — so they are written out) +// --------------------------------------------------------------------------- +fn dsp_fabs(v: Float) -> Float { + if v < 0.0 { + return 0.0 - v + } + return v +} + +// --------------------------------------------------------------------------- +// computeAudio — the compact audio descriptor. +// Returns the 8-number vector [seconds, sr, ch, rms, peak, zcr, centroid, f0]. +// Empty list if the WAV cannot be read (no exceptions in El — sentinels only). +// --------------------------------------------------------------------------- +fn dsp_compute_audio(path: String) -> [Float] { + let vec: [Float] = native_list_empty() + let w: [Float] = dsp_read_wav(path) + let n: Int = dsp_wav_n(w) + if n <= 0 { + return vec + } + let sr: Int = dsp_wav_sr(w) + let ch: Int = dsp_wav_ch(w) + let s: [Float] = dsp_wav_pcm(w) + let nf: Float = int_to_float(n) + let srf: Float = int_to_float(sr) + let seconds: Float = nf / srf + + // energy / peak / zero crossings + let sumsq: Float = 0.0 + let peak: Float = 0.0 + let zc: Float = 0.0 + let i: Int = 0 + while i < n { + let v: Float = native_list_get(s, i) + sumsq = sumsq + v * v + let av: Float = dsp_fabs(v) + if av > peak { + peak = av + } + if i > 0 { + let pv: Float = native_list_get(s, i - 1) + let a: Bool = pv < 0.0 + let b: Bool = v < 0.0 + if a != b { + zc = zc + 1.0 + } + } + i = i + 1 + } + let rms: Float = math_sqrt(sumsq / nf) + let zcr: Float = zc / nf * srf // ~2*dominant freq for tonal + + // Spectral centroid via a coarse 64-bin DFT on a centered 2048 window. + let ww: Int = 2048 + if n < ww { + ww = n + } + let off: Int = (n - ww) / 2 + if off < 0 { + off = 0 + } + let two: Float = 2.0 + let pi: Float = math_pi() + let num: Float = 0.0 + let den: Float = 0.0 + let bins: Int = 64 + let binsf: Float = 128.0 // Double(2*bins) + let k: Int = 1 + while k < bins { + let kf: Float = int_to_float(k) + let f: Float = kf * srf / binsf + let re: Float = 0.0 + let im: Float = 0.0 + let j: Int = 0 + while j < ww { + let jf: Float = int_to_float(j) + let ang: Float = (0.0 - two) * pi * kf * jf / binsf + let xv: Float = native_list_get(s, off + j) + let cv: Float = math_cos(ang) + let sv: Float = math_sin(ang) + re = re + xv * cv + im = im + xv * sv + j = j + 1 + } + let mag: Float = math_sqrt(re * re + im * im) + num = num + f * mag + den = den + mag + k = k + 1 + } + let centroid: Float = 0.0 + if den > 0.0 { + centroid = num / den + } + + // F0 by autocorrelation over the plausible speech range 70-400 Hz. + let lag_min: Int = sr / 400 + let lag_max: Int = sr / 70 + if lag_max > n - 1 { + lag_max = n - 1 + } + let bound: Int = off + ww + if bound > n { + bound = n + } + let best_lag: Int = 0 + let best_corr: Float = 0.0 + if lag_max > lag_min { + let lag: Int = lag_min + while lag <= lag_max { + let c: Float = 0.0 + let p: Int = 0 + while p + lag < bound { + let a1: Float = native_list_get(s, off + p) + let a2: Float = native_list_get(s, off + p + lag) + c = c + a1 * a2 + p = p + 1 + } + if c > best_corr { + best_corr = c + best_lag = lag + } + lag = lag + 1 + } + } + let f0: Float = 0.0 + if best_lag > 0 { + f0 = srf / int_to_float(best_lag) + } + + vec = native_list_append(vec, seconds) + vec = native_list_append(vec, srf) + let chf: Float = int_to_float(ch) + vec = native_list_append(vec, chf) + vec = native_list_append(vec, rms) + vec = native_list_append(vec, peak) + vec = native_list_append(vec, zcr) + vec = native_list_append(vec, centroid) + vec = native_list_append(vec, f0) + return vec +} + +// --------------------------------------------------------------------------- +// LPC core: hamming, autocorr, Levinson-Durbin, formant peak-pick, pitch. +// --------------------------------------------------------------------------- + +fn dsp_hamming(x: [Float]) -> [Float] { + let n: Int = native_list_len(x) + if n < 2 { + return x + } + let out: [Float] = native_list_empty() + let pi: Float = math_pi() + let dn: Float = int_to_float(n - 1) + let i: Int = 0 + while i < n { + let v: Float = native_list_get(x, i) + let ang: Float = 2.0 * pi * int_to_float(i) / dn + let cv: Float = math_cos(ang) + let wv: Float = 0.54 - 0.46 * cv + out = native_list_append(out, v * wv) + i = i + 1 + } + return out +} + +// r[lag] = sum_i x[i]*x[i-lag], lag = 0..p. Returns p+1 numbers. +fn dsp_autocorr(x: [Float], p: Int) -> [Float] { + let n: Int = native_list_len(x) + let r: [Float] = native_list_empty() + let lag: Int = 0 + while lag <= p { + let acc: Float = 0.0 + let i: Int = lag + while i < n { + let a: Float = native_list_get(x, i) + let b: Float = native_list_get(x, i - lag) + acc = acc + a * b + i = i + 1 + } + r = native_list_append(r, acc) + lag = lag + 1 + } + return r +} + +// Levinson-Durbin -> LPC coeffs a[0..p] with A(z) = 1 + sum a[k] z^-k. +// Returns p+2 numbers: a[0..p] followed by the residual energy at index p+1. +fn dsp_levinson(r: [Float], p: Int) -> [Float] { + let a: [Float] = native_list_empty() + a = native_list_append(a, 1.0) + let z: Int = 1 + while z <= p { + a = native_list_append(a, 0.0) + z = z + 1 + } + let err: Float = native_list_get(r, 0) + if err <= 0.0 { + a = native_list_append(a, 0.0) + return a + } + let i: Int = 1 + let stopped: Bool = false + while i <= p { + if stopped { + i = p + 1 + } else { + let acc: Float = native_list_get(r, i) + if i > 1 { + let j: Int = 1 + while j < i { + let aj: Float = native_list_get(a, j) + let rij: Float = native_list_get(r, i - j) + acc = acc + aj * rij + j = j + 1 + } + } + let k: Float = (0.0 - acc) / err + // na = a, then na[i] = k, then na[j] = a[j] + k*a[i-j] for 1<=j [Float] { + let peaks: [Float] = native_list_empty() + let p: Int = native_list_len(a) - 1 + if p < 1 { + return peaks + } + let steps: Int = 512 + let stepsf: Float = 512.0 + let srf: Float = int_to_float(sr) + let pi: Float = math_pi() + let mag: [Float] = native_list_empty() + let s: Int = 0 + while s < steps { + let wq: Float = pi * int_to_float(s) / stepsf // 0..pi -> 0..sr/2 + let re: Float = 0.0 + let im: Float = 0.0 + let k: Int = 0 + while k <= p { + let ak: Float = native_list_get(a, k) + let ang: Float = wq * int_to_float(k) + let cv: Float = math_cos(ang) + let sv: Float = math_sin(ang) + re = re + ak * cv + im = im - ak * sv + k = k + 1 + } + let d: Float = math_sqrt(re * re + im * im) + if d < 0.000000001 { + d = 0.000000001 + } + mag = native_list_append(mag, 1.0 / d) + s = s + 1 + } + let found: Int = 0 + let t: Int = 1 + while t < steps - 1 { + if found < 5 { + let m0: Float = native_list_get(mag, t - 1) + let m1: Float = native_list_get(mag, t) + let m2: Float = native_list_get(mag, t + 1) + let rise: Bool = m1 > m0 + let fall: Bool = m1 >= m2 + if rise { + if fall { + let f: Float = int_to_float(t) * srf / 2.0 / stepsf + if f > 150.0 { + if f < 5200.0 { + let thr: Float = m1 / 1.4142 + let lo: Int = t + let scan_lo: Bool = true + while scan_lo { + if lo > 0 { + let mv: Float = native_list_get(mag, lo) + if mv > thr { + lo = lo - 1 + } else { + scan_lo = false + } + } else { + scan_lo = false + } + } + let hi: Int = t + let scan_hi: Bool = true + while scan_hi { + if hi < steps - 1 { + let mv2: Float = native_list_get(mag, hi) + if mv2 > thr { + hi = hi + 1 + } else { + scan_hi = false + } + } else { + scan_hi = false + } + } + let bw: Float = int_to_float(hi - lo) * srf / 2.0 / stepsf + peaks = native_list_append(peaks, f) + peaks = native_list_append(peaks, bw) + found = found + 1 + } + } + } + } + } + t = t + 1 + } + return peaks +} + +// Autocorrelation pitch over 70-400 Hz with the 0.30 voicing threshold. +// Returns 0.0 for an unvoiced (or silent) frame. +fn dsp_pitch_of(frame: [Float], sr: Int) -> Float { + let n: Int = native_list_len(frame) + let lag_min: Int = sr / 400 + let lag_max: Int = sr / 70 + if lag_max > n - 1 { + lag_max = n - 1 + } + if lag_max <= lag_min { + return 0.0 + } + let r0: Float = 0.0 + let i: Int = 0 + while i < n { + let v: Float = native_list_get(frame, i) + r0 = r0 + v * v + i = i + 1 + } + if r0 < 0.00001 { + return 0.0 + } + let best_lag: Int = 0 + let best: Float = 0.0 + let lag: Int = lag_min + while lag <= lag_max { + let c: Float = 0.0 + let j: Int = lag + while j < n { + let a: Float = native_list_get(frame, j) + let b: Float = native_list_get(frame, j - lag) + c = c + a * b + j = j + 1 + } + if c > best { + best = c + best_lag = lag + } + lag = lag + 1 + } + if best_lag > 0 { + let ratio: Float = best / r0 + if ratio > 0.30 { + let srf: Float = int_to_float(sr) + return srf / int_to_float(best_lag) + } + } + return 0.0 +} + +// A copy of x[pos..pos+n) — the El stand-in for Swift's Array(x[a.. [Float] { + let out: [Float] = native_list_empty() + let total: Int = native_list_len(x) + let i: Int = 0 + while i < n { + if pos + i < total { + let v: Float = native_list_get(x, pos + i) + out = native_list_append(out, v) + } + i = i + 1 + } + return out +} + +// median == sorted()[count/2]. There is no list-set in El, so instead of +// sorting we find the value whose rank bracket contains index count/2 — +// numerically identical to the Swift expression, without a mutable buffer. +fn dsp_median(v: [Float]) -> Float { + let n: Int = native_list_len(v) + if n == 0 { + return 0.0 + } + let target: Int = n / 2 + let i: Int = 0 + while i < n { + let x: Float = native_list_get(v, i) + let less: Int = 0 + let eq: Int = 0 + let j: Int = 0 + while j < n { + let y: Float = native_list_get(v, j) + if y < x { + less = less + 1 + } + if y == x { + eq = eq + 1 + } + j = j + 1 + } + if less <= target { + if target < less + eq { + return x + } + } + i = i + 1 + } + return 0.0 +} + +fn dsp_min_of(v: [Float]) -> Float { + let n: Int = native_list_len(v) + if n == 0 { + return 0.0 + } + let m: Float = native_list_get(v, 0) + let i: Int = 1 + while i < n { + let x: Float = native_list_get(v, i) + if x < m { + m = x + } + i = i + 1 + } + return m +} + +fn dsp_max_of(v: [Float]) -> Float { + let n: Int = native_list_len(v) + if n == 0 { + return 0.0 + } + let m: Float = native_list_get(v, 0) + let i: Int = 1 + while i < n { + let x: Float = native_list_get(v, i) + if x > m { + m = x + } + i = i + 1 + } + return m +} + +// --------------------------------------------------------------------------- +// voiceprint — the voice-signature: median F0 over voiced frames + the median +// of each formant F1..F5 (and its bandwidth), plus the F0 min/max range. +// FRAME = 400 (25 ms @ 16k), HOP = 160 (10 ms), LPC order 16. +// +// Returns PACKED: [ f0med, f0lo, f0hi, nf, f1, b1, f2, b2, ... ] where nf is +// how many formants were recovered (0..5). Empty list if unreadable. +// --------------------------------------------------------------------------- +fn dsp_voiceprint(path: String) -> [Float] { + let out: [Float] = native_list_empty() + let w: [Float] = dsp_read_wav(path) + let n: Int = dsp_wav_n(w) + let frame_len: Int = 400 + let hop: Int = 160 + let order: Int = 16 + if n <= frame_len { + return out + } + let sr: Int = dsp_wav_sr(w) + let x: [Float] = dsp_wav_pcm(w) + + let f0s: [Float] = native_list_empty() + // five formant banks + five bandwidth banks, flat lists each + let f1s: [Float] = native_list_empty() + let f2s: [Float] = native_list_empty() + let f3s: [Float] = native_list_empty() + let f4s: [Float] = native_list_empty() + let f5s: [Float] = native_list_empty() + let b1s: [Float] = native_list_empty() + let b2s: [Float] = native_list_empty() + let b3s: [Float] = native_list_empty() + let b4s: [Float] = native_list_empty() + let b5s: [Float] = native_list_empty() + + let pos: Int = 0 + while pos + frame_len <= n { + let raw: [Float] = dsp_slice(x, pos, frame_len) + let f0: Float = dsp_pitch_of(raw, sr) + if f0 > 0.0 { + f0s = native_list_append(f0s, f0) + let win: [Float] = dsp_hamming(raw) + let r: [Float] = dsp_autocorr(win, order) + let r0: Float = native_list_get(r, 0) + if r0 > 0.000001 { + let al: [Float] = dsp_levinson(r, order) + // strip the trailing residual energy: coeffs are a[0..order] + let a: [Float] = native_list_empty() + let ci: Int = 0 + while ci <= order { + let av: Float = native_list_get(al, ci) + a = native_list_append(a, av) + ci = ci + 1 + } + let fs: [Float] = dsp_formants(a, sr) + let nfs: Int = native_list_len(fs) / 2 + if nfs > 0 { + let fv: Float = native_list_get(fs, 0) + let bv: Float = native_list_get(fs, 1) + f1s = native_list_append(f1s, fv) + b1s = native_list_append(b1s, bv) + } + if nfs > 1 { + let fv2: Float = native_list_get(fs, 2) + let bv2: Float = native_list_get(fs, 3) + f2s = native_list_append(f2s, fv2) + b2s = native_list_append(b2s, bv2) + } + if nfs > 2 { + let fv3: Float = native_list_get(fs, 4) + let bv3: Float = native_list_get(fs, 5) + f3s = native_list_append(f3s, fv3) + b3s = native_list_append(b3s, bv3) + } + if nfs > 3 { + let fv4: Float = native_list_get(fs, 6) + let bv4: Float = native_list_get(fs, 7) + f4s = native_list_append(f4s, fv4) + b4s = native_list_append(b4s, bv4) + } + if nfs > 4 { + let fv5: Float = native_list_get(fs, 8) + let bv5: Float = native_list_get(fs, 9) + f5s = native_list_append(f5s, fv5) + b5s = native_list_append(b5s, bv5) + } + } + } + pos = pos + hop + } + + let f0med: Float = dsp_median(f0s) + let f0lo: Float = dsp_min_of(f0s) + let f0hi: Float = dsp_max_of(f0s) + out = native_list_append(out, f0med) + out = native_list_append(out, f0lo) + out = native_list_append(out, f0hi) + + let nf: Int = 0 + if native_list_len(f1s) > 0 { + nf = nf + 1 + } + if native_list_len(f2s) > 0 { + nf = nf + 1 + } + if native_list_len(f3s) > 0 { + nf = nf + 1 + } + if native_list_len(f4s) > 0 { + nf = nf + 1 + } + if native_list_len(f5s) > 0 { + nf = nf + 1 + } + let nff: Float = int_to_float(nf) + out = native_list_append(out, nff) + if native_list_len(f1s) > 0 { + let m: Float = dsp_median(f1s) + let b: Float = dsp_median(b1s) + out = native_list_append(out, m) + out = native_list_append(out, b) + } + if native_list_len(f2s) > 0 { + let m2: Float = dsp_median(f2s) + let bb2: Float = dsp_median(b2s) + out = native_list_append(out, m2) + out = native_list_append(out, bb2) + } + if native_list_len(f3s) > 0 { + let m3: Float = dsp_median(f3s) + let bb3: Float = dsp_median(b3s) + out = native_list_append(out, m3) + out = native_list_append(out, bb3) + } + if native_list_len(f4s) > 0 { + let m4: Float = dsp_median(f4s) + let bb4: Float = dsp_median(b4s) + out = native_list_append(out, m4) + out = native_list_append(out, bb4) + } + if native_list_len(f5s) > 0 { + let m5: Float = dsp_median(f5s) + let bb5: Float = dsp_median(b5s) + out = native_list_append(out, m5) + out = native_list_append(out, bb5) + } + return out +} + +// --------------------------------------------------------------------------- +// imitate — LPC analysis-resynthesis. Per frame: autocorr + Levinson, gain = +// sqrt(residual energy), excitation = an energy-normalized glottal impulse +// train at F0 for voiced frames or white noise for unvoiced, run through the +// all-pole filter using the past-output state. The whole output is normalized +// to peak 0.9 and returned as int16 samples — hand them to speech.el's +// write_wav(samples, sr, path). +// --------------------------------------------------------------------------- +fn dsp_imitate(path: String) -> [Int] { + let res: [Int] = native_list_empty() + let w: [Float] = dsp_read_wav(path) + let n: Int = dsp_wav_n(w) + let frame_len: Int = 400 + let hop: Int = 160 + let order: Int = 16 + if n <= frame_len { + return res + } + let sr: Int = dsp_wav_sr(w) + let srf: Float = int_to_float(sr) + let x: [Float] = dsp_wav_pcm(w) + + let out: [Float] = native_list_empty() + // past outputs, order deep + let state: [Float] = native_list_empty() + let si: Int = 0 + while si < order { + state = native_list_append(state, 0.0) + si = si + 1 + } + let phase: Float = 0.0 + let last_f0: Float = 0.0 + let nstate: Int = 22695 // LCG for the unvoiced source + let written: Int = 0 + + let pos: Int = 0 + while pos + frame_len <= n { + let raw: [Float] = dsp_slice(x, pos, frame_len) + let win: [Float] = dsp_hamming(raw) + let r: [Float] = dsp_autocorr(win, order) + let f0: Float = dsp_pitch_of(raw, sr) + let r0: Float = native_list_get(r, 0) + if r0 < 0.0000001 { + // frame skipped in periph.swift -> those output samples stay zero + let z: Int = 0 + while z < hop { + if written < n { + out = native_list_append(out, 0.0) + written = written + 1 + } + z = z + 1 + } + } else { + let al: [Float] = dsp_levinson(r, order) + let errv: Float = native_list_get(al, order + 1) + let ge: Float = errv + if ge < 0.0 { + ge = 0.0 + } + let gain: Float = math_sqrt(ge) + let use_f0: Float = f0 + if f0 <= 0.0 { + use_f0 = 0.0 + if last_f0 > 0.0 { + use_f0 = last_f0 + } + } + last_f0 = f0 + let i: Int = 0 + while i < hop { + if written < n { + let e: Float = 0.0 + if use_f0 > 0.0 { + phase = phase + use_f0 / srf + if phase >= 1.0 { + phase = phase - 1.0 + e = math_sqrt(srf / use_f0) + } + } else { + nstate = nstate * 1103515245 + 12345 + nstate = nstate - (nstate / 2147483648) * 2147483648 + if nstate < 0 { + nstate = 0 - nstate + } + let u: Float = int_to_float(nstate) / 2147483648.0 + e = u * 2.0 - 1.0 + } + let y: Float = gain * e + let k: Int = 1 + while k <= order { + let ak: Float = native_list_get(al, k) + let sk: Float = native_list_get(state, k - 1) + y = y - ak * sk + k = k + 1 + } + let ns: [Float] = native_list_empty() + ns = native_list_append(ns, y) + let m: Int = 0 + while m < order - 1 { + let sv: Float = native_list_get(state, m) + ns = native_list_append(ns, sv) + m = m + 1 + } + state = ns + out = native_list_append(out, y) + written = written + 1 + } + i = i + 1 + } + } + pos = pos + hop + } + // the tail past the last full frame stays silent, exactly as in periph.swift + while written < n { + out = native_list_append(out, 0.0) + written = written + 1 + } + + // normalize to peak 0.9, then to int16 + let peak: Float = 0.0 + let q: Int = 0 + while q < n { + let v: Float = native_list_get(out, q) + let av: Float = dsp_fabs(v) + if av > peak { + peak = av + } + q = q + 1 + } + let scale: Float = 1.0 + if peak > 0.000000001 { + scale = 0.9 / peak + } + let t: Int = 0 + while t < n { + let v2: Float = native_list_get(out, t) + let sv2: Float = v2 * scale * 32767.0 + if sv2 > 32767.0 { + sv2 = 32767.0 + } + if sv2 < 0.0 - 32767.0 { + sv2 = 0.0 - 32767.0 + } + let iv: Int = float_to_int(sv2) + res = native_list_append(res, iv) + t = t + 1 + } + return res +}