diff --git a/lang/runtime/el_audio_darwin.m b/lang/runtime/el_audio_darwin.m new file mode 100644 index 0000000..ff73964 --- /dev/null +++ b/lang/runtime/el_audio_darwin.m @@ -0,0 +1,381 @@ +/* el_audio_darwin.m — the SPEAKER realizer. El's native audio output on Darwin. + * + * WHY THIS FILE EXISTS. + * + * Neuron could already turn meaning into samples — the render path in + * elp/src/speech.el superposes formant resonances over a glottal source and + * produces PCM. What it could not do was make a sound. Every path from those + * samples to the air ran outside the language: a 939-line Swift program + * (peripheral/src/periph.swift) that shelled out to /usr/bin/afplay. So the + * voice was not a capability of El or of Neuron. It was a separate binary + * standing next to them, and "speak" meant "ask that binary to speak." + * + * A speaker is not a language feature the way a string is, but it is exactly + * the kind of thing a runtime owns: a device. El already owns the filesystem, + * the network, the clock, and a graph. It should own the one output device that + * makes it audible. After this file, `speak` is an El operation. + * + * WHY IT IS A REALIZER AND NOT PURE EL. + * + * This is the boundary the whole design turns on. Everything ABOVE the sample + * buffer is arithmetic and belongs in El: formant geometry, superposition, + * envelopes, WAV framing, the voice signature. Everything in this file is the + * part that cannot be arithmetic — handing a buffer to CoreAudio and waiting + * for the hardware to drain it. There is no way to express "the DAC has now + * played these samples" in El, and there should not be. So the split is: El + * computes the sound, the realizer emits it, and the realizer is as thin as it + * can possibly be — it makes no decisions about content, it has no opinion + * about audio, and it cannot synthesize anything. + * + * The precedent is eg_cosine_batch_strategy_metal_hand.m: a platform-bound + * capability compiled as its OWN translation unit, declared in el_runtime.h, + * and linked in where the platform supports it. Deliberately NOT a patch to + * el_runtime.c — adding a device to El must not mean editing the core runtime, + * for the same reason adding a modality must not (see el_runtime.c's realizer + * registry: a realizer is resolved by name, so new organs never touch the + * middle of the language). el_audio_null.c is the same two entry points for + * every platform that is not Darwin, so El code that speaks still links + * everywhere and simply reports that it has no speaker. + * + * WHY AudioQueue AND NOT afplay. + * + * afplay is a process. Using it means the sound Neuron makes is a file it wrote + * and asked something else to open — which forces every utterance through the + * disk, cannot start until the whole utterance exists, and puts a fork/exec + * between the intent to speak and the sound. AudioQueue takes the samples + * directly out of memory. Nothing is written, nothing is spawned, and a caller + * that wants to stream can push buffers as it renders them. + * + * AudioToolbox ships with macOS, so this stays own-core: no cloud, no library + * to install, no model. The output is the local speaker and nothing leaves the + * machine — there is no network path in this file at all, by construction. + */ + +#import +#import +#include +#include +#include +#include +#include "el_runtime.h" + +/* Three buffers is the standard AudioQueue depth: one being played by the + * hardware, one queued behind it, one being refilled. Fewer risks a gap on a + * busy machine; more only adds latency before the first sound. */ +#define EL_AQ_NBUF 3 +#define EL_AQ_FRAMES 8192 + +typedef struct { + const int16_t* pcm; + int64_t frames; + int64_t pos; + volatile int inflight; /* buffers CoreAudio still owns */ + volatile int drained; /* set once the last buffer has been played */ +} ElAqState; + +/* Called on an AudioQueue-internal thread each time a buffer finishes playing. + * Refills and re-enqueues while samples remain; when the source is exhausted it + * lets the buffer die and counts it out. `drained` flips only when the queue is + * holding nothing, which is what makes the play call synchronous without + * clipping the tail — the same reason periph.swift used .dataPlayedBack rather + * than treating "consumed" as "heard". */ +static void el_aq_callback(void* userData, AudioQueueRef q, AudioQueueBufferRef buf) { + ElAqState* st = (ElAqState*)userData; + int64_t remain = st->frames - st->pos; + if (remain <= 0) { + if (--st->inflight <= 0) st->drained = 1; + return; + } + int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(buf->mAudioData, st->pcm + st->pos, (size_t)n * sizeof(int16_t)); + buf->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); + st->pos += n; + if (AudioQueueEnqueueBuffer(q, buf, 0, NULL) != noErr) { + if (--st->inflight <= 0) st->drained = 1; + } +} + +/* Play a 16-bit mono PCM buffer out the default output device, blocking until + * the hardware has actually finished. Returns 1 on success, 0 on any failure — + * never throws, never hangs indefinitely. */ +static int el_audio_play_raw(const int16_t* pcm, int64_t frames, int32_t sample_rate) { + if (!pcm || frames <= 0 || sample_rate <= 0) return 0; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sample_rate; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + ElAqState st; + memset(&st, 0, sizeof(st)); + st.pcm = pcm; + st.frames = frames; + + AudioQueueRef q = NULL; + /* NULL run loop => callbacks arrive on an AudioQueue-internal thread, so + * this function can simply wait rather than having to pump a run loop it + * does not own. El programs are not required to have one. */ + if (AudioQueueNewOutput(&fmt, el_aq_callback, &st, NULL, NULL, 0, &q) != noErr || !q) { + return 0; + } + + AudioQueueBufferRef bufs[EL_AQ_NBUF]; + int prepared = 0; + for (int i = 0; i < EL_AQ_NBUF; i++) { + if (AudioQueueAllocateBuffer(q, EL_AQ_FRAMES * sizeof(int16_t), &bufs[i]) != noErr) break; + prepared++; + } + if (prepared == 0) { AudioQueueDispose(q, true); return 0; } + + /* Prime: fill what we can before starting, so playback begins immediately + * rather than after the first underrun. */ + for (int i = 0; i < prepared; i++) { + int64_t remain = st.frames - st.pos; + if (remain <= 0) break; + int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(bufs[i]->mAudioData, st.pcm + st.pos, (size_t)n * sizeof(int16_t)); + bufs[i]->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); + st.pos += n; + if (AudioQueueEnqueueBuffer(q, bufs[i], 0, NULL) != noErr) break; + st.inflight++; + } + if (st.inflight == 0) { AudioQueueDispose(q, true); return 0; } + + if (AudioQueueStart(q, NULL) != noErr) { AudioQueueDispose(q, true); return 0; } + + /* Bound the wait by the material's own duration plus a margin. A speaker + * that wedges a program is worse than a speaker that gives up. */ + double seconds = (double)frames / (double)sample_rate; + int64_t max_us = (int64_t)((seconds + 5.0) * 1000000.0); + int64_t waited = 0; + const int64_t tick = 5000; /* 5 ms */ + while (!st.drained && waited < max_us) { + usleep((useconds_t)tick); + waited += tick; + } + + AudioQueueStop(q, true); + AudioQueueDispose(q, true); + return st.drained ? 1 : 0; +} + +/* ── El entry points ──────────────────────────────────────────────────────── + * Declared in el_runtime.h; see there for the El-facing contract. */ + +/* 1 when this build has a real speaker behind it. El code should ask before + * speaking so the no-speaker case is a reported condition, not a silence that + * looks like success. */ +el_val_t speaker_available(void) { + return (el_val_t)1; +} + +el_val_t speaker_name(void) { + return EL_STR("coreaudio-audioqueue"); +} + +/* Play an El [Int] of 16-bit samples. Values are clamped, not wrapped: a + * render that overshoots should distort at the rails the way real clipping + * does, rather than invert phase and produce a sound nothing in the signal + * chain intended. */ +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { + int64_t n = (int64_t)el_list_len(samples); + int32_t sr = (int32_t)sample_rate; + if (n <= 0 || sr <= 0) return (el_val_t)0; + + int16_t* pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); + if (!pcm) return (el_val_t)0; + + for (int64_t i = 0; i < n; i++) { + int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); + if (v > 32767) v = 32767; + if (v < -32768) v = -32768; + pcm[i] = (int16_t)v; + } + + int ok = el_audio_play_raw(pcm, n, sr); + free(pcm); + return (el_val_t)(ok ? 1 : 0); +} + +/* ── Asynchronous playback ─────────────────────────────────────────────────── + * + * converse needs this and a blocking play cannot give it. Barge-in means + * stopping ON THE SPOT when the user starts talking — not at the end of the + * current buffer, and certainly not at the end of the utterance. So the async + * path keeps one queue alive, reports how far the hardware actually got, and + * can be halted mid-buffer. + * + * `played_frames` is what makes an interrupted utterance resumable at the + * sample rather than at the segment: it is the position the DAC reached, not + * the position we enqueued to, and those differ by up to the full queue depth. + * + * One utterance at a time. A second async play stops the first — a mouth that + * can say two things at once is not a feature. */ + +static AudioQueueRef g_aq = NULL; +static ElAqState* g_aq_state = NULL; +static int16_t* g_aq_pcm = NULL; +static int32_t g_aq_sr = 0; + +static void el_audio_teardown(void) { + if (g_aq) { + AudioQueueStop(g_aq, true); + AudioQueueDispose(g_aq, true); + g_aq = NULL; + } + free(g_aq_pcm); g_aq_pcm = NULL; + free(g_aq_state); g_aq_state = NULL; + g_aq_sr = 0; +} + +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { + el_audio_teardown(); + + int64_t n = (int64_t)el_list_len(samples); + int32_t sr = (int32_t)sample_rate; + if (n <= 0 || sr <= 0) return (el_val_t)0; + + g_aq_pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); + if (!g_aq_pcm) return (el_val_t)0; + for (int64_t i = 0; i < n; i++) { + int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); + if (v > 32767) v = 32767; + if (v < -32768) v = -32768; + g_aq_pcm[i] = (int16_t)v; + } + + g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); + if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } + g_aq_state->pcm = g_aq_pcm; + g_aq_state->frames = n; + g_aq_sr = sr; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sr; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { + el_audio_teardown(); + return (el_val_t)0; + } + + for (int i = 0; i < EL_AQ_NBUF; i++) { + int64_t remain = g_aq_state->frames - g_aq_state->pos; + if (remain <= 0) break; + AudioQueueBufferRef b = NULL; + if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; + int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); + b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); + g_aq_state->pos += k; + if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; + g_aq_state->inflight++; + } + if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } + + if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } + return (el_val_t)1; +} + +el_val_t speaker_playing(void) { + if (!g_aq || !g_aq_state) return (el_val_t)0; + return (el_val_t)(g_aq_state->drained ? 0 : 1); +} + +/* Frames the DAC has actually rendered. AudioQueueGetCurrentTime's mSampleTime + * is relative to queue start, which is exactly the "where was I really" figure + * a resumable utterance needs. Falls back to the enqueued position if the + * timeline is unavailable (it is, briefly, right after start). */ +el_val_t speaker_played_frames(void) { + if (!g_aq || !g_aq_state) return (el_val_t)0; + AudioTimeStamp ts; + memset(&ts, 0, sizeof(ts)); + Boolean discontinuity = false; + if (AudioQueueGetCurrentTime(g_aq, NULL, &ts, &discontinuity) == noErr && + (ts.mFlags & kAudioTimeStampSampleTimeValid)) { + int64_t played = (int64_t)ts.mSampleTime; + if (played < 0) played = 0; + if (played > g_aq_state->frames) played = g_aq_state->frames; + return (el_val_t)played; + } + return (el_val_t)g_aq_state->pos; +} + +el_val_t speaker_stop(void) { + if (!g_aq) return (el_val_t)0; + /* immediate: do NOT let the queue finish what it is holding */ + AudioQueueStop(g_aq, true); + el_audio_teardown(); + return (el_val_t)1; +} + +/* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows + * how to write a WAV and a caller may reasonably want to hear one back without + * re-rendering it; the parse is deliberately minimal and chunk-walking, so the + * JUNK/FLLR padding that recorders emit does not defeat it. */ +el_val_t speaker_play_wav(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p) return (el_val_t)0; + FILE* f = fopen(p, "rb"); + if (!f) return (el_val_t)0; + + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return (el_val_t)0; } + long size = ftell(f); + if (size <= 44) { fclose(f); return (el_val_t)0; } + rewind(f); + + unsigned char* d = (unsigned char*)malloc((size_t)size); + if (!d) { fclose(f); return (el_val_t)0; } + size_t got = fread(d, 1, (size_t)size, f); + fclose(f); + if (got != (size_t)size) { free(d); return (el_val_t)0; } + + if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return (el_val_t)0; } + + int32_t sr = 0, channels = 0, bits = 0; + long dataOff = -1, dataLen = 0; + long o = 12; + while (o + 8 <= size) { + long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); + if (sz < 0) break; + if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { + channels = (int32_t)(d[o+10] | (d[o+11] << 8)); + sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); + bits = (int32_t)(d[o+22] | (d[o+23] << 8)); + } else if (memcmp(d + o, "data", 4) == 0) { + dataOff = o + 8; + dataLen = sz; + if (dataOff + dataLen > size) dataLen = size - dataOff; + } + o += 8 + sz + (sz & 1); + } + if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return (el_val_t)0; } + + long frames = dataLen / (2 * channels); + int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); + if (!pcm) { free(d); return (el_val_t)0; } + /* Take channel 0; the organ is mono by design and downmixing would be an + * opinion about content this layer is not entitled to have. */ + for (long i = 0; i < frames; i++) { + long b = dataOff + i * 2 * channels; + pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); + } + free(d); + + int ok = el_audio_play_raw(pcm, frames, sr); + free(pcm); + return (el_val_t)(ok ? 1 : 0); +} diff --git a/lang/runtime/el_capture_darwin.m b/lang/runtime/el_capture_darwin.m new file mode 100644 index 0000000..3651408 --- /dev/null +++ b/lang/runtime/el_capture_darwin.m @@ -0,0 +1,841 @@ +/* el_capture_darwin.m — the MICROPHONE and CAMERA realizers. El's afferent + * organ on Darwin: the two entry points through which the world gets in. + * + * WHY THIS FILE EXISTS, AND WHY IT IS A REALIZER RATHER THAN PURE EL. + * + * el_audio_darwin.m argued the efferent half of this: El can compute a sound + * but it cannot make one, because "the DAC has now played these samples" is not + * a fact any amount of arithmetic can produce. This file is the same argument + * run backwards. El can compute *about* a sound — it can window it, take its + * autocorrelation, run Levinson-Durbin over that, find the formant peaks in the + * resulting all-pole envelope, and hand back a voiceprint — but it cannot ASK. + * There is no expression in El, and there must not be, whose value is "the next + * 1024 frames the microphone hears" or "what the camera is pointed at right + * now." Those are not computed; they are *requested*, from an operating system + * that owns the device, mediates consent for it, and delivers the answer on a + * thread of its choosing whenever it feels like it. Asking is the one primitive + * operation here. Everything else in this file is bookkeeping around the ask. + * + * So the line is drawn exactly where el_audio_darwin.m drew it, at the sample + * buffer, and it is drawn on purpose: + * + * BELOW the line (here): open the device, honour the OS permission gate, + * install a tap or a frame delegate, convert whatever the hardware happens to + * emit into the one shape El asked for, and hand it up. No opinions about + * content. No analysis. No decisions. + * + * ABOVE the line (El): energy, zero-crossing rate, spectral centroid, F0 by + * autocorrelation, LPC, formants F1-F5, the compact descriptors, the + * scene-geometry grid, the yield-or-hold turn-taking decision. All of it is + * arithmetic over a buffer, all of it belongs in El, and none of it appears + * below. The reference this file ports — peripheral/src/periph.swift — held + * both halves, and that was the problem worth fixing: the descriptors were + * trapped in a 939-line binary standing next to the language instead of being + * written in it. Porting the *whole* of periph.swift down here would have + * reproduced that mistake in C. Only the ask came down. + * + * The precedent for the file's SHAPE is eg_cosine_batch_strategy_metal_hand.m: + * a platform-bound capability compiled as its own translation unit, declared in + * el_runtime.h, linked in where the platform supports it, and deliberately NOT + * a patch to the middle of el_runtime.c. Acquiring a device must not mean + * editing the language, for the same reason acquiring a modality must not (see + * the realizer registry: organs are resolved by name). el_peripheral_null.c is + * the same entry points everywhere else, so El code that listens still links on + * every platform and merely reports having no ear. + * + * FAIL CLOSED, ALWAYS. + * + * A capture path that returns plausible-looking zeros when it was denied is + * worse than one that returns nothing, because the caller cannot tell the + * difference between a silent room and a refused microphone. Every entry point + * here checks AVCaptureDevice's authorization status BEFORE touching hardware + * and returns the empty value — an empty list, a 0 map — on anything short of + * .authorized. mic_available() and camera_available() report that state WITHOUT + * prompting, so El can ask "may I?" without the act of asking being a prompt. + * + * NEVER HANG. + * + * Every wait in this file is bounded: 30s on a permission prompt (the user has + * to walk to the dialog), seconds+5 on a capture of `seconds`, 10s on a camera + * frame. An organ that wedges the program holding it is not an organ, it is a + * fault. Every path also tears the device down on the way out, including the + * failure paths, so a timed-out capture does not leave the mic light on. + * + * OWN-CORE AND LOCAL BY CONSTRUCTION. + * + * AVFoundation, CoreVideo, CoreGraphics and ImageIO ship with macOS. There is + * no third-party library here, no model, and — the part that matters — no + * network path of any kind. Samples and pixels move from local hardware into an + * El value and stop. periph.swift had a URLSession in it; this file has no + * socket, no URL, and nothing that could grow one without being obvious in + * review. Consent is enforced above this layer in El and below it by the OS; + * this layer's whole contribution to that is refusing to proceed. + * + * DISCLOSURE. + * + * Every actual device touch writes one line to stderr and flushes it, before + * the device opens. stderr and not stdout: a program that announces "I am about + * to open the microphone" on stdout has corrupted its own output, and the + * caller must be able to separate the answer from how it was obtained. One line + * per touch, no more — a disclosure rail that spams is a rail people learn to + * ignore. + */ + +#if defined(__APPLE__) + +#import +#import +#import +#import +#import +#import + +#include +#include +#include +#include +#include +#include + +#include "el_runtime.h" + +/* ── Disclosure ────────────────────────────────────────────────────────────── + * One flushed line per real device touch, on stderr. Flushed rather than + * buffered so the line reaches the terminal BEFORE the mic light comes on + * rather than whenever the buffer happens to drain. */ +static void el_cap_disclose(const char* what) { + fprintf(stderr, " [peripheral] %s\n", what); + fflush(stderr); +} + +/* ── Permission ────────────────────────────────────────────────────────────── + * authorizationStatus is a pure read of the TCC database: it never prompts and + * never blocks, which is what lets mic_available()/camera_available() answer + * honestly without the question itself becoming an event. requestAccess DOES + * prompt, so it lives behind its own entry point and nothing calls it + * implicitly. */ + +static int el_cap_authorized(AVMediaType media) { + @try { + return [AVCaptureDevice authorizationStatusForMediaType:media] + == AVAuthorizationStatusAuthorized ? 1 : 0; + } @catch (NSException* e) { + (void)e; + return 0; + } +} + +static int el_cap_device_present(AVMediaType media) { + @try { + return [AVCaptureDevice defaultDeviceWithMediaType:media] != nil ? 1 : 0; + } @catch (NSException* e) { + (void)e; + return 0; + } +} + +/* Prompt once and wait, bounded. 30 seconds is the same budget periph.swift + * used: long enough for a human to notice a dialog and decide, short enough + * that an unattended run fails rather than parks forever. A timeout is reported + * as "not granted", which is the safe reading — we genuinely do not know that + * it was. */ +static int el_cap_request(AVMediaType media) { + __block int granted = 0; + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + @try { + [AVCaptureDevice requestAccessForMediaType:media + completionHandler:^(BOOL ok) { + granted = ok ? 1 : 0; + dispatch_semaphore_signal(sem); + }]; + } @catch (NSException* e) { + (void)e; + return 0; + } + if (dispatch_semaphore_wait(sem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(30 * NSEC_PER_SEC))) != 0) { + return 0; /* timed out — treat as refused */ + } + return granted; +} + +/* ════════════════════════════════════════════════════════════════════════════ + * MICROPHONE — one-shot capture + * ══════════════════════════════════════════════════════════════════════════ */ + +/* The sink the tap block writes into. An object rather than a static so two + * captures can never share state, and so ARC keeps it alive for exactly as long + * as the block that captured it. The lock is real, not decorative: the tap runs + * on an AVAudioEngine-internal thread and the waiter runs on the caller's. */ +@interface ElCapMicSink : NSObject +@property (nonatomic, strong) NSMutableData* pcm; +@property (nonatomic, strong) NSLock* lock; +@end + +@implementation ElCapMicSink +- (instancetype)init { + self = [super init]; + if (self) { + _pcm = [NSMutableData data]; + _lock = [[NSLock alloc] init]; + } + return self; +} +@end + +/* Capture `seconds` of mono 16-bit PCM at `sample_rate`. + * + * The hardware format is NOT assumed. A built-in mic will typically hand back + * float32 at 44.1 or 48 kHz, an aggregate device may be 8 channels at 96 kHz, + * and a caller asking for 16 kHz mono (which is what the formant path wants) + * gets 16 kHz mono either way. AVAudioConverter does the rate conversion and + * the downmix; doing it by hand would mean writing a resampler in the one file + * that is supposed to contain no arithmetic. + * + * The converter is built ONCE, outside the tap, because a sample-rate converter + * carries filter state across buffers — rebuilding it per callback would put a + * discontinuity at every buffer boundary, which is audible and which would then + * show up in El's spectral descriptors as energy that was never in the room. */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { + el_val_t empty = el_list_empty(); + + if (!el_cap_authorized(AVMediaTypeAudio)) return empty; + + int64_t secs = (int64_t)seconds; + int64_t sr = (int64_t)sample_rate; + if (secs <= 0 || sr <= 0) return empty; + /* Bound the ask. A caller that asks for a year of audio has made a mistake, + * and honouring it would mean an unkillable capture and an OOM. */ + if (secs > 300) secs = 300; + if (sr > 384000) sr = 384000; + + __block AVAudioEngine* engine = nil; + AVAudioInputNode* input = nil; + AVAudioFormat* hwFmt = nil; + int tapped = 0; + + @try { + engine = [[AVAudioEngine alloc] init]; + input = [engine inputNode]; + hwFmt = [input inputFormatForBus:0]; + } @catch (NSException* e) { + (void)e; + return empty; + } + if (!input || !hwFmt || hwFmt.sampleRate <= 0 || hwFmt.channelCount == 0) { + return empty; + } + + /* Preferred target: mono int16 at the requested rate. If the converter + * refuses that pairing (some exotic input layouts will not downmix), fall + * back to keeping the hardware's channel count and taking channel 0 on the + * way out — the organ is mono by design and inventing a downmix here would + * be an opinion about content this layer is not entitled to have. */ + AVAudioFormat* outFmt = + [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:(double)sr + channels:1 + interleaved:YES]; + AVAudioConverter* conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt + toFormat:outFmt] : nil; + AVAudioChannelCount outCh = 1; + if (!conv) { + outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:(double)sr + channels:hwFmt.channelCount + interleaved:YES]; + conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil; + outCh = hwFmt.channelCount; + } + if (!conv || !outFmt) return empty; + + ElCapMicSink* sink = [[ElCapMicSink alloc] init]; + const int64_t want = secs * sr; /* frames we are waiting for */ + + { + char msg[192]; + snprintf(msg, sizeof(msg), + "MIC: opening the microphone for %llds -> %lld Hz mono PCM " + "(local, never egresses).", (long long)secs, (long long)sr); + el_cap_disclose(msg); + } + + const double ratio = (double)sr / hwFmt.sampleRate; + + @try { + [input installTapOnBus:0 + bufferSize:4096 + format:hwFmt + block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { + (void)when; + if (!buf || buf.frameLength == 0) return; + + AVAudioFrameCount cap = + (AVAudioFrameCount)((double)buf.frameLength * ratio) + 1024; + AVAudioPCMBuffer* out = + [[AVAudioPCMBuffer alloc] initWithPCMFormat:outFmt frameCapacity:cap]; + if (!out) return; + + __block BOOL fed = NO; + AVAudioConverterInputBlock feed = + ^AVAudioBuffer* _Nullable (AVAudioPacketCount need, + AVAudioConverterInputStatus* _Nonnull status) { + (void)need; + if (fed) { *status = AVAudioConverterInputStatus_NoDataNow; return nil; } + fed = YES; + *status = AVAudioConverterInputStatus_HaveData; + return buf; + }; + + NSError* err = nil; + AVAudioConverterOutputStatus st = + [conv convertToBuffer:out error:&err withInputFromBlock:feed]; + if (st == AVAudioConverterOutputStatus_Error || out.frameLength == 0) return; + + const int16_t* src = out.int16ChannelData ? out.int16ChannelData[0] : NULL; + if (!src) return; + + NSUInteger n = (NSUInteger)out.frameLength; + [sink.lock lock]; + if (outCh == 1) { + [sink.pcm appendBytes:src length:n * sizeof(int16_t)]; + } else { + /* Interleaved: stride to channel 0. */ + for (NSUInteger i = 0; i < n; i++) { + int16_t v = src[i * outCh]; + [sink.pcm appendBytes:&v length:sizeof(int16_t)]; + } + } + [sink.lock unlock]; + }]; + tapped = 1; + + [engine prepare]; + NSError* startErr = nil; + if (![engine startAndReturnError:&startErr]) { + [input removeTapOnBus:0]; + return empty; + } + } @catch (NSException* e) { + (void)e; + @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } + @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } + return empty; + } + + /* Wait for `want` frames, bounded by the material's own duration plus a + * margin. A device that stops producing must not become a hang. */ + const int64_t deadline_us = (secs + 5) * 1000000; + int64_t waited_us = 0; + const int64_t tick_us = 5000; + for (;;) { + [sink.lock lock]; + int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); + [sink.lock unlock]; + if (have >= want || waited_us >= deadline_us) break; + usleep((useconds_t)tick_us); + waited_us += tick_us; + } + + @try { [input removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } + @try { [engine stop]; } @catch (NSException* e) { (void)e; } + engine = nil; + + /* Hand up exactly what was asked for, or everything we got if the device + * came up short. Never padded: silence we invented is indistinguishable + * from silence we heard, and El has no way to tell them apart afterwards. */ + [sink.lock lock]; + int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); + int64_t n = have < want ? have : want; + const int16_t* pcm = (const int16_t*)[sink.pcm bytes]; + el_val_t list = empty; + for (int64_t i = 0; i < n; i++) { + list = el_list_append(list, (el_val_t)(int64_t)pcm[i]); + } + [sink.lock unlock]; + + return list; +} + +el_val_t mic_available(void) { + if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; + return (el_val_t)(el_cap_device_present(AVMediaTypeAudio) ? 1 : 0); +} + +el_val_t mic_request_access(void) { + return (el_val_t)(el_cap_request(AVMediaTypeAudio) ? 1 : 0); +} + +/* ════════════════════════════════════════════════════════════════════════════ + * MICROPHONE — live monitor (the full-duplex ear) + * + * converse needs to keep listening WHILE it speaks, which means the mic is open + * at the same time as the speaker. In a real room that is a feedback path: + * without cancellation Neuron hears its own voice, decides someone is talking, + * and barges in on itself. setVoiceProcessingEnabled: hands the input node to + * the OS voice-processing unit, which subtracts the known output signal from + * the input — the single thing that makes barge-in work outside a headset. + * + * It is not always available (some aggregate and virtual devices refuse it), so + * failure to enable it is reported as a DISTINCT return value (2) rather than + * folded into success. The caller needs to know, because the correct response + * is to raise the VAD floor, and a caller that thinks AEC is on will set that + * floor far too low. + * + * The tap keeps only a running short-window RMS in a static behind a mutex. + * Deliberately not a queue of samples: this path is polled at ~50 Hz by a loop + * that only ever asks "is someone talking", and buffering audio nobody reads + * would be an unbounded allocation in the middle of a conversation. + * ══════════════════════════════════════════════════════════════════════════ */ + +static AVAudioEngine* g_mon_engine = nil; +static int g_mon_running = 0; +static int g_mon_code = 0; /* what the successful start reported */ +static double g_mon_rms = 0.0; +static pthread_mutex_t g_mon_lock = PTHREAD_MUTEX_INITIALIZER; + +el_val_t mic_monitor_start(void) { + /* Idempotent, and it re-reports the ORIGINAL code rather than a bare 1: a + * caller that starts twice must not be told AEC is on when the first start + * already discovered it was not. */ + if (g_mon_running) return (el_val_t)g_mon_code; + if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; + + AVAudioEngine* engine = nil; + AVAudioInputNode* input = nil; + AVAudioFormat* fmt = nil; + int aec = 0; + int tapped = 0; + + @try { + engine = [[AVAudioEngine alloc] init]; + input = [engine inputNode]; + } @catch (NSException* e) { + (void)e; + return (el_val_t)0; + } + if (!input) return (el_val_t)0; + + /* Enable AEC BEFORE reading the format: the voice-processing unit imposes + * its own input format, and a tap installed with the pre-VP format would be + * rejected at start. */ + @try { + NSError* vpErr = nil; + if ([input respondsToSelector:@selector(setVoiceProcessingEnabled:error:)]) { + aec = [input setVoiceProcessingEnabled:YES error:&vpErr] ? 1 : 0; + } + } @catch (NSException* e) { + (void)e; + aec = 0; + } + + @try { + fmt = [input inputFormatForBus:0]; + } @catch (NSException* e) { + (void)e; + return (el_val_t)0; + } + if (!fmt || fmt.sampleRate <= 0 || fmt.channelCount == 0) return (el_val_t)0; + + el_cap_disclose(aec + ? "MIC: opening the microphone for live monitoring, echo-cancelled (local)." + : "MIC: opening the microphone for live monitoring, NO echo cancellation (local)."); + + @try { + [input installTapOnBus:0 + bufferSize:1024 + format:fmt + block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { + (void)when; + if (!buf) return; + AVAudioFrameCount n = buf.frameLength; + if (n == 0) return; + + double sum = 0.0; + /* Whatever the VP unit hands back — float32 is the norm, int16 and + * int32 are possible on odd hardware — normalise to -1..1 so the + * Float El sees means the same thing on every device. */ + if (buf.floatChannelData) { + const float* ch = buf.floatChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) sum += (double)ch[i] * (double)ch[i]; + } else if (buf.int16ChannelData) { + const int16_t* ch = buf.int16ChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) { + double v = (double)ch[i] / 32768.0; + sum += v * v; + } + } else if (buf.int32ChannelData) { + const int32_t* ch = buf.int32ChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) { + double v = (double)ch[i] / 2147483648.0; + sum += v * v; + } + } else { + return; + } + + double rms = sqrt(sum / (double)n); + if (rms < 0.0) rms = 0.0; + if (rms > 1.0) rms = 1.0; + + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = rms; + pthread_mutex_unlock(&g_mon_lock); + }]; + tapped = 1; + + [engine prepare]; + NSError* startErr = nil; + if (![engine startAndReturnError:&startErr]) { + [input removeTapOnBus:0]; + return (el_val_t)0; + } + } @catch (NSException* e) { + (void)e; + @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } + @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } + return (el_val_t)0; + } + + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = 0.0; + pthread_mutex_unlock(&g_mon_lock); + + g_mon_engine = engine; + g_mon_running = 1; + g_mon_code = aec ? 1 : 2; + return (el_val_t)g_mon_code; +} + +/* Float in 0..1. Reads the last window the tap computed; never blocks on the + * audio thread beyond the mutex, because this is polled inside a turn-taking + * loop where a stall IS a missed barge-in. */ +el_val_t mic_monitor_rms(void) { + double rms = 0.0; + pthread_mutex_lock(&g_mon_lock); + rms = g_mon_rms; + pthread_mutex_unlock(&g_mon_lock); + return el_from_float(rms); +} + +el_val_t mic_monitor_stop(void) { + AVAudioEngine* engine = g_mon_engine; + g_mon_engine = nil; + g_mon_running = 0; + g_mon_code = 0; + + if (engine) { + @try { [[engine inputNode] removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } + @try { [engine stop]; } @catch (NSException* e) { (void)e; } + } + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = 0.0; + pthread_mutex_unlock(&g_mon_lock); + return (el_val_t)1; +} + +/* ════════════════════════════════════════════════════════════════════════════ + * CAMERA + * ══════════════════════════════════════════════════════════════════════════ */ + +static void el_cap_free_bitmap(void* info, const void* data, size_t size) { + (void)info; (void)size; + free((void*)data); +} + +/* CVPixelBuffer -> CGImage, own-core, no CoreImage. + * + * The output is pinned to 32BGRA at the AVCaptureVideoDataOutput (see below) + * precisely so this conversion can be a memcpy and a CGImageCreate. The + * alternative — accepting the camera's native 2vuy/420v and colour-converting + * here — would mean either pulling in CoreImage or writing a YUV->RGB matrix in + * the file that is supposed to contain no arithmetic. Asking the capture output + * for BGRA moves that work into AVFoundation, where it is already written and + * already hardware-accelerated. + * + * The rows are copied out rather than aliased because the CVPixelBuffer is + * recycled by the capture session the moment the delegate returns; a CGImage + * pointing at it would be pointing at the NEXT frame by the time anyone looked. */ +static CGImageRef el_cap_cgimage_from_pixelbuffer(CVPixelBufferRef pb) { + if (!pb) return NULL; + if (CVPixelBufferGetPixelFormatType(pb) != kCVPixelFormatType_32BGRA) return NULL; + if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return NULL; + + size_t w = CVPixelBufferGetWidth(pb); + size_t h = CVPixelBufferGetHeight(pb); + size_t src_bpr = CVPixelBufferGetBytesPerRow(pb); + const uint8_t* base = (const uint8_t*)CVPixelBufferGetBaseAddress(pb); + + CGImageRef img = NULL; + if (base && w > 0 && h > 0 && src_bpr >= w * 4) { + size_t dst_bpr = w * 4; + uint8_t* copy = (uint8_t*)malloc(dst_bpr * h); + if (copy) { + for (size_t y = 0; y < h; y++) { + memcpy(copy + y * dst_bpr, base + y * src_bpr, dst_bpr); + } + CGDataProviderRef dp = + CGDataProviderCreateWithData(NULL, copy, dst_bpr * h, el_cap_free_bitmap); + if (dp) { + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + if (cs) { + img = CGImageCreate(w, h, 8, 32, dst_bpr, cs, + (CGBitmapInfo)(kCGBitmapByteOrder32Little | + kCGImageAlphaNoneSkipFirst), + dp, NULL, false, kCGRenderingIntentDefault); + CGColorSpaceRelease(cs); + } + CGDataProviderRelease(dp); /* provider owns `copy` from here */ + } else { + free(copy); + } + } + } + + CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + return img; +} + +/* The frame delegate. AVCaptureVideoDataOutput is used rather than + * AVCapturePhotoOutput for the same reason periph.swift used it: the photo path + * wants KVO and a session owned by an app object, and this runs in a plain CLI + * process with no run loop it can assume. A data output just calls back. + * + * The first frames are dropped on purpose. A camera that has just been powered + * on is still converging exposure and white balance, and the first frame is + * reliably darker and greener than the room. El's scene-geometry descriptors + * are brightness and mean-colour statistics, so handing up an unsettled frame + * would not produce a slightly worse answer, it would produce a confidently + * wrong one. */ +@interface ElCapFrameGrabber : NSObject { + CGImageRef _img; + int _seen; + dispatch_semaphore_t _sem; +} +- (dispatch_semaphore_t)sem; +- (CGImageRef)takeImage; /* transfers ownership to the caller */ +@end + +@implementation ElCapFrameGrabber + +- (instancetype)init { + self = [super init]; + if (self) { + _img = NULL; + _seen = 0; + _sem = dispatch_semaphore_create(0); + } + return self; +} + +- (dispatch_semaphore_t)sem { return _sem; } + +- (CGImageRef)takeImage { + CGImageRef out = _img; + _img = NULL; + return out; +} + +- (void)dealloc { + if (_img) { CGImageRelease(_img); _img = NULL; } +} + +/* Runs on the serial delegate queue, so no lock is needed among callbacks; the + * waiter only reads _img after the semaphore has been signalled AND the session + * has been stopped, which orders it after the last callback. */ +- (void)captureOutput:(AVCaptureOutput*)output +didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection*)connection { + (void)output; (void)connection; + _seen++; + if (_img != NULL || _seen < 5) return; /* let exposure settle */ + + CVImageBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pb) return; + CGImageRef img = el_cap_cgimage_from_pixelbuffer(pb); + if (!img) return; + _img = img; + dispatch_semaphore_signal(_sem); +} + +@end + +/* Bring the camera up, take exactly one settled frame, put it back down. + * Returns a +1 CGImageRef the caller releases, or NULL. Bounded at 10s: a + * camera held by another process, or one whose TCC grant was revoked between + * the check and the open, must fail rather than park. */ +static CGImageRef el_cap_grab_frame(void) { + AVCaptureSession* session = nil; + AVCaptureVideoDataOutput* output = nil; + ElCapFrameGrabber* grabber = nil; + CGImageRef img = NULL; + + @try { + AVCaptureDevice* dev = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; + if (!dev) return NULL; + + NSError* err = nil; + AVCaptureDeviceInput* in = [AVCaptureDeviceInput deviceInputWithDevice:dev error:&err]; + if (!in) return NULL; + + session = [[AVCaptureSession alloc] init]; + session.sessionPreset = AVCaptureSessionPresetPhoto; + if (![session canAddInput:in]) return NULL; + [session addInput:in]; + + output = [[AVCaptureVideoDataOutput alloc] init]; + output.alwaysDiscardsLateVideoFrames = YES; + /* Pin the pixel format so the CGImage conversion above stays a memcpy. + * Every macOS capture device advertises 32BGRA. */ + output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey : + @(kCVPixelFormatType_32BGRA) }; + + grabber = [[ElCapFrameGrabber alloc] init]; + dispatch_queue_t q = dispatch_queue_create("el.capture.camera", DISPATCH_QUEUE_SERIAL); + [output setSampleBufferDelegate:grabber queue:q]; + + if (![session canAddOutput:output]) return NULL; + [session addOutput:output]; + + el_cap_disclose("CAMERA: opening the camera for one frame (local, never egresses)."); + [session startRunning]; + } @catch (NSException* e) { + (void)e; + @try { [session stopRunning]; } @catch (NSException* e2) { (void)e2; } + return NULL; + } + + long timed_out = dispatch_semaphore_wait([grabber sem], + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC))); + + /* Stop first, then detach the delegate, then read. In that order the last + * callback has already returned by the time anyone touches the image. */ + @try { [session stopRunning]; } @catch (NSException* e) { (void)e; } + @try { [output setSampleBufferDelegate:nil queue:NULL]; } @catch (NSException* e) { (void)e; } + + if (timed_out == 0) img = [grabber takeImage]; + return img; +} + +el_val_t camera_available(void) { + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + return (el_val_t)(el_cap_device_present(AVMediaTypeVideo) ? 1 : 0); +} + +el_val_t camera_request_access(void) { + return (el_val_t)(el_cap_request(AVMediaTypeVideo) ? 1 : 0); +} + +/* Longest edge of the grid handed to El. 64 is not a resolution, it is a budget: + * a 1920x1080 frame is 6.2 MILLION packed RGB ints, and building that as an El + * list would cost more time and memory than everything El then does with it. + * The descriptors El computes over this — mean colour, brightness, a 3x3 + * luminance grid — are region statistics, and region statistics do not get + * meaningfully better above a 64-wide grid. The TRUE frame dimensions are + * reported separately so nothing downstream has to guess what was thrown away. */ +#define EL_CAP_GRID_MAX 64 + +/* One frame as Map{width, height, grid_w, grid_h, pixels:[Int]}. + * + * "pixels" is packed R,G,B with NO alpha — three ints per grid cell, row-major + * from the TOP-LEFT. (CGBitmapContext lays its buffer out top row first and + * CGContextDrawImage does the flip, so row 0 here is the top of the frame, the + * same convention periph.swift's grid indexing assumed.) Alpha is dropped + * because a camera frame has none worth carrying and it would inflate the list + * by a third to say "opaque" six thousand times. */ +el_val_t camera_capture_rgb(void) { + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + + CGImageRef img = el_cap_grab_frame(); + if (!img) return (el_val_t)0; + + size_t w = CGImageGetWidth(img); + size_t h = CGImageGetHeight(img); + if (w == 0 || h == 0) { CGImageRelease(img); return (el_val_t)0; } + + /* Preserve aspect ratio, longest edge capped. */ + size_t gw = w, gh = h; + size_t longest = w > h ? w : h; + if (longest > EL_CAP_GRID_MAX) { + double s = (double)EL_CAP_GRID_MAX / (double)longest; + gw = (size_t)((double)w * s + 0.5); + gh = (size_t)((double)h * s + 0.5); + if (gw == 0) gw = 1; + if (gh == 0) gh = 1; + } + + size_t bpr = gw * 4; + uint8_t* buf = (uint8_t*)calloc(1, bpr * gh); + if (!buf) { CGImageRelease(img); return (el_val_t)0; } + + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + CGContextRef ctx = cs ? CGBitmapContextCreate(buf, gw, gh, 8, bpr, cs, + (CGBitmapInfo)kCGImageAlphaPremultipliedLast) + : NULL; + if (cs) CGColorSpaceRelease(cs); + if (!ctx) { free(buf); CGImageRelease(img); return (el_val_t)0; } + + /* Nearest-neighbour. This is a decimation for statistics, not a thumbnail + * for a human to look at; smoothing would only cost time and blur the very + * region boundaries the grid exists to measure. */ + CGContextSetInterpolationQuality(ctx, kCGInterpolationNone); + CGContextDrawImage(ctx, CGRectMake(0, 0, (CGFloat)gw, (CGFloat)gh), img); + CGContextRelease(ctx); + CGImageRelease(img); + + el_val_t pixels = el_list_empty(); + for (size_t y = 0; y < gh; y++) { + const uint8_t* row = buf + y * bpr; + for (size_t x = 0; x < gw; x++) { + const uint8_t* p = row + x * 4; /* RGBA8, premultiplied-last */ + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[0]); + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[1]); + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[2]); + } + } + free(buf); + + el_val_t m = el_map_new((el_val_t)0); + if (!m) return (el_val_t)0; + m = el_map_set(m, EL_STR("width"), (el_val_t)(int64_t)w); + m = el_map_set(m, EL_STR("height"), (el_val_t)(int64_t)h); + m = el_map_set(m, EL_STR("grid_w"), (el_val_t)(int64_t)gw); + m = el_map_set(m, EL_STR("grid_h"), (el_val_t)(int64_t)gh); + m = el_map_set(m, EL_STR("pixels"), pixels); + return m; +} + +/* One frame to disk as JPEG, at FULL resolution — the opposite budget from + * camera_capture_rgb, and for the opposite reason. A file is not being walked + * element-by-element by an interpreter; it costs one ImageIO call and it is the + * artefact a human or a later pass will actually look at. The encoder is + * ImageIO's because a JPEG encoder is a codec, and re-implementing one in El + * would be a large amount of arithmetic that buys nothing: the point of keeping + * work in El is the reasoning, not the entropy coding. */ +el_val_t camera_capture_jpeg(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p || !*p) return (el_val_t)0; + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + + CGImageRef img = el_cap_grab_frame(); + if (!img) return (el_val_t)0; + + int ok = 0; + @autoreleasepool { + NSString* ns = [NSString stringWithUTF8String:p]; + NSURL* url = ns ? [NSURL fileURLWithPath:ns] : nil; + if (url) { + CGImageDestinationRef dst = + CGImageDestinationCreateWithURL((__bridge CFURLRef)url, CFSTR("public.jpeg"), 1, NULL); + if (dst) { + CGImageDestinationAddImage(dst, img, NULL); + ok = CGImageDestinationFinalize(dst) ? 1 : 0; + CFRelease(dst); + } + } + } + CGImageRelease(img); + return (el_val_t)ok; +} + +#endif /* __APPLE__ */ diff --git a/lang/runtime/el_peripheral_null.c b/lang/runtime/el_peripheral_null.c new file mode 100644 index 0000000..ff9b4e2 --- /dev/null +++ b/lang/runtime/el_peripheral_null.c @@ -0,0 +1,76 @@ +/* el_peripheral_null.c — the no-device build of El's I/O organ. + * + * Every entry point declared in el_runtime.h's "Peripheral" block, implemented + * as an honest refusal. This is what a platform without an El audio/capture + * realizer links instead of el_audio_darwin.m + el_capture_darwin.m, so an El + * program that speaks or listens still COMPILES AND LINKS everywhere. + * + * The distinction that matters: these do not pretend. speaker_available() and + * mic_available() return 0, and every operation returns its failure sentinel. + * A program asking "can I speak here?" gets a truthful no, rather than a + * silence it would have to infer something from. Silent success is the failure + * mode this whole change exists to eliminate — El spent this entire codebase's + * history writing WAV files full of zeros and reporting ok=true, and nobody + * caught it because nothing ever said "there is no sound here". + * + * Compiled INSTEAD OF the Darwin realizers, never alongside them — the symbols + * are the same by design, which is the point: the El side never branches on + * platform, it branches on speaker_available(). + */ + +#include "el_runtime.h" + +#if !defined(__APPLE__) + +/* ── Speaker ─────────────────────────────────────────────────────────────── */ + +el_val_t speaker_available(void) { return (el_val_t)0; } +el_val_t speaker_name(void) { return EL_STR("none"); } + +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { + (void)samples; (void)sample_rate; + return (el_val_t)0; +} + +el_val_t speaker_play_wav(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { + (void)samples; (void)sample_rate; + return (el_val_t)0; +} + +el_val_t speaker_playing(void) { return (el_val_t)0; } +el_val_t speaker_stop(void) { return (el_val_t)0; } +el_val_t speaker_played_frames(void) { return (el_val_t)0; } + +/* ── Microphone ──────────────────────────────────────────────────────────── */ + +el_val_t mic_available(void) { return (el_val_t)0; } +el_val_t mic_request_access(void) { return (el_val_t)0; } + +/* Empty list, not 0: the contract says capture returns samples, and a caller + * iterating the result must find nothing rather than dereference a non-list. */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { + (void)seconds; (void)sample_rate; + return el_list_empty(); +} + +el_val_t mic_monitor_start(void) { return (el_val_t)0; } +el_val_t mic_monitor_rms(void) { return el_from_float(0.0); } +el_val_t mic_monitor_stop(void) { return (el_val_t)0; } + +/* ── Camera ──────────────────────────────────────────────────────────────── */ + +el_val_t camera_available(void) { return (el_val_t)0; } +el_val_t camera_request_access(void) { return (el_val_t)0; } +el_val_t camera_capture_rgb(void) { return (el_val_t)0; } + +el_val_t camera_capture_jpeg(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +#endif /* !__APPLE__ */ diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index cfae8e8..165aa6b 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -80,6 +80,84 @@ void println(el_val_t s); void print(el_val_t s); el_val_t readline(void); +/* stderr counterpart of println (defined in el_seed.c). El could write to + * stdout and nowhere else, which is right for a program's RESULT and wrong for + * everything about how that result was produced. Disclosure especially has to + * leave on a stream the caller can separate from the answer: a program that + * announces "I am about to open the microphone" on stdout has corrupted its own + * output. Flushed on every call, so a disclosure reaches the terminal BEFORE + * the device it describes is touched rather than whenever the buffer drains. */ +void eprintln(el_val_t s); + +/* ── Peripheral: the speaker, the microphone, the camera ───────────────────── + * + * El's I/O organ. Implemented per platform in its OWN translation unit — + * el_audio_darwin.m / el_capture_darwin.m on Darwin, el_peripheral_null.c + * everywhere else — so El code that speaks or listens links on every platform + * and merely reports having no device where there isn't one. Declared here and + * deliberately NOT implemented in el_runtime.c: acquiring a device must not + * mean editing the middle of the language, the same rule the realizer registry + * follows for modalities. + * + * These are the ONLY parts of the organ that are not El. Everything above the + * sample buffer — WAV encode/decode, LPC autocorrelation, Levinson-Durbin, + * formant extraction, source-filter resynthesis, the compact descriptors, the + * converse decision loop — is arithmetic, and arithmetic belongs in El. What + * remains here is what El cannot express: handing a buffer to the DAC and + * waiting for it to drain, and asking the OS for frames off a capture device. + * + * Local by construction: none of these entry points has a network path. Samples + * and pixels go to and from local hardware and nowhere else. Consent is + * enforced ABOVE this layer in El (peripheral/src/organ.el) for the Neuron-level + * grant, and BELOW it by the OS for TCC; capture fails closed on either. */ + +/* Speaker (efferent). speaker_play_pcm16 BLOCKS until the audio has actually + * been played rather than merely queued, so a caller can sequence utterances + * without guessing durations and without clipping each tail. */ +el_val_t speaker_available(void); /* 1 if a real speaker backs this build */ +el_val_t speaker_name(void); /* backend id, e.g. "coreaudio-audioqueue" */ +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate); /* [Int] 16-bit mono; 1 ok */ +el_val_t speaker_play_wav(el_val_t path); /* 16-bit mono RIFF/WAVE; 1 ok */ + +/* Asynchronous playback — required by converse, which must keep listening while + * it speaks and must be able to stop ON THE SPOT mid-buffer. A blocking play + * cannot be interrupted, and "finish the current buffer" is not barge-in. + * speaker_stop() halts output immediately; speaker_playing() reports whether + * the hardware is still going; speaker_played_frames() is how far it actually + * got, which is what makes an interrupted utterance resumable at the sample. */ +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate); +el_val_t speaker_playing(void); +el_val_t speaker_stop(void); +el_val_t speaker_played_frames(void); + +/* Microphone (afferent). Fails CLOSED: returns 0 unless the OS has granted + * capture access. mic_capture_pcm16 blocks for `seconds` and returns an [Int] + * of 16-bit mono samples at `sample_rate` — the raw stream is handed to El and + * never written anywhere by this layer. mic_available() reports device + + * permission state without prompting. */ +el_val_t mic_available(void); /* 1 device present AND OS-authorized */ +el_val_t mic_request_access(void); /* prompt once; 1 if granted */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate); /* [Int], empty on refusal */ + +/* Live monitoring for full-duplex converse. mic_monitor_start enables the OS + * voice-processing unit (acoustic echo cancellation) so the microphone does not + * hear the speaker — without AEC, Neuron barges in on its own voice and + * turn-taking is unusable in a real room. mic_monitor_rms returns the current + * short-window RMS as a Float in 0..1. */ +el_val_t mic_monitor_start(void); /* 1 ok; 2 = started but AEC unavailable */ +el_val_t mic_monitor_rms(void); /* Float */ +el_val_t mic_monitor_stop(void); + +/* Camera (afferent). Fails CLOSED like the microphone. camera_capture_rgb + * returns a Map with width/height and the frame as an [Int] of packed RGB + * bytes, so the descriptor arithmetic can happen in El rather than here. + * camera_capture_jpeg writes an encoded frame via ImageIO, which is a codec and + * not something El should re-implement. */ +el_val_t camera_available(void); +el_val_t camera_request_access(void); +el_val_t camera_capture_rgb(void); /* Map{width,height,pixels:[Int]} or 0 */ +el_val_t camera_capture_jpeg(el_val_t path); /* 1 ok */ + /* ── String builtins ─────────────────────────────────────────────────────── */ el_val_t el_str_concat(el_val_t a, el_val_t b); diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index 8313183..15b5ba6 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -154,9 +154,18 @@ static void seed_request_start(void) { * file still links on its own. */ __attribute__((weak)) void el_str_cache_flush(void); +/* Byte-buffer capacity registry (defined below, next to the string + * primitives). The arena frees the pointers it tracked, so any capacity + * entry for those addresses must go with them — otherwise a later malloc + * reusing the address would inherit a stale width. */ +static void seed_cap_drop(const char* p); + static void seed_request_end(void) { _seed_arena_on = 0; - for (size_t i = 0; i < _seed_arena.count; i++) free(_seed_arena.ptrs[i]); + for (size_t i = 0; i < _seed_arena.count; i++) { + seed_cap_drop(_seed_arena.ptrs[i]); + free(_seed_arena.ptrs[i]); + } _seed_arena.count = 0; if (el_str_cache_flush) el_str_cache_flush(); /* freed pointers may be reused */ } @@ -188,6 +197,114 @@ static char* seed_strbuf(size_t n) { static el_val_t seed_wrap_str(char* s) { return EL_STR(s); } +/* ── Byte-buffer capacity registry ──────────────────────────────────────────── + * A String produced by __str_alloc is a fixed-size BYTE BUFFER, not text. Its + * length is the capacity it was asked for; strlen() is meaningless on it, + * because the buffer is zero-filled and binary content (PCM audio, RIFF + * headers, image rasters) contains NUL bytes by nature. + * + * Before this registry existed, __str_set_char bounds-checked the write index + * against strlen(p). For a freshly __str_alloc'd buffer strlen(p) == 0, so the + * check `idx >= len` rejected EVERY index and the function was a total no-op: + * every El program that built bytes this way wrote a file of pure zeros and + * still saw a success return. That is why El's own-core WAV writer emitted + * 55,244 silent bytes with a correct-looking header length and no header. + * + * The fix cannot be "trust the index", because that removes the bound. It also + * cannot be a length header stored behind the pointer, because __str_set_char + * accepts any String — including a string literal in .rodata, where reading the + * bytes preceding the pointer is undefined and may fault. So capacity is kept + * in a side table keyed by the pointer itself: allocation registers, the arena + * sweep unregisters, and anything not registered keeps the exact strlen + * behaviour it had before. Text semantics are unchanged; byte buffers gain the + * bound they always should have had. */ + +typedef struct { + char* ptr; /* NULL = empty slot, (char*)1 = tombstone */ + size_t cap; +} SeedCapEntry; + +#define SEED_CAP_TOMB ((char*)1) + +static _Thread_local SeedCapEntry* _seed_cap = NULL; +static _Thread_local size_t _seed_cap_mask = 0; /* table size - 1 */ +static _Thread_local size_t _seed_cap_used = 0; /* live + tombstoned */ + +static size_t seed_cap_hash(const char* p) { + uintptr_t h = (uintptr_t)p >> 4; /* malloc alignment: low bits are dead */ + h *= (uintptr_t)0x9E3779B97F4A7C15ull; + return (size_t)(h >> 32); +} + +static void seed_cap_put(char* p, size_t cap); + +static void seed_cap_grow(void) { + size_t old_size = _seed_cap_mask ? _seed_cap_mask + 1 : 0; + SeedCapEntry* old = _seed_cap; + size_t new_size = old_size ? old_size * 2 : 256; + SeedCapEntry* fresh = calloc(new_size, sizeof(SeedCapEntry)); + if (!fresh) return; /* out of memory: keep old table */ + _seed_cap = fresh; + _seed_cap_mask = new_size - 1; + _seed_cap_used = 0; + for (size_t i = 0; i < old_size; i++) { + if (old[i].ptr && old[i].ptr != SEED_CAP_TOMB) seed_cap_put(old[i].ptr, old[i].cap); + } + free(old); +} + +static void seed_cap_put(char* p, size_t cap) { + if (!p) return; + if (!_seed_cap || (_seed_cap_used + 1) * 4 >= (_seed_cap_mask + 1) * 3) { + seed_cap_grow(); + if (!_seed_cap) return; + } + size_t i = seed_cap_hash(p) & _seed_cap_mask; + size_t first_free = (size_t)-1; + for (;;) { + char* e = _seed_cap[i].ptr; + if (e == p) { _seed_cap[i].cap = cap; return; } /* address reused */ + if (e == SEED_CAP_TOMB && first_free == (size_t)-1) first_free = i; + if (!e) { + if (first_free != (size_t)-1) i = first_free; else _seed_cap_used++; + _seed_cap[i].ptr = p; + _seed_cap[i].cap = cap; + return; + } + i = (i + 1) & _seed_cap_mask; + } +} + +/* Capacity of a registered byte buffer, or -1 when the pointer is not one. */ +static int64_t seed_cap_get(const char* p) { + if (!p || !_seed_cap) return -1; + size_t i = seed_cap_hash(p) & _seed_cap_mask; + for (;;) { + char* e = _seed_cap[i].ptr; + if (!e) return -1; + if (e == (char*)p) return (int64_t)_seed_cap[i].cap; + i = (i + 1) & _seed_cap_mask; + } +} + +static void seed_cap_drop(const char* p) { + if (!p || !_seed_cap) return; + size_t i = seed_cap_hash(p) & _seed_cap_mask; + for (;;) { + char* e = _seed_cap[i].ptr; + if (!e) return; + if (e == (char*)p) { _seed_cap[i].ptr = SEED_CAP_TOMB; return; } + i = (i + 1) & _seed_cap_mask; + } +} + +/* Effective addressable length of a String: its buffer capacity when it is a + * byte buffer, otherwise strlen. */ +static int64_t seed_addressable_len(const char* p) { + int64_t cap = seed_cap_get(p); + return cap >= 0 ? cap : (int64_t)strlen(p); +} + /* ── String primitives ───────────────────────────────────────────────────── */ el_val_t __str_len(el_val_t s) { @@ -199,7 +316,7 @@ el_val_t __str_len(el_val_t s) { el_val_t __str_char_at(el_val_t s, el_val_t i) { const char* p = EL_CSTR(s); if (!p) return 0; - int64_t len = (int64_t)strlen(p); + int64_t len = seed_addressable_len(p); /* capacity for byte buffers */ int64_t idx = (int64_t)i; if (idx < 0 || idx >= len) return 0; return (el_val_t)(unsigned char)p[idx]; @@ -210,13 +327,14 @@ el_val_t __str_alloc(el_val_t n) { if (sz < 0) sz = 0; char* buf = seed_strbuf((size_t)sz); memset(buf, 0, (size_t)sz + 1); + seed_cap_put(buf, (size_t)sz); /* this is a byte buffer of width sz */ return seed_wrap_str(buf); } el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c) { char* p = (char*)(uintptr_t)s; if (!p) return s; - int64_t len = (int64_t)strlen(p); + int64_t len = seed_addressable_len(p); /* capacity for byte buffers */ int64_t idx = (int64_t)i; if (idx < 0 || idx >= len) return s; p[idx] = (char)(unsigned char)(int64_t)c; @@ -406,6 +524,15 @@ el_val_t __fs_mkdir(el_val_t path) { return 1; } +/* stderr counterpart of println. Flushed immediately: a disclosure line is only + * worth anything if it lands before the thing it discloses happens. */ +void eprintln(el_val_t s) { + const char* p = EL_CSTR(s); + fputs(p ? p : "", stderr); + fputc('\n', stderr); + fflush(stderr); +} + el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n) { const char* p = EL_CSTR(path); const char* b = EL_CSTR(bytes); diff --git a/peripheral/src/organ.el b/peripheral/src/organ.el new file mode 100644 index 0000000..498369d --- /dev/null +++ b/peripheral/src/organ.el @@ -0,0 +1,378 @@ +// organ.el — Neuron's I/O organ, in El. +// +// THE PRINCIPLE. El speaks. The engram stores geometry and does not speak. +// Before this file the organ was a 939-line Swift program standing next to the +// language (peripheral/src/periph.swift): Neuron's mouth and ears were a +// separate binary, and "speak" meant "shell out to that binary, which shells +// out to afplay." That is not a voice, it is a subprocess. The voice belongs to +// the language and its runtime. +// +// THE SPLIT. Exactly two things here are not El, and they are the two things El +// cannot express as arithmetic: +// +// the speaker — handing a buffer to the DAC and waiting for it to drain +// the capture — asking the OS for samples off a mic or frames off a camera +// +// Those live in lang/runtime/el_audio_darwin.m and el_capture_darwin.m, as +// their own translation units, declared in el_runtime.h. Everything ELSE that +// the Swift did — WAV encode and decode, LPC autocorrelation, Levinson-Durbin, +// formant extraction off the all-pole envelope, source-filter resynthesis, the +// compact descriptors, the converse yield-or-hold decision — is arithmetic, and +// arithmetic is El's. See organ_dsp.el for that half. +// +// WHERE THE VOICE COMES FROM. Not from this file, and not from a JSON manifest +// on disk. A voice is GEOMETRY IN THE ENGRAM, and the organ goes and gets it by +// asking the engram, the same way anything else asks the engram for anything: +// a query against the graph, then read the numbers off the node that comes +// back. organ_voice_fetch is that. The previous path, load_voice("...json"), +// parsed a file — which quietly made the voice a build artifact instead of a +// memory. If the region is not in the graph, the honest answer is an empty +// result, not a default voice. +// +// WHAT THE ORGAN NEVER DOES. It never learns a word. Pronunciation, vocabulary +// and phonemes are the language faculty's, already built as ingested geometry — +// "the engram knows how to pronounce." The seam is synth_codes(codes, voice, +// pmap): the codes and the phoneme map arrive from the language side as +// geometry, and the organ's whole job is turning them into samples and getting +// the samples out the speaker, plus the same trip in reverse for the senses. +// +// RAILS, all non-negotiable: +// own-core — CoreAudio / AVFoundation / ImageIO, all shipped with the OS. +// No cloud, no model, no heavy dependency. There is no network +// call anywhere in the organ, by construction. +// local-only — raw streams stay on the machine. What leaves a capture is a +// DESCRIPTOR of a few dozen numbers, never the stream. +// consent — two locks on the sensitive senses: a Neuron-level grant AND +// the OS TCC permission. Camera and mic FAIL CLOSED without +// both. The speaker is disclosed but not gated (see below). +// disclosed — every device touch prints a [peripheral] line on stderr. +// Nothing here is ever silent about being a device. + +// ── Disclosure ─────────────────────────────────────────────────────────────── +// +// stderr, not stdout: a program that announces "I am opening the microphone" on +// stdout has corrupted its own output. And flushed immediately, so the line is +// on the terminal BEFORE the device is touched — a disclosure that arrives +// after the fact is a log, not a disclosure. + +fn organ_disclose(msg: String) -> Bool { + eprintln(" [peripheral] " + msg) + return true +} + +// ── Consent, the Neuron-level lock ─────────────────────────────────────────── +// +// The OS has its own lock (TCC) and it is not enough on its own: TCC grants the +// TERMINAL access to the microphone, once, more or less forever. That says the +// user trusts the app. It does not say the user consents to THIS program +// listening THIS time. So Neuron keeps its own grant, revocable, on the same +// footing — and both must be open for a sensitive sense to work. +// +// Stored next to the organ rather than in the engram deliberately: consent must +// be inspectable and revocable without a running graph, and a permission that +// can only be revoked by the system it governs is not a permission. + +fn organ_consent_path() -> String { + let home: String = env("PERIPH_HOME") + if str_eq(home, "") { + return "peripheral/.consent.json" + } + return home + "/.consent.json" +} + +fn organ_consent_granted(device: String) -> Bool { + let raw: String = fs_read(organ_consent_path()) + if str_eq(raw, "") { + return false + } + // A device is granted only on an explicit true. Anything unparseable, + // missing or malformed reads as NOT granted — the failure direction for a + // permission file is always closed. + let key: String = "\"" + device + "\"" + let at: Int = str_index_of(raw, key) + if at < 0 { + return false + } + let tail: String = str_slice(raw, at, str_len(raw)) + let t: Int = str_index_of(tail, "true") + let f: Int = str_index_of(tail, "false") + if t < 0 { + return false + } + if f < 0 { + return true + } + // whichever token appears first after the key is this device's value + if t < f { + return true + } + return false +} + +fn organ_consent_write(camera: Bool, mic: Bool) -> Bool { + let c: String = "false" + if camera { + c = "true" + } + let m: String = "false" + if mic { + m = "true" + } + return fs_write(organ_consent_path(), "{\"camera\": " + c + ", \"mic\": " + m + "}\n") +} + +fn organ_grant(device: String) -> Bool { + let cam: Bool = organ_consent_granted("camera") + let mic: Bool = organ_consent_granted("mic") + if str_eq(device, "camera") { + cam = true + } + if str_eq(device, "mic") { + mic = true + } + let ok: Bool = organ_consent_write(cam, mic) + organ_disclose("granted '" + device + "' (Neuron-level) — raw stream stays local, never egresses.") + return ok +} + +fn organ_revoke(device: String) -> Bool { + let cam: Bool = organ_consent_granted("camera") + let mic: Bool = organ_consent_granted("mic") + if str_eq(device, "camera") { + cam = false + } + if str_eq(device, "mic") { + mic = false + } + let ok: Bool = organ_consent_write(cam, mic) + organ_disclose("revoked '" + device + "' (Neuron-level).") + return ok +} + +fn organ_consent_status() -> String { + let cam: String = "denied" + if organ_consent_granted("camera") { + cam = "granted" + } + let mic: String = "denied" + if organ_consent_granted("mic") { + mic = "granted" + } + return "camera=" + cam + " mic=" + mic +} + +// Both locks, in order, with a disclosure for each outcome. Returns false and +// says exactly which lock is shut — a refusal that does not say why is +// indistinguishable from a bug. +fn organ_may_listen() -> Bool { + if organ_consent_granted("mic") == false { + organ_disclose("CONSENT DENIED for 'mic' (Neuron-level). Run: organ grant mic") + return false + } + if mic_available() == 0 { + organ_disclose("CONSENT DENIED for 'mic' (OS/TCC), or no input device. Grant microphone access to this terminal in System Settings > Privacy.") + return false + } + organ_disclose("consent OK (Neuron + OS) for 'mic' — local only, never egresses.") + return true +} + +fn organ_may_see() -> Bool { + if organ_consent_granted("camera") == false { + organ_disclose("CONSENT DENIED for 'camera' (Neuron-level). Run: organ grant camera") + return false + } + if camera_available() == 0 { + organ_disclose("CONSENT DENIED for 'camera' (OS/TCC), or no capture device. Grant camera access to this terminal in System Settings > Privacy.") + return false + } + organ_disclose("consent OK (Neuron + OS) for 'camera' — local only, never egresses.") + return true +} + +// ── SPEAKER (efferent) ─────────────────────────────────────────────────────── +// +// Not consent-gated, and that is a deliberate asymmetry rather than an +// oversight. The microphone and camera take information OFF the user without +// them necessarily knowing; the speaker puts information INTO a room the user +// is in, audibly, which is self-disclosing by its nature — you cannot secretly +// speak aloud. So the speaker is DISCLOSED (every utterance announces itself on +// stderr) but not gated. Gating it would mean Neuron needs permission to answer. + +fn organ_speak_samples(samples: [Int], sr: Int) -> Bool { + let n: Int = native_list_len(samples) + if n <= 0 { + organ_disclose("SPEAKER: nothing to say (0 samples) — not touching the device.") + return false + } + if speaker_available() == 0 { + organ_disclose("SPEAKER: no audio output on this build (" + speaker_name() + ") — cannot speak.") + return false + } + let secs: Int = n * 1000 / sr + organ_disclose("SPEAKER: playing " + int_to_str(n) + " samples (" + int_to_str(secs) + " ms @ " + int_to_str(sr) + " Hz) ALOUD via " + speaker_name() + " (efferent).") + let ok: Int = speaker_play_pcm16(samples, sr) + if ok == 1 { + organ_disclose("SPEAKER: done — Neuron spoke aloud.") + return true + } + organ_disclose("SPEAKER: playback FAILED.") + return false +} + +fn organ_speak_wav(path: String) -> Bool { + if speaker_available() == 0 { + organ_disclose("SPEAKER: no audio output on this build — cannot speak.") + return false + } + if fs_exists(path) == false { + organ_disclose("SPEAKER: no such file: " + path) + return false + } + organ_disclose("SPEAKER: playing '" + path + "' ALOUD via " + speaker_name() + " (efferent).") + let ok: Int = speaker_play_wav(path) + if ok == 1 { + organ_disclose("SPEAKER: done — Neuron spoke aloud.") + return true + } + organ_disclose("SPEAKER: playback FAILED.") + return false +} + +// ── The voice, fetched FROM THE ENGRAM ─────────────────────────────────────── +// +// This is the part that matters most and is easiest to get subtly wrong. A +// voice is not a constant in code and it is not a JSON file next to the code — +// it is a region of the graph, put there by having heard someone, and the organ +// retrieves it the way anything retrieves a memory: by asking. +// +// The node content is the geometry, in the engram's own flat key=value form: +// voice will | f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531 ... +// so the read is: query the graph, take the returned node, pull the numbers off +// it. Nothing here opens a file. +// +// Returns [f0, f0_end, kf, f1, f2, f3], or an EMPTY list when the region is not +// in the graph. Empty is the honest answer — a caller that gets no voice must +// not be handed a plausible default and left unable to tell the difference +// between "this is how they sound" and "I never heard them." + +// Read an unsigned integer that follows `key` in `s`. Stops at the first +// non-digit, returns 0 when the key is absent. +fn organ_int_after(s: String, key: String) -> Int { + let at: Int = str_index_of(s, key) + if at < 0 { + return 0 + } + let i: Int = at + str_len(key) + let n: Int = str_len(s) + let v: Int = 0 + let seen: Int = 0 + while i < n { + let c: Int = str_char_code(s, i) + if c < 48 { + i = n + } else { + if c > 57 { + i = n + } else { + v = v * 10 + (c - 48) + seen = seen + 1 + i = i + 1 + } + } + } + if seen == 0 { + return 0 + } + return v +} + +// Ask the engram for a named voice region and read its geometry back. +fn organ_voice_fetch(name: String) -> [Int] { + let out: [Int] = native_list_empty() + let marker: String = "voice " + name + " |" + // The graph is asked by MEANING, not by id or by path. + let hits: String = engram_search_json("voice " + name + " f0 formants", 12) + let at: Int = str_index_of(hits, marker) + if at < 0 { + // Fall back to a scan of the resident graph before giving up: search is + // geometric and a small graph may not rank the region first. + let scan: String = engram_scan_nodes_json(500, 0) + at = str_index_of(scan, marker) + if at < 0 { + organ_disclose("VOICE: no region for '" + name + "' in the engram — nothing to speak with.") + return out + } + hits = scan + } + let win: String = str_slice(hits, at, at + 240) + out = native_list_append(out, organ_int_after(win, "f0=")) + out = native_list_append(out, organ_int_after(win, "f0_end=")) + out = native_list_append(out, organ_int_after(win, "kf=")) + out = native_list_append(out, organ_int_after(win, "f1=")) + out = native_list_append(out, organ_int_after(win, "f2=")) + out = native_list_append(out, organ_int_after(win, "f3=")) + organ_disclose("VOICE: fetched '" + name + "' FROM THE ENGRAM — f0=" + int_to_str(native_list_get(out, 0)) + " f0_end=" + int_to_str(native_list_get(out, 1)) + " kf=" + int_to_str(native_list_get(out, 2)) + " f1=" + int_to_str(native_list_get(out, 3)) + " f2=" + int_to_str(native_list_get(out, 4)) + " f3=" + int_to_str(native_list_get(out, 5))) + return out +} + +// Put a measured voice INTO the engram as geometry. This is the afferent end of +// the same wire: a voiceprint (organ_dsp.el's LPC analysis) becomes a node, and +// from then on the voice is a memory rather than a measurement someone happened +// to write down. `prov` carries the honesty: COARSE means one formant triple, no +// coarticulation, no prosody — an impression, explicitly not a clone. +fn organ_voice_ingest(name: String, f0: Int, f0_end: Int, kf: Int, f1: Int, f2: Int, f3: Int, src: String, prov: String) -> String { + let hub: String = engram_node("voice-signature-set " + name + " grounding=measured src=" + src, "VoiceSet", 90) + let body: String = "voice " + name + " | f0=" + int_to_str(f0) + " f0_end=" + int_to_str(f0_end) + " kf=" + int_to_str(kf) + " f1=" + int_to_str(f1) + " f2=" + int_to_str(f2) + " f3=" + int_to_str(f3) + " grounding=measured src=" + src + " prov=" + prov + let vid: String = engram_node(body, "Voice", 90) + engram_connect(hub, vid, 90, "has-signature") + organ_disclose("VOICE: ingested '" + name + "' into the engram as geometry (node " + vid + ").") + return vid +} + +// Turn the fetched geometry into the voice slot-map the render consumes. Kept +// separate from the fetch so the organ never invents a voice: if the fetch came +// back empty this returns empty too, and the caller has to deal with it. +fn organ_voice_profile(name: String, g: [Int]) -> [String] { + let empty: [String] = native_list_empty() + if native_list_len(g) < 6 { + return empty + } + return voice_new(name, native_list_get(g, 0), native_list_get(g, 1), native_list_get(g, 2), 1000, 1000, 8) +} + +// ── Own-core tone ──────────────────────────────────────────────────────────── +// +// The smallest possible proof that the organ owns its medium end to end: a sine +// with a gentle attack and release, computed here, played by us, no file and no +// library anywhere in the path. +fn organ_tone(hz: Int, ms: Int, sr: Int) -> [Int] { + let n: Int = sr * ms / 1000 + let out: [Int] = native_list_empty() + let two_pi: Float = 6.283185307 + let srf: Float = int_to_float(sr) + let hzf: Float = int_to_float(hz) + let i: Int = 0 + // 20 ms of ramp at each end; a square-edged tone clicks, and a click is the + // organ announcing that it does not understand envelopes. + let ramp: Int = sr / 50 + if ramp < 1 { + ramp = 1 + } + while i < n { + let t: Float = int_to_float(i) / srf + let s: Float = math_sin(two_pi * hzf * t) + let env: Int = 32767 + if i < ramp { + env = 32767 * i / ramp + } + let tail: Int = n - i + if tail < ramp { + env = 32767 * tail / ramp + } + let v: Int = float_to_int(s * 9000.0) * env / 32767 + out = native_list_append(out, v) + i = i + 1 + } + return out +}