/* el_audio_darwin.m — the SPEAKER realizer. El's native audio output on Darwin. * * WHY THIS FILE EXISTS. * * Neuron could already turn meaning into samples — the render path in * elp/src/speech.el superposes formant resonances over a glottal source and * produces PCM. What it could not do was make a sound. Every path from those * samples to the air ran outside the language: a 939-line Swift program * (peripheral/src/periph.swift) that shelled out to /usr/bin/afplay. So the * voice was not a capability of El or of Neuron. It was a separate binary * standing next to them, and "speak" meant "ask that binary to speak." * * A speaker is not a language feature the way a string is, but it is exactly * the kind of thing a runtime owns: a device. El already owns the filesystem, * the network, the clock, and a graph. It should own the one output device that * makes it audible. After this file, `speak` is an El operation. * * WHY IT IS A REALIZER AND NOT PURE EL. * * This is the boundary the whole design turns on. Everything ABOVE the sample * buffer is arithmetic and belongs in El: formant geometry, superposition, * envelopes, WAV framing, the voice signature. Everything in this file is the * part that cannot be arithmetic — handing a buffer to CoreAudio and waiting * for the hardware to drain it. There is no way to express "the DAC has now * played these samples" in El, and there should not be. So the split is: El * computes the sound, the realizer emits it, and the realizer is as thin as it * can possibly be — it makes no decisions about content, it has no opinion * about audio, and it cannot synthesize anything. * * The precedent is eg_cosine_batch_strategy_metal_hand.m: a platform-bound * capability compiled as its OWN translation unit, declared in el_runtime.h, * and linked in where the platform supports it. Deliberately NOT a patch to * el_runtime.c — adding a device to El must not mean editing the core runtime, * for the same reason adding a modality must not (see el_runtime.c's realizer * registry: a realizer is resolved by name, so new organs never touch the * middle of the language). el_audio_null.c is the same two entry points for * every platform that is not Darwin, so El code that speaks still links * everywhere and simply reports that it has no speaker. * * WHY AudioQueue AND NOT afplay. * * afplay is a process. Using it means the sound Neuron makes is a file it wrote * and asked something else to open — which forces every utterance through the * disk, cannot start until the whole utterance exists, and puts a fork/exec * between the intent to speak and the sound. AudioQueue takes the samples * directly out of memory. Nothing is written, nothing is spawned, and a caller * that wants to stream can push buffers as it renders them. * * AudioToolbox ships with macOS, so this stays own-core: no cloud, no library * to install, no model. The output is the local speaker and nothing leaves the * machine — there is no network path in this file at all, by construction. */ #import #import #include #include #include #include #include "el_runtime.h" /* Three buffers is the standard AudioQueue depth: one being played by the * hardware, one queued behind it, one being refilled. Fewer risks a gap on a * busy machine; more only adds latency before the first sound. */ #define EL_AQ_NBUF 3 #define EL_AQ_FRAMES 8192 typedef struct { const int16_t* pcm; int64_t frames; int64_t pos; volatile int inflight; /* buffers CoreAudio still owns */ volatile int drained; /* set once the last buffer has been played */ } ElAqState; /* Called on an AudioQueue-internal thread each time a buffer finishes playing. * Refills and re-enqueues while samples remain; when the source is exhausted it * lets the buffer die and counts it out. `drained` flips only when the queue is * holding nothing, which is what makes the play call synchronous without * clipping the tail — the same reason periph.swift used .dataPlayedBack rather * than treating "consumed" as "heard". */ static void el_aq_callback(void* userData, AudioQueueRef q, AudioQueueBufferRef buf) { ElAqState* st = (ElAqState*)userData; int64_t remain = st->frames - st->pos; if (remain <= 0) { if (--st->inflight <= 0) st->drained = 1; return; } int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; memcpy(buf->mAudioData, st->pcm + st->pos, (size_t)n * sizeof(int16_t)); buf->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); st->pos += n; if (AudioQueueEnqueueBuffer(q, buf, 0, NULL) != noErr) { if (--st->inflight <= 0) st->drained = 1; } } /* Play a 16-bit mono PCM buffer out the default output device, blocking until * the hardware has actually finished. Returns 1 on success, 0 on any failure — * never throws, never hangs indefinitely. */ static int el_audio_play_raw(const int16_t* pcm, int64_t frames, int32_t sample_rate) { if (!pcm || frames <= 0 || sample_rate <= 0) return 0; AudioStreamBasicDescription fmt; memset(&fmt, 0, sizeof(fmt)); fmt.mSampleRate = (Float64)sample_rate; fmt.mFormatID = kAudioFormatLinearPCM; fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; fmt.mFramesPerPacket = 1; fmt.mChannelsPerFrame = 1; fmt.mBitsPerChannel = 16; fmt.mBytesPerFrame = 2; fmt.mBytesPerPacket = 2; ElAqState st; memset(&st, 0, sizeof(st)); st.pcm = pcm; st.frames = frames; AudioQueueRef q = NULL; /* NULL run loop => callbacks arrive on an AudioQueue-internal thread, so * this function can simply wait rather than having to pump a run loop it * does not own. El programs are not required to have one. */ if (AudioQueueNewOutput(&fmt, el_aq_callback, &st, NULL, NULL, 0, &q) != noErr || !q) { return 0; } AudioQueueBufferRef bufs[EL_AQ_NBUF]; int prepared = 0; for (int i = 0; i < EL_AQ_NBUF; i++) { if (AudioQueueAllocateBuffer(q, EL_AQ_FRAMES * sizeof(int16_t), &bufs[i]) != noErr) break; prepared++; } if (prepared == 0) { AudioQueueDispose(q, true); return 0; } /* Prime: fill what we can before starting, so playback begins immediately * rather than after the first underrun. */ for (int i = 0; i < prepared; i++) { int64_t remain = st.frames - st.pos; if (remain <= 0) break; int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; memcpy(bufs[i]->mAudioData, st.pcm + st.pos, (size_t)n * sizeof(int16_t)); bufs[i]->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); st.pos += n; if (AudioQueueEnqueueBuffer(q, bufs[i], 0, NULL) != noErr) break; st.inflight++; } if (st.inflight == 0) { AudioQueueDispose(q, true); return 0; } if (AudioQueueStart(q, NULL) != noErr) { AudioQueueDispose(q, true); return 0; } /* Bound the wait by the material's own duration plus a margin. A speaker * that wedges a program is worse than a speaker that gives up. */ double seconds = (double)frames / (double)sample_rate; int64_t max_us = (int64_t)((seconds + 5.0) * 1000000.0); int64_t waited = 0; const int64_t tick = 5000; /* 5 ms */ while (!st.drained && waited < max_us) { usleep((useconds_t)tick); waited += tick; } AudioQueueStop(q, true); AudioQueueDispose(q, true); return st.drained ? 1 : 0; } /* ── El entry points ──────────────────────────────────────────────────────── * Declared in el_runtime.h; see there for the El-facing contract. */ /* 1 when this build has a real speaker behind it. El code should ask before * speaking so the no-speaker case is a reported condition, not a silence that * looks like success. */ el_val_t speaker_available(void) { return (el_val_t)1; } el_val_t speaker_name(void) { return EL_STR("coreaudio-audioqueue"); } /* Play an El [Int] of 16-bit samples. Values are clamped, not wrapped: a * render that overshoots should distort at the rails the way real clipping * does, rather than invert phase and produce a sound nothing in the signal * chain intended. */ el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { int64_t n = (int64_t)el_list_len(samples); int32_t sr = (int32_t)sample_rate; if (n <= 0 || sr <= 0) return (el_val_t)0; int16_t* pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); if (!pcm) return (el_val_t)0; for (int64_t i = 0; i < n; i++) { int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); if (v > 32767) v = 32767; if (v < -32768) v = -32768; pcm[i] = (int16_t)v; } int ok = el_audio_play_raw(pcm, n, sr); free(pcm); return (el_val_t)(ok ? 1 : 0); } /* ── Asynchronous playback ─────────────────────────────────────────────────── * * converse needs this and a blocking play cannot give it. Barge-in means * stopping ON THE SPOT when the user starts talking — not at the end of the * current buffer, and certainly not at the end of the utterance. So the async * path keeps one queue alive, reports how far the hardware actually got, and * can be halted mid-buffer. * * `played_frames` is what makes an interrupted utterance resumable at the * sample rather than at the segment: it is the position the DAC reached, not * the position we enqueued to, and those differ by up to the full queue depth. * * One utterance at a time. A second async play stops the first — a mouth that * can say two things at once is not a feature. */ static AudioQueueRef g_aq = NULL; static ElAqState* g_aq_state = NULL; static int16_t* g_aq_pcm = NULL; static int32_t g_aq_sr = 0; static void el_audio_teardown(void) { if (g_aq) { AudioQueueStop(g_aq, true); AudioQueueDispose(g_aq, true); g_aq = NULL; } free(g_aq_pcm); g_aq_pcm = NULL; free(g_aq_state); g_aq_state = NULL; g_aq_sr = 0; } el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { el_audio_teardown(); int64_t n = (int64_t)el_list_len(samples); int32_t sr = (int32_t)sample_rate; if (n <= 0 || sr <= 0) return (el_val_t)0; g_aq_pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); if (!g_aq_pcm) return (el_val_t)0; for (int64_t i = 0; i < n; i++) { int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); if (v > 32767) v = 32767; if (v < -32768) v = -32768; g_aq_pcm[i] = (int16_t)v; } g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } g_aq_state->pcm = g_aq_pcm; g_aq_state->frames = n; g_aq_sr = sr; AudioStreamBasicDescription fmt; memset(&fmt, 0, sizeof(fmt)); fmt.mSampleRate = (Float64)sr; fmt.mFormatID = kAudioFormatLinearPCM; fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; fmt.mFramesPerPacket = 1; fmt.mChannelsPerFrame = 1; fmt.mBitsPerChannel = 16; fmt.mBytesPerFrame = 2; fmt.mBytesPerPacket = 2; if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { el_audio_teardown(); return (el_val_t)0; } for (int i = 0; i < EL_AQ_NBUF; i++) { int64_t remain = g_aq_state->frames - g_aq_state->pos; if (remain <= 0) break; AudioQueueBufferRef b = NULL; if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); g_aq_state->pos += k; if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; g_aq_state->inflight++; } if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } return (el_val_t)1; } el_val_t speaker_playing(void) { if (!g_aq || !g_aq_state) return (el_val_t)0; return (el_val_t)(g_aq_state->drained ? 0 : 1); } /* Frames the DAC has actually rendered. AudioQueueGetCurrentTime's mSampleTime * is relative to queue start, which is exactly the "where was I really" figure * a resumable utterance needs. Falls back to the enqueued position if the * timeline is unavailable (it is, briefly, right after start). */ el_val_t speaker_played_frames(void) { if (!g_aq || !g_aq_state) return (el_val_t)0; AudioTimeStamp ts; memset(&ts, 0, sizeof(ts)); Boolean discontinuity = false; if (AudioQueueGetCurrentTime(g_aq, NULL, &ts, &discontinuity) == noErr && (ts.mFlags & kAudioTimeStampSampleTimeValid)) { int64_t played = (int64_t)ts.mSampleTime; if (played < 0) played = 0; if (played > g_aq_state->frames) played = g_aq_state->frames; return (el_val_t)played; } return (el_val_t)g_aq_state->pos; } /* Pause where we are, keeping the queue and its position intact. * * This is the difference between barge-in and "finish the buffer". The moment * the microphone hears speech, output must stop AT THAT SAMPLE — a listener * experiences even 200ms of continued talking as being talked over. Pause * rather than stop because the interruption might turn out to be a backchannel * ("mm-hm"), and the right response to a backchannel is to carry on as though * nothing happened, which requires the queue to still be exactly where it was. * A stop-and-restart would re-attack the buffer and be audible as a stutter. */ el_val_t speaker_pause(void) { if (!g_aq) return (el_val_t)0; return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0); } el_val_t speaker_resume(void) { if (!g_aq) return (el_val_t)0; return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0); } el_val_t speaker_stop(void) { if (!g_aq) return (el_val_t)0; /* immediate: do NOT let the queue finish what it is holding */ AudioQueueStop(g_aq, true); el_audio_teardown(); return (el_val_t)1; } /* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer. * Returns frames, or 0 on any failure; *out is set only on success. Shared by * the blocking and async WAV paths. */ static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) { if (!path || !out) return 0; FILE* f = fopen(path, "rb"); if (!f) return 0; if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; } long size = ftell(f); if (size <= 44) { fclose(f); return 0; } rewind(f); unsigned char* d = (unsigned char*)malloc((size_t)size); if (!d) { fclose(f); return 0; } size_t got = fread(d, 1, (size_t)size, f); fclose(f); if (got != (size_t)size) { free(d); return 0; } if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; } int32_t sr = 0, channels = 0, bits = 0; long dataOff = -1, dataLen = 0, o = 12; /* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders * routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads * padding as audio. */ while (o + 8 <= size) { long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); if (sz < 0) break; if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { channels = (int32_t)(d[o+10] | (d[o+11] << 8)); sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); bits = (int32_t)(d[o+22] | (d[o+23] << 8)); } else if (memcmp(d + o, "data", 4) == 0) { dataOff = o + 8; dataLen = sz; if (dataOff + dataLen > size) dataLen = size - dataOff; } o += 8 + sz + (sz & 1); } if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; } long frames = dataLen / (2 * channels); int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); if (!pcm) { free(d); return 0; } for (long i = 0; i < frames; i++) { long b = dataOff + i * 2 * channels; pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); } free(d); *out = pcm; if (out_sr) *out_sr = sr; return (int64_t)frames; } /* Async WAV playback. converse speaks PRE-RENDERED segments and must keep * listening while it does, so it needs the file on the queue without blocking * and needs to be able to stop it mid-buffer. Going through the file rather * than an El [Int] also avoids marshalling a million-element list per segment * for audio the caller never intends to look at. */ el_val_t speaker_play_wav_async(el_val_t path) { const char* p = EL_CSTR(path); if (!p) return (el_val_t)0; el_audio_teardown(); int32_t sr = 0; int16_t* pcm = NULL; int64_t frames = el_wav_load(p, &pcm, &sr); if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; } g_aq_pcm = pcm; g_aq_sr = sr; g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } g_aq_state->pcm = g_aq_pcm; g_aq_state->frames = frames; AudioStreamBasicDescription fmt; memset(&fmt, 0, sizeof(fmt)); fmt.mSampleRate = (Float64)sr; fmt.mFormatID = kAudioFormatLinearPCM; fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; fmt.mFramesPerPacket = 1; fmt.mChannelsPerFrame = 1; fmt.mBitsPerChannel = 16; fmt.mBytesPerFrame = 2; fmt.mBytesPerPacket = 2; if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { el_audio_teardown(); return (el_val_t)0; } for (int i = 0; i < EL_AQ_NBUF; i++) { int64_t remain = g_aq_state->frames - g_aq_state->pos; if (remain <= 0) break; AudioQueueBufferRef b = NULL; if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); g_aq_state->pos += k; if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; g_aq_state->inflight++; } if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } return (el_val_t)1; } /* Total frames and sample rate of a WAV, without playing it — wav-info, and the * duration converse needs to compute progress through a segment. */ el_val_t wav_frames(el_val_t path) { const char* p = EL_CSTR(path); int16_t* pcm = NULL; int32_t sr = 0; int64_t n = el_wav_load(p, &pcm, &sr); free(pcm); return (el_val_t)n; } el_val_t wav_rate(el_val_t path) { const char* p = EL_CSTR(path); int16_t* pcm = NULL; int32_t sr = 0; int64_t n = el_wav_load(p, &pcm, &sr); free(pcm); return (el_val_t)(n > 0 ? sr : 0); } /* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows * how to write a WAV and a caller may reasonably want to hear one back without * re-rendering it; the parse is deliberately minimal and chunk-walking, so the * JUNK/FLLR padding that recorders emit does not defeat it. */ el_val_t speaker_play_wav(el_val_t path) { const char* p = EL_CSTR(path); if (!p) return (el_val_t)0; FILE* f = fopen(p, "rb"); if (!f) return (el_val_t)0; if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return (el_val_t)0; } long size = ftell(f); if (size <= 44) { fclose(f); return (el_val_t)0; } rewind(f); unsigned char* d = (unsigned char*)malloc((size_t)size); if (!d) { fclose(f); return (el_val_t)0; } size_t got = fread(d, 1, (size_t)size, f); fclose(f); if (got != (size_t)size) { free(d); return (el_val_t)0; } if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return (el_val_t)0; } int32_t sr = 0, channels = 0, bits = 0; long dataOff = -1, dataLen = 0; long o = 12; while (o + 8 <= size) { long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); if (sz < 0) break; if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { channels = (int32_t)(d[o+10] | (d[o+11] << 8)); sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); bits = (int32_t)(d[o+22] | (d[o+23] << 8)); } else if (memcmp(d + o, "data", 4) == 0) { dataOff = o + 8; dataLen = sz; if (dataOff + dataLen > size) dataLen = size - dataOff; } o += 8 + sz + (sz & 1); } if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return (el_val_t)0; } long frames = dataLen / (2 * channels); int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); if (!pcm) { free(d); return (el_val_t)0; } /* Take channel 0; the organ is mono by design and downmixing would be an * opinion about content this layer is not entitled to have. */ for (long i = 0; i < frames; i++) { long b = dataOff + i * 2 * channels; pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); } free(d); int ok = el_audio_play_raw(pcm, frames, sr); free(pcm); return (el_val_t)(ok ? 1 : 0); }