Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c26b6aac82 | |||
| 5503e1d9a4 |
@@ -0,0 +1,6 @@
|
||||
|
||||
# organ: local device state and its own engram store — never production's
|
||||
peripheral/.consent.json
|
||||
peripheral/.resume.json
|
||||
peripheral/.engram/
|
||||
peripheral/organ
|
||||
@@ -0,0 +1,525 @@
|
||||
/* el_audio_darwin.m — the SPEAKER realizer. El's native audio output on Darwin.
|
||||
*
|
||||
* WHY THIS FILE EXISTS.
|
||||
*
|
||||
* Neuron could already turn meaning into samples — the render path in
|
||||
* elp/src/speech.el superposes formant resonances over a glottal source and
|
||||
* produces PCM. What it could not do was make a sound. Every path from those
|
||||
* samples to the air ran outside the language: a 939-line Swift program
|
||||
* (peripheral/src/periph.swift) that shelled out to /usr/bin/afplay. So the
|
||||
* voice was not a capability of El or of Neuron. It was a separate binary
|
||||
* standing next to them, and "speak" meant "ask that binary to speak."
|
||||
*
|
||||
* A speaker is not a language feature the way a string is, but it is exactly
|
||||
* the kind of thing a runtime owns: a device. El already owns the filesystem,
|
||||
* the network, the clock, and a graph. It should own the one output device that
|
||||
* makes it audible. After this file, `speak` is an El operation.
|
||||
*
|
||||
* WHY IT IS A REALIZER AND NOT PURE EL.
|
||||
*
|
||||
* This is the boundary the whole design turns on. Everything ABOVE the sample
|
||||
* buffer is arithmetic and belongs in El: formant geometry, superposition,
|
||||
* envelopes, WAV framing, the voice signature. Everything in this file is the
|
||||
* part that cannot be arithmetic — handing a buffer to CoreAudio and waiting
|
||||
* for the hardware to drain it. There is no way to express "the DAC has now
|
||||
* played these samples" in El, and there should not be. So the split is: El
|
||||
* computes the sound, the realizer emits it, and the realizer is as thin as it
|
||||
* can possibly be — it makes no decisions about content, it has no opinion
|
||||
* about audio, and it cannot synthesize anything.
|
||||
*
|
||||
* The precedent is eg_cosine_batch_strategy_metal_hand.m: a platform-bound
|
||||
* capability compiled as its OWN translation unit, declared in el_runtime.h,
|
||||
* and linked in where the platform supports it. Deliberately NOT a patch to
|
||||
* el_runtime.c — adding a device to El must not mean editing the core runtime,
|
||||
* for the same reason adding a modality must not (see el_runtime.c's realizer
|
||||
* registry: a realizer is resolved by name, so new organs never touch the
|
||||
* middle of the language). el_audio_null.c is the same two entry points for
|
||||
* every platform that is not Darwin, so El code that speaks still links
|
||||
* everywhere and simply reports that it has no speaker.
|
||||
*
|
||||
* WHY AudioQueue AND NOT afplay.
|
||||
*
|
||||
* afplay is a process. Using it means the sound Neuron makes is a file it wrote
|
||||
* and asked something else to open — which forces every utterance through the
|
||||
* disk, cannot start until the whole utterance exists, and puts a fork/exec
|
||||
* between the intent to speak and the sound. AudioQueue takes the samples
|
||||
* directly out of memory. Nothing is written, nothing is spawned, and a caller
|
||||
* that wants to stream can push buffers as it renders them.
|
||||
*
|
||||
* AudioToolbox ships with macOS, so this stays own-core: no cloud, no library
|
||||
* to install, no model. The output is the local speaker and nothing leaves the
|
||||
* machine — there is no network path in this file at all, by construction.
|
||||
*/
|
||||
|
||||
#import <AudioToolbox/AudioToolbox.h>
|
||||
#import <Foundation/Foundation.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include "el_runtime.h"
|
||||
|
||||
/* Three buffers is the standard AudioQueue depth: one being played by the
|
||||
* hardware, one queued behind it, one being refilled. Fewer risks a gap on a
|
||||
* busy machine; more only adds latency before the first sound. */
|
||||
#define EL_AQ_NBUF 3
|
||||
#define EL_AQ_FRAMES 8192
|
||||
|
||||
typedef struct {
|
||||
const int16_t* pcm;
|
||||
int64_t frames;
|
||||
int64_t pos;
|
||||
volatile int inflight; /* buffers CoreAudio still owns */
|
||||
volatile int drained; /* set once the last buffer has been played */
|
||||
} ElAqState;
|
||||
|
||||
/* Called on an AudioQueue-internal thread each time a buffer finishes playing.
|
||||
* Refills and re-enqueues while samples remain; when the source is exhausted it
|
||||
* lets the buffer die and counts it out. `drained` flips only when the queue is
|
||||
* holding nothing, which is what makes the play call synchronous without
|
||||
* clipping the tail — the same reason periph.swift used .dataPlayedBack rather
|
||||
* than treating "consumed" as "heard". */
|
||||
static void el_aq_callback(void* userData, AudioQueueRef q, AudioQueueBufferRef buf) {
|
||||
ElAqState* st = (ElAqState*)userData;
|
||||
int64_t remain = st->frames - st->pos;
|
||||
if (remain <= 0) {
|
||||
if (--st->inflight <= 0) st->drained = 1;
|
||||
return;
|
||||
}
|
||||
int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
|
||||
memcpy(buf->mAudioData, st->pcm + st->pos, (size_t)n * sizeof(int16_t));
|
||||
buf->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t));
|
||||
st->pos += n;
|
||||
if (AudioQueueEnqueueBuffer(q, buf, 0, NULL) != noErr) {
|
||||
if (--st->inflight <= 0) st->drained = 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Play a 16-bit mono PCM buffer out the default output device, blocking until
|
||||
* the hardware has actually finished. Returns 1 on success, 0 on any failure —
|
||||
* never throws, never hangs indefinitely. */
|
||||
static int el_audio_play_raw(const int16_t* pcm, int64_t frames, int32_t sample_rate) {
|
||||
if (!pcm || frames <= 0 || sample_rate <= 0) return 0;
|
||||
|
||||
AudioStreamBasicDescription fmt;
|
||||
memset(&fmt, 0, sizeof(fmt));
|
||||
fmt.mSampleRate = (Float64)sample_rate;
|
||||
fmt.mFormatID = kAudioFormatLinearPCM;
|
||||
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
|
||||
fmt.mFramesPerPacket = 1;
|
||||
fmt.mChannelsPerFrame = 1;
|
||||
fmt.mBitsPerChannel = 16;
|
||||
fmt.mBytesPerFrame = 2;
|
||||
fmt.mBytesPerPacket = 2;
|
||||
|
||||
ElAqState st;
|
||||
memset(&st, 0, sizeof(st));
|
||||
st.pcm = pcm;
|
||||
st.frames = frames;
|
||||
|
||||
AudioQueueRef q = NULL;
|
||||
/* NULL run loop => callbacks arrive on an AudioQueue-internal thread, so
|
||||
* this function can simply wait rather than having to pump a run loop it
|
||||
* does not own. El programs are not required to have one. */
|
||||
if (AudioQueueNewOutput(&fmt, el_aq_callback, &st, NULL, NULL, 0, &q) != noErr || !q) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
AudioQueueBufferRef bufs[EL_AQ_NBUF];
|
||||
int prepared = 0;
|
||||
for (int i = 0; i < EL_AQ_NBUF; i++) {
|
||||
if (AudioQueueAllocateBuffer(q, EL_AQ_FRAMES * sizeof(int16_t), &bufs[i]) != noErr) break;
|
||||
prepared++;
|
||||
}
|
||||
if (prepared == 0) { AudioQueueDispose(q, true); return 0; }
|
||||
|
||||
/* Prime: fill what we can before starting, so playback begins immediately
|
||||
* rather than after the first underrun. */
|
||||
for (int i = 0; i < prepared; i++) {
|
||||
int64_t remain = st.frames - st.pos;
|
||||
if (remain <= 0) break;
|
||||
int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
|
||||
memcpy(bufs[i]->mAudioData, st.pcm + st.pos, (size_t)n * sizeof(int16_t));
|
||||
bufs[i]->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t));
|
||||
st.pos += n;
|
||||
if (AudioQueueEnqueueBuffer(q, bufs[i], 0, NULL) != noErr) break;
|
||||
st.inflight++;
|
||||
}
|
||||
if (st.inflight == 0) { AudioQueueDispose(q, true); return 0; }
|
||||
|
||||
if (AudioQueueStart(q, NULL) != noErr) { AudioQueueDispose(q, true); return 0; }
|
||||
|
||||
/* Bound the wait by the material's own duration plus a margin. A speaker
|
||||
* that wedges a program is worse than a speaker that gives up. */
|
||||
double seconds = (double)frames / (double)sample_rate;
|
||||
int64_t max_us = (int64_t)((seconds + 5.0) * 1000000.0);
|
||||
int64_t waited = 0;
|
||||
const int64_t tick = 5000; /* 5 ms */
|
||||
while (!st.drained && waited < max_us) {
|
||||
usleep((useconds_t)tick);
|
||||
waited += tick;
|
||||
}
|
||||
|
||||
AudioQueueStop(q, true);
|
||||
AudioQueueDispose(q, true);
|
||||
return st.drained ? 1 : 0;
|
||||
}
|
||||
|
||||
/* ── El entry points ────────────────────────────────────────────────────────
|
||||
* Declared in el_runtime.h; see there for the El-facing contract. */
|
||||
|
||||
/* 1 when this build has a real speaker behind it. El code should ask before
|
||||
* speaking so the no-speaker case is a reported condition, not a silence that
|
||||
* looks like success. */
|
||||
el_val_t speaker_available(void) {
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
el_val_t speaker_name(void) {
|
||||
return EL_STR("coreaudio-audioqueue");
|
||||
}
|
||||
|
||||
/* Play an El [Int] of 16-bit samples. Values are clamped, not wrapped: a
|
||||
* render that overshoots should distort at the rails the way real clipping
|
||||
* does, rather than invert phase and produce a sound nothing in the signal
|
||||
* chain intended. */
|
||||
el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) {
|
||||
int64_t n = (int64_t)el_list_len(samples);
|
||||
int32_t sr = (int32_t)sample_rate;
|
||||
if (n <= 0 || sr <= 0) return (el_val_t)0;
|
||||
|
||||
int16_t* pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t));
|
||||
if (!pcm) return (el_val_t)0;
|
||||
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
int64_t v = (int64_t)el_list_get(samples, (el_val_t)i);
|
||||
if (v > 32767) v = 32767;
|
||||
if (v < -32768) v = -32768;
|
||||
pcm[i] = (int16_t)v;
|
||||
}
|
||||
|
||||
int ok = el_audio_play_raw(pcm, n, sr);
|
||||
free(pcm);
|
||||
return (el_val_t)(ok ? 1 : 0);
|
||||
}
|
||||
|
||||
/* ── Asynchronous playback ───────────────────────────────────────────────────
|
||||
*
|
||||
* converse needs this and a blocking play cannot give it. Barge-in means
|
||||
* stopping ON THE SPOT when the user starts talking — not at the end of the
|
||||
* current buffer, and certainly not at the end of the utterance. So the async
|
||||
* path keeps one queue alive, reports how far the hardware actually got, and
|
||||
* can be halted mid-buffer.
|
||||
*
|
||||
* `played_frames` is what makes an interrupted utterance resumable at the
|
||||
* sample rather than at the segment: it is the position the DAC reached, not
|
||||
* the position we enqueued to, and those differ by up to the full queue depth.
|
||||
*
|
||||
* One utterance at a time. A second async play stops the first — a mouth that
|
||||
* can say two things at once is not a feature. */
|
||||
|
||||
static AudioQueueRef g_aq = NULL;
|
||||
static ElAqState* g_aq_state = NULL;
|
||||
static int16_t* g_aq_pcm = NULL;
|
||||
static int32_t g_aq_sr = 0;
|
||||
|
||||
static void el_audio_teardown(void) {
|
||||
if (g_aq) {
|
||||
AudioQueueStop(g_aq, true);
|
||||
AudioQueueDispose(g_aq, true);
|
||||
g_aq = NULL;
|
||||
}
|
||||
free(g_aq_pcm); g_aq_pcm = NULL;
|
||||
free(g_aq_state); g_aq_state = NULL;
|
||||
g_aq_sr = 0;
|
||||
}
|
||||
|
||||
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) {
|
||||
el_audio_teardown();
|
||||
|
||||
int64_t n = (int64_t)el_list_len(samples);
|
||||
int32_t sr = (int32_t)sample_rate;
|
||||
if (n <= 0 || sr <= 0) return (el_val_t)0;
|
||||
|
||||
g_aq_pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t));
|
||||
if (!g_aq_pcm) return (el_val_t)0;
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
int64_t v = (int64_t)el_list_get(samples, (el_val_t)i);
|
||||
if (v > 32767) v = 32767;
|
||||
if (v < -32768) v = -32768;
|
||||
g_aq_pcm[i] = (int16_t)v;
|
||||
}
|
||||
|
||||
g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState));
|
||||
if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; }
|
||||
g_aq_state->pcm = g_aq_pcm;
|
||||
g_aq_state->frames = n;
|
||||
g_aq_sr = sr;
|
||||
|
||||
AudioStreamBasicDescription fmt;
|
||||
memset(&fmt, 0, sizeof(fmt));
|
||||
fmt.mSampleRate = (Float64)sr;
|
||||
fmt.mFormatID = kAudioFormatLinearPCM;
|
||||
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
|
||||
fmt.mFramesPerPacket = 1;
|
||||
fmt.mChannelsPerFrame = 1;
|
||||
fmt.mBitsPerChannel = 16;
|
||||
fmt.mBytesPerFrame = 2;
|
||||
fmt.mBytesPerPacket = 2;
|
||||
|
||||
if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) {
|
||||
el_audio_teardown();
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
for (int i = 0; i < EL_AQ_NBUF; i++) {
|
||||
int64_t remain = g_aq_state->frames - g_aq_state->pos;
|
||||
if (remain <= 0) break;
|
||||
AudioQueueBufferRef b = NULL;
|
||||
if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break;
|
||||
int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
|
||||
memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t));
|
||||
b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t));
|
||||
g_aq_state->pos += k;
|
||||
if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break;
|
||||
g_aq_state->inflight++;
|
||||
}
|
||||
if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; }
|
||||
|
||||
if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; }
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
el_val_t speaker_playing(void) {
|
||||
if (!g_aq || !g_aq_state) return (el_val_t)0;
|
||||
return (el_val_t)(g_aq_state->drained ? 0 : 1);
|
||||
}
|
||||
|
||||
/* Frames the DAC has actually rendered. AudioQueueGetCurrentTime's mSampleTime
|
||||
* is relative to queue start, which is exactly the "where was I really" figure
|
||||
* a resumable utterance needs. Falls back to the enqueued position if the
|
||||
* timeline is unavailable (it is, briefly, right after start). */
|
||||
el_val_t speaker_played_frames(void) {
|
||||
if (!g_aq || !g_aq_state) return (el_val_t)0;
|
||||
AudioTimeStamp ts;
|
||||
memset(&ts, 0, sizeof(ts));
|
||||
Boolean discontinuity = false;
|
||||
if (AudioQueueGetCurrentTime(g_aq, NULL, &ts, &discontinuity) == noErr &&
|
||||
(ts.mFlags & kAudioTimeStampSampleTimeValid)) {
|
||||
int64_t played = (int64_t)ts.mSampleTime;
|
||||
if (played < 0) played = 0;
|
||||
if (played > g_aq_state->frames) played = g_aq_state->frames;
|
||||
return (el_val_t)played;
|
||||
}
|
||||
return (el_val_t)g_aq_state->pos;
|
||||
}
|
||||
|
||||
/* Pause where we are, keeping the queue and its position intact.
|
||||
*
|
||||
* This is the difference between barge-in and "finish the buffer". The moment
|
||||
* the microphone hears speech, output must stop AT THAT SAMPLE — a listener
|
||||
* experiences even 200ms of continued talking as being talked over. Pause
|
||||
* rather than stop because the interruption might turn out to be a backchannel
|
||||
* ("mm-hm"), and the right response to a backchannel is to carry on as though
|
||||
* nothing happened, which requires the queue to still be exactly where it was.
|
||||
* A stop-and-restart would re-attack the buffer and be audible as a stutter. */
|
||||
el_val_t speaker_pause(void) {
|
||||
if (!g_aq) return (el_val_t)0;
|
||||
return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0);
|
||||
}
|
||||
|
||||
el_val_t speaker_resume(void) {
|
||||
if (!g_aq) return (el_val_t)0;
|
||||
return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0);
|
||||
}
|
||||
|
||||
el_val_t speaker_stop(void) {
|
||||
if (!g_aq) return (el_val_t)0;
|
||||
/* immediate: do NOT let the queue finish what it is holding */
|
||||
AudioQueueStop(g_aq, true);
|
||||
el_audio_teardown();
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
/* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer.
|
||||
* Returns frames, or 0 on any failure; *out is set only on success. Shared by
|
||||
* the blocking and async WAV paths. */
|
||||
static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) {
|
||||
if (!path || !out) return 0;
|
||||
FILE* f = fopen(path, "rb");
|
||||
if (!f) return 0;
|
||||
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; }
|
||||
long size = ftell(f);
|
||||
if (size <= 44) { fclose(f); return 0; }
|
||||
rewind(f);
|
||||
unsigned char* d = (unsigned char*)malloc((size_t)size);
|
||||
if (!d) { fclose(f); return 0; }
|
||||
size_t got = fread(d, 1, (size_t)size, f);
|
||||
fclose(f);
|
||||
if (got != (size_t)size) { free(d); return 0; }
|
||||
if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; }
|
||||
|
||||
int32_t sr = 0, channels = 0, bits = 0;
|
||||
long dataOff = -1, dataLen = 0, o = 12;
|
||||
/* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders
|
||||
* routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads
|
||||
* padding as audio. */
|
||||
while (o + 8 <= size) {
|
||||
long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24);
|
||||
if (sz < 0) break;
|
||||
if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) {
|
||||
channels = (int32_t)(d[o+10] | (d[o+11] << 8));
|
||||
sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24));
|
||||
bits = (int32_t)(d[o+22] | (d[o+23] << 8));
|
||||
} else if (memcmp(d + o, "data", 4) == 0) {
|
||||
dataOff = o + 8;
|
||||
dataLen = sz;
|
||||
if (dataOff + dataLen > size) dataLen = size - dataOff;
|
||||
}
|
||||
o += 8 + sz + (sz & 1);
|
||||
}
|
||||
if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; }
|
||||
|
||||
long frames = dataLen / (2 * channels);
|
||||
int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t));
|
||||
if (!pcm) { free(d); return 0; }
|
||||
for (long i = 0; i < frames; i++) {
|
||||
long b = dataOff + i * 2 * channels;
|
||||
pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8));
|
||||
}
|
||||
free(d);
|
||||
*out = pcm;
|
||||
if (out_sr) *out_sr = sr;
|
||||
return (int64_t)frames;
|
||||
}
|
||||
|
||||
/* Async WAV playback. converse speaks PRE-RENDERED segments and must keep
|
||||
* listening while it does, so it needs the file on the queue without blocking
|
||||
* and needs to be able to stop it mid-buffer. Going through the file rather
|
||||
* than an El [Int] also avoids marshalling a million-element list per segment
|
||||
* for audio the caller never intends to look at. */
|
||||
el_val_t speaker_play_wav_async(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
if (!p) return (el_val_t)0;
|
||||
|
||||
el_audio_teardown();
|
||||
|
||||
int32_t sr = 0;
|
||||
int16_t* pcm = NULL;
|
||||
int64_t frames = el_wav_load(p, &pcm, &sr);
|
||||
if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; }
|
||||
|
||||
g_aq_pcm = pcm;
|
||||
g_aq_sr = sr;
|
||||
g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState));
|
||||
if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; }
|
||||
g_aq_state->pcm = g_aq_pcm;
|
||||
g_aq_state->frames = frames;
|
||||
|
||||
AudioStreamBasicDescription fmt;
|
||||
memset(&fmt, 0, sizeof(fmt));
|
||||
fmt.mSampleRate = (Float64)sr;
|
||||
fmt.mFormatID = kAudioFormatLinearPCM;
|
||||
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
|
||||
fmt.mFramesPerPacket = 1;
|
||||
fmt.mChannelsPerFrame = 1;
|
||||
fmt.mBitsPerChannel = 16;
|
||||
fmt.mBytesPerFrame = 2;
|
||||
fmt.mBytesPerPacket = 2;
|
||||
|
||||
if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) {
|
||||
el_audio_teardown();
|
||||
return (el_val_t)0;
|
||||
}
|
||||
for (int i = 0; i < EL_AQ_NBUF; i++) {
|
||||
int64_t remain = g_aq_state->frames - g_aq_state->pos;
|
||||
if (remain <= 0) break;
|
||||
AudioQueueBufferRef b = NULL;
|
||||
if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break;
|
||||
int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
|
||||
memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t));
|
||||
b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t));
|
||||
g_aq_state->pos += k;
|
||||
if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break;
|
||||
g_aq_state->inflight++;
|
||||
}
|
||||
if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; }
|
||||
if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; }
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
/* Total frames and sample rate of a WAV, without playing it — wav-info, and the
|
||||
* duration converse needs to compute progress through a segment. */
|
||||
el_val_t wav_frames(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
int16_t* pcm = NULL; int32_t sr = 0;
|
||||
int64_t n = el_wav_load(p, &pcm, &sr);
|
||||
free(pcm);
|
||||
return (el_val_t)n;
|
||||
}
|
||||
|
||||
el_val_t wav_rate(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
int16_t* pcm = NULL; int32_t sr = 0;
|
||||
int64_t n = el_wav_load(p, &pcm, &sr);
|
||||
free(pcm);
|
||||
return (el_val_t)(n > 0 ? sr : 0);
|
||||
}
|
||||
|
||||
/* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows
|
||||
* how to write a WAV and a caller may reasonably want to hear one back without
|
||||
* re-rendering it; the parse is deliberately minimal and chunk-walking, so the
|
||||
* JUNK/FLLR padding that recorders emit does not defeat it. */
|
||||
el_val_t speaker_play_wav(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
if (!p) return (el_val_t)0;
|
||||
FILE* f = fopen(p, "rb");
|
||||
if (!f) return (el_val_t)0;
|
||||
|
||||
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return (el_val_t)0; }
|
||||
long size = ftell(f);
|
||||
if (size <= 44) { fclose(f); return (el_val_t)0; }
|
||||
rewind(f);
|
||||
|
||||
unsigned char* d = (unsigned char*)malloc((size_t)size);
|
||||
if (!d) { fclose(f); return (el_val_t)0; }
|
||||
size_t got = fread(d, 1, (size_t)size, f);
|
||||
fclose(f);
|
||||
if (got != (size_t)size) { free(d); return (el_val_t)0; }
|
||||
|
||||
if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return (el_val_t)0; }
|
||||
|
||||
int32_t sr = 0, channels = 0, bits = 0;
|
||||
long dataOff = -1, dataLen = 0;
|
||||
long o = 12;
|
||||
while (o + 8 <= size) {
|
||||
long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24);
|
||||
if (sz < 0) break;
|
||||
if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) {
|
||||
channels = (int32_t)(d[o+10] | (d[o+11] << 8));
|
||||
sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24));
|
||||
bits = (int32_t)(d[o+22] | (d[o+23] << 8));
|
||||
} else if (memcmp(d + o, "data", 4) == 0) {
|
||||
dataOff = o + 8;
|
||||
dataLen = sz;
|
||||
if (dataOff + dataLen > size) dataLen = size - dataOff;
|
||||
}
|
||||
o += 8 + sz + (sz & 1);
|
||||
}
|
||||
if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return (el_val_t)0; }
|
||||
|
||||
long frames = dataLen / (2 * channels);
|
||||
int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t));
|
||||
if (!pcm) { free(d); return (el_val_t)0; }
|
||||
/* Take channel 0; the organ is mono by design and downmixing would be an
|
||||
* opinion about content this layer is not entitled to have. */
|
||||
for (long i = 0; i < frames; i++) {
|
||||
long b = dataOff + i * 2 * channels;
|
||||
pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8));
|
||||
}
|
||||
free(d);
|
||||
|
||||
int ok = el_audio_play_raw(pcm, frames, sr);
|
||||
free(pcm);
|
||||
return (el_val_t)(ok ? 1 : 0);
|
||||
}
|
||||
@@ -0,0 +1,841 @@
|
||||
/* el_capture_darwin.m — the MICROPHONE and CAMERA realizers. El's afferent
|
||||
* organ on Darwin: the two entry points through which the world gets in.
|
||||
*
|
||||
* WHY THIS FILE EXISTS, AND WHY IT IS A REALIZER RATHER THAN PURE EL.
|
||||
*
|
||||
* el_audio_darwin.m argued the efferent half of this: El can compute a sound
|
||||
* but it cannot make one, because "the DAC has now played these samples" is not
|
||||
* a fact any amount of arithmetic can produce. This file is the same argument
|
||||
* run backwards. El can compute *about* a sound — it can window it, take its
|
||||
* autocorrelation, run Levinson-Durbin over that, find the formant peaks in the
|
||||
* resulting all-pole envelope, and hand back a voiceprint — but it cannot ASK.
|
||||
* There is no expression in El, and there must not be, whose value is "the next
|
||||
* 1024 frames the microphone hears" or "what the camera is pointed at right
|
||||
* now." Those are not computed; they are *requested*, from an operating system
|
||||
* that owns the device, mediates consent for it, and delivers the answer on a
|
||||
* thread of its choosing whenever it feels like it. Asking is the one primitive
|
||||
* operation here. Everything else in this file is bookkeeping around the ask.
|
||||
*
|
||||
* So the line is drawn exactly where el_audio_darwin.m drew it, at the sample
|
||||
* buffer, and it is drawn on purpose:
|
||||
*
|
||||
* BELOW the line (here): open the device, honour the OS permission gate,
|
||||
* install a tap or a frame delegate, convert whatever the hardware happens to
|
||||
* emit into the one shape El asked for, and hand it up. No opinions about
|
||||
* content. No analysis. No decisions.
|
||||
*
|
||||
* ABOVE the line (El): energy, zero-crossing rate, spectral centroid, F0 by
|
||||
* autocorrelation, LPC, formants F1-F5, the compact descriptors, the
|
||||
* scene-geometry grid, the yield-or-hold turn-taking decision. All of it is
|
||||
* arithmetic over a buffer, all of it belongs in El, and none of it appears
|
||||
* below. The reference this file ports — peripheral/src/periph.swift — held
|
||||
* both halves, and that was the problem worth fixing: the descriptors were
|
||||
* trapped in a 939-line binary standing next to the language instead of being
|
||||
* written in it. Porting the *whole* of periph.swift down here would have
|
||||
* reproduced that mistake in C. Only the ask came down.
|
||||
*
|
||||
* The precedent for the file's SHAPE is eg_cosine_batch_strategy_metal_hand.m:
|
||||
* a platform-bound capability compiled as its own translation unit, declared in
|
||||
* el_runtime.h, linked in where the platform supports it, and deliberately NOT
|
||||
* a patch to the middle of el_runtime.c. Acquiring a device must not mean
|
||||
* editing the language, for the same reason acquiring a modality must not (see
|
||||
* the realizer registry: organs are resolved by name). el_peripheral_null.c is
|
||||
* the same entry points everywhere else, so El code that listens still links on
|
||||
* every platform and merely reports having no ear.
|
||||
*
|
||||
* FAIL CLOSED, ALWAYS.
|
||||
*
|
||||
* A capture path that returns plausible-looking zeros when it was denied is
|
||||
* worse than one that returns nothing, because the caller cannot tell the
|
||||
* difference between a silent room and a refused microphone. Every entry point
|
||||
* here checks AVCaptureDevice's authorization status BEFORE touching hardware
|
||||
* and returns the empty value — an empty list, a 0 map — on anything short of
|
||||
* .authorized. mic_available() and camera_available() report that state WITHOUT
|
||||
* prompting, so El can ask "may I?" without the act of asking being a prompt.
|
||||
*
|
||||
* NEVER HANG.
|
||||
*
|
||||
* Every wait in this file is bounded: 30s on a permission prompt (the user has
|
||||
* to walk to the dialog), seconds+5 on a capture of `seconds`, 10s on a camera
|
||||
* frame. An organ that wedges the program holding it is not an organ, it is a
|
||||
* fault. Every path also tears the device down on the way out, including the
|
||||
* failure paths, so a timed-out capture does not leave the mic light on.
|
||||
*
|
||||
* OWN-CORE AND LOCAL BY CONSTRUCTION.
|
||||
*
|
||||
* AVFoundation, CoreVideo, CoreGraphics and ImageIO ship with macOS. There is
|
||||
* no third-party library here, no model, and — the part that matters — no
|
||||
* network path of any kind. Samples and pixels move from local hardware into an
|
||||
* El value and stop. periph.swift had a URLSession in it; this file has no
|
||||
* socket, no URL, and nothing that could grow one without being obvious in
|
||||
* review. Consent is enforced above this layer in El and below it by the OS;
|
||||
* this layer's whole contribution to that is refusing to proceed.
|
||||
*
|
||||
* DISCLOSURE.
|
||||
*
|
||||
* Every actual device touch writes one line to stderr and flushes it, before
|
||||
* the device opens. stderr and not stdout: a program that announces "I am about
|
||||
* to open the microphone" on stdout has corrupted its own output, and the
|
||||
* caller must be able to separate the answer from how it was obtained. One line
|
||||
* per touch, no more — a disclosure rail that spams is a rail people learn to
|
||||
* ignore.
|
||||
*/
|
||||
|
||||
#if defined(__APPLE__)
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
#import <CoreGraphics/CoreGraphics.h>
|
||||
#import <ImageIO/ImageIO.h>
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <pthread.h>
|
||||
|
||||
#include "el_runtime.h"
|
||||
|
||||
/* ── Disclosure ──────────────────────────────────────────────────────────────
|
||||
* One flushed line per real device touch, on stderr. Flushed rather than
|
||||
* buffered so the line reaches the terminal BEFORE the mic light comes on
|
||||
* rather than whenever the buffer happens to drain. */
|
||||
static void el_cap_disclose(const char* what) {
|
||||
fprintf(stderr, " [peripheral] %s\n", what);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
/* ── Permission ──────────────────────────────────────────────────────────────
|
||||
* authorizationStatus is a pure read of the TCC database: it never prompts and
|
||||
* never blocks, which is what lets mic_available()/camera_available() answer
|
||||
* honestly without the question itself becoming an event. requestAccess DOES
|
||||
* prompt, so it lives behind its own entry point and nothing calls it
|
||||
* implicitly. */
|
||||
|
||||
static int el_cap_authorized(AVMediaType media) {
|
||||
@try {
|
||||
return [AVCaptureDevice authorizationStatusForMediaType:media]
|
||||
== AVAuthorizationStatusAuthorized ? 1 : 0;
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int el_cap_device_present(AVMediaType media) {
|
||||
@try {
|
||||
return [AVCaptureDevice defaultDeviceWithMediaType:media] != nil ? 1 : 0;
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Prompt once and wait, bounded. 30 seconds is the same budget periph.swift
|
||||
* used: long enough for a human to notice a dialog and decide, short enough
|
||||
* that an unattended run fails rather than parks forever. A timeout is reported
|
||||
* as "not granted", which is the safe reading — we genuinely do not know that
|
||||
* it was. */
|
||||
static int el_cap_request(AVMediaType media) {
|
||||
__block int granted = 0;
|
||||
dispatch_semaphore_t sem = dispatch_semaphore_create(0);
|
||||
@try {
|
||||
[AVCaptureDevice requestAccessForMediaType:media
|
||||
completionHandler:^(BOOL ok) {
|
||||
granted = ok ? 1 : 0;
|
||||
dispatch_semaphore_signal(sem);
|
||||
}];
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
return 0;
|
||||
}
|
||||
if (dispatch_semaphore_wait(sem,
|
||||
dispatch_time(DISPATCH_TIME_NOW, (int64_t)(30 * NSEC_PER_SEC))) != 0) {
|
||||
return 0; /* timed out — treat as refused */
|
||||
}
|
||||
return granted;
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* MICROPHONE — one-shot capture
|
||||
* ══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* The sink the tap block writes into. An object rather than a static so two
|
||||
* captures can never share state, and so ARC keeps it alive for exactly as long
|
||||
* as the block that captured it. The lock is real, not decorative: the tap runs
|
||||
* on an AVAudioEngine-internal thread and the waiter runs on the caller's. */
|
||||
@interface ElCapMicSink : NSObject
|
||||
@property (nonatomic, strong) NSMutableData* pcm;
|
||||
@property (nonatomic, strong) NSLock* lock;
|
||||
@end
|
||||
|
||||
@implementation ElCapMicSink
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_pcm = [NSMutableData data];
|
||||
_lock = [[NSLock alloc] init];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
@end
|
||||
|
||||
/* Capture `seconds` of mono 16-bit PCM at `sample_rate`.
|
||||
*
|
||||
* The hardware format is NOT assumed. A built-in mic will typically hand back
|
||||
* float32 at 44.1 or 48 kHz, an aggregate device may be 8 channels at 96 kHz,
|
||||
* and a caller asking for 16 kHz mono (which is what the formant path wants)
|
||||
* gets 16 kHz mono either way. AVAudioConverter does the rate conversion and
|
||||
* the downmix; doing it by hand would mean writing a resampler in the one file
|
||||
* that is supposed to contain no arithmetic.
|
||||
*
|
||||
* The converter is built ONCE, outside the tap, because a sample-rate converter
|
||||
* carries filter state across buffers — rebuilding it per callback would put a
|
||||
* discontinuity at every buffer boundary, which is audible and which would then
|
||||
* show up in El's spectral descriptors as energy that was never in the room. */
|
||||
el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) {
|
||||
el_val_t empty = el_list_empty();
|
||||
|
||||
if (!el_cap_authorized(AVMediaTypeAudio)) return empty;
|
||||
|
||||
int64_t secs = (int64_t)seconds;
|
||||
int64_t sr = (int64_t)sample_rate;
|
||||
if (secs <= 0 || sr <= 0) return empty;
|
||||
/* Bound the ask. A caller that asks for a year of audio has made a mistake,
|
||||
* and honouring it would mean an unkillable capture and an OOM. */
|
||||
if (secs > 300) secs = 300;
|
||||
if (sr > 384000) sr = 384000;
|
||||
|
||||
__block AVAudioEngine* engine = nil;
|
||||
AVAudioInputNode* input = nil;
|
||||
AVAudioFormat* hwFmt = nil;
|
||||
int tapped = 0;
|
||||
|
||||
@try {
|
||||
engine = [[AVAudioEngine alloc] init];
|
||||
input = [engine inputNode];
|
||||
hwFmt = [input inputFormatForBus:0];
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
return empty;
|
||||
}
|
||||
if (!input || !hwFmt || hwFmt.sampleRate <= 0 || hwFmt.channelCount == 0) {
|
||||
return empty;
|
||||
}
|
||||
|
||||
/* Preferred target: mono int16 at the requested rate. If the converter
|
||||
* refuses that pairing (some exotic input layouts will not downmix), fall
|
||||
* back to keeping the hardware's channel count and taking channel 0 on the
|
||||
* way out — the organ is mono by design and inventing a downmix here would
|
||||
* be an opinion about content this layer is not entitled to have. */
|
||||
AVAudioFormat* outFmt =
|
||||
[[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
|
||||
sampleRate:(double)sr
|
||||
channels:1
|
||||
interleaved:YES];
|
||||
AVAudioConverter* conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt
|
||||
toFormat:outFmt] : nil;
|
||||
AVAudioChannelCount outCh = 1;
|
||||
if (!conv) {
|
||||
outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
|
||||
sampleRate:(double)sr
|
||||
channels:hwFmt.channelCount
|
||||
interleaved:YES];
|
||||
conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil;
|
||||
outCh = hwFmt.channelCount;
|
||||
}
|
||||
if (!conv || !outFmt) return empty;
|
||||
|
||||
ElCapMicSink* sink = [[ElCapMicSink alloc] init];
|
||||
const int64_t want = secs * sr; /* frames we are waiting for */
|
||||
|
||||
{
|
||||
char msg[192];
|
||||
snprintf(msg, sizeof(msg),
|
||||
"MIC: opening the microphone for %llds -> %lld Hz mono PCM "
|
||||
"(local, never egresses).", (long long)secs, (long long)sr);
|
||||
el_cap_disclose(msg);
|
||||
}
|
||||
|
||||
const double ratio = (double)sr / hwFmt.sampleRate;
|
||||
|
||||
@try {
|
||||
[input installTapOnBus:0
|
||||
bufferSize:4096
|
||||
format:hwFmt
|
||||
block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) {
|
||||
(void)when;
|
||||
if (!buf || buf.frameLength == 0) return;
|
||||
|
||||
AVAudioFrameCount cap =
|
||||
(AVAudioFrameCount)((double)buf.frameLength * ratio) + 1024;
|
||||
AVAudioPCMBuffer* out =
|
||||
[[AVAudioPCMBuffer alloc] initWithPCMFormat:outFmt frameCapacity:cap];
|
||||
if (!out) return;
|
||||
|
||||
__block BOOL fed = NO;
|
||||
AVAudioConverterInputBlock feed =
|
||||
^AVAudioBuffer* _Nullable (AVAudioPacketCount need,
|
||||
AVAudioConverterInputStatus* _Nonnull status) {
|
||||
(void)need;
|
||||
if (fed) { *status = AVAudioConverterInputStatus_NoDataNow; return nil; }
|
||||
fed = YES;
|
||||
*status = AVAudioConverterInputStatus_HaveData;
|
||||
return buf;
|
||||
};
|
||||
|
||||
NSError* err = nil;
|
||||
AVAudioConverterOutputStatus st =
|
||||
[conv convertToBuffer:out error:&err withInputFromBlock:feed];
|
||||
if (st == AVAudioConverterOutputStatus_Error || out.frameLength == 0) return;
|
||||
|
||||
const int16_t* src = out.int16ChannelData ? out.int16ChannelData[0] : NULL;
|
||||
if (!src) return;
|
||||
|
||||
NSUInteger n = (NSUInteger)out.frameLength;
|
||||
[sink.lock lock];
|
||||
if (outCh == 1) {
|
||||
[sink.pcm appendBytes:src length:n * sizeof(int16_t)];
|
||||
} else {
|
||||
/* Interleaved: stride to channel 0. */
|
||||
for (NSUInteger i = 0; i < n; i++) {
|
||||
int16_t v = src[i * outCh];
|
||||
[sink.pcm appendBytes:&v length:sizeof(int16_t)];
|
||||
}
|
||||
}
|
||||
[sink.lock unlock];
|
||||
}];
|
||||
tapped = 1;
|
||||
|
||||
[engine prepare];
|
||||
NSError* startErr = nil;
|
||||
if (![engine startAndReturnError:&startErr]) {
|
||||
[input removeTapOnBus:0];
|
||||
return empty;
|
||||
}
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
@try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; }
|
||||
@try { [engine stop]; } @catch (NSException* e2) { (void)e2; }
|
||||
return empty;
|
||||
}
|
||||
|
||||
/* Wait for `want` frames, bounded by the material's own duration plus a
|
||||
* margin. A device that stops producing must not become a hang. */
|
||||
const int64_t deadline_us = (secs + 5) * 1000000;
|
||||
int64_t waited_us = 0;
|
||||
const int64_t tick_us = 5000;
|
||||
for (;;) {
|
||||
[sink.lock lock];
|
||||
int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t));
|
||||
[sink.lock unlock];
|
||||
if (have >= want || waited_us >= deadline_us) break;
|
||||
usleep((useconds_t)tick_us);
|
||||
waited_us += tick_us;
|
||||
}
|
||||
|
||||
@try { [input removeTapOnBus:0]; } @catch (NSException* e) { (void)e; }
|
||||
@try { [engine stop]; } @catch (NSException* e) { (void)e; }
|
||||
engine = nil;
|
||||
|
||||
/* Hand up exactly what was asked for, or everything we got if the device
|
||||
* came up short. Never padded: silence we invented is indistinguishable
|
||||
* from silence we heard, and El has no way to tell them apart afterwards. */
|
||||
[sink.lock lock];
|
||||
int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t));
|
||||
int64_t n = have < want ? have : want;
|
||||
const int16_t* pcm = (const int16_t*)[sink.pcm bytes];
|
||||
el_val_t list = empty;
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
list = el_list_append(list, (el_val_t)(int64_t)pcm[i]);
|
||||
}
|
||||
[sink.lock unlock];
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
el_val_t mic_available(void) {
|
||||
if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0;
|
||||
return (el_val_t)(el_cap_device_present(AVMediaTypeAudio) ? 1 : 0);
|
||||
}
|
||||
|
||||
el_val_t mic_request_access(void) {
|
||||
return (el_val_t)(el_cap_request(AVMediaTypeAudio) ? 1 : 0);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* MICROPHONE — live monitor (the full-duplex ear)
|
||||
*
|
||||
* converse needs to keep listening WHILE it speaks, which means the mic is open
|
||||
* at the same time as the speaker. In a real room that is a feedback path:
|
||||
* without cancellation Neuron hears its own voice, decides someone is talking,
|
||||
* and barges in on itself. setVoiceProcessingEnabled: hands the input node to
|
||||
* the OS voice-processing unit, which subtracts the known output signal from
|
||||
* the input — the single thing that makes barge-in work outside a headset.
|
||||
*
|
||||
* It is not always available (some aggregate and virtual devices refuse it), so
|
||||
* failure to enable it is reported as a DISTINCT return value (2) rather than
|
||||
* folded into success. The caller needs to know, because the correct response
|
||||
* is to raise the VAD floor, and a caller that thinks AEC is on will set that
|
||||
* floor far too low.
|
||||
*
|
||||
* The tap keeps only a running short-window RMS in a static behind a mutex.
|
||||
* Deliberately not a queue of samples: this path is polled at ~50 Hz by a loop
|
||||
* that only ever asks "is someone talking", and buffering audio nobody reads
|
||||
* would be an unbounded allocation in the middle of a conversation.
|
||||
* ══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static AVAudioEngine* g_mon_engine = nil;
|
||||
static int g_mon_running = 0;
|
||||
static int g_mon_code = 0; /* what the successful start reported */
|
||||
static double g_mon_rms = 0.0;
|
||||
static pthread_mutex_t g_mon_lock = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
el_val_t mic_monitor_start(void) {
|
||||
/* Idempotent, and it re-reports the ORIGINAL code rather than a bare 1: a
|
||||
* caller that starts twice must not be told AEC is on when the first start
|
||||
* already discovered it was not. */
|
||||
if (g_mon_running) return (el_val_t)g_mon_code;
|
||||
if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0;
|
||||
|
||||
AVAudioEngine* engine = nil;
|
||||
AVAudioInputNode* input = nil;
|
||||
AVAudioFormat* fmt = nil;
|
||||
int aec = 0;
|
||||
int tapped = 0;
|
||||
|
||||
@try {
|
||||
engine = [[AVAudioEngine alloc] init];
|
||||
input = [engine inputNode];
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
if (!input) return (el_val_t)0;
|
||||
|
||||
/* Enable AEC BEFORE reading the format: the voice-processing unit imposes
|
||||
* its own input format, and a tap installed with the pre-VP format would be
|
||||
* rejected at start. */
|
||||
@try {
|
||||
NSError* vpErr = nil;
|
||||
if ([input respondsToSelector:@selector(setVoiceProcessingEnabled:error:)]) {
|
||||
aec = [input setVoiceProcessingEnabled:YES error:&vpErr] ? 1 : 0;
|
||||
}
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
aec = 0;
|
||||
}
|
||||
|
||||
@try {
|
||||
fmt = [input inputFormatForBus:0];
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
if (!fmt || fmt.sampleRate <= 0 || fmt.channelCount == 0) return (el_val_t)0;
|
||||
|
||||
el_cap_disclose(aec
|
||||
? "MIC: opening the microphone for live monitoring, echo-cancelled (local)."
|
||||
: "MIC: opening the microphone for live monitoring, NO echo cancellation (local).");
|
||||
|
||||
@try {
|
||||
[input installTapOnBus:0
|
||||
bufferSize:1024
|
||||
format:fmt
|
||||
block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) {
|
||||
(void)when;
|
||||
if (!buf) return;
|
||||
AVAudioFrameCount n = buf.frameLength;
|
||||
if (n == 0) return;
|
||||
|
||||
double sum = 0.0;
|
||||
/* Whatever the VP unit hands back — float32 is the norm, int16 and
|
||||
* int32 are possible on odd hardware — normalise to -1..1 so the
|
||||
* Float El sees means the same thing on every device. */
|
||||
if (buf.floatChannelData) {
|
||||
const float* ch = buf.floatChannelData[0];
|
||||
for (AVAudioFrameCount i = 0; i < n; i++) sum += (double)ch[i] * (double)ch[i];
|
||||
} else if (buf.int16ChannelData) {
|
||||
const int16_t* ch = buf.int16ChannelData[0];
|
||||
for (AVAudioFrameCount i = 0; i < n; i++) {
|
||||
double v = (double)ch[i] / 32768.0;
|
||||
sum += v * v;
|
||||
}
|
||||
} else if (buf.int32ChannelData) {
|
||||
const int32_t* ch = buf.int32ChannelData[0];
|
||||
for (AVAudioFrameCount i = 0; i < n; i++) {
|
||||
double v = (double)ch[i] / 2147483648.0;
|
||||
sum += v * v;
|
||||
}
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
double rms = sqrt(sum / (double)n);
|
||||
if (rms < 0.0) rms = 0.0;
|
||||
if (rms > 1.0) rms = 1.0;
|
||||
|
||||
pthread_mutex_lock(&g_mon_lock);
|
||||
g_mon_rms = rms;
|
||||
pthread_mutex_unlock(&g_mon_lock);
|
||||
}];
|
||||
tapped = 1;
|
||||
|
||||
[engine prepare];
|
||||
NSError* startErr = nil;
|
||||
if (![engine startAndReturnError:&startErr]) {
|
||||
[input removeTapOnBus:0];
|
||||
return (el_val_t)0;
|
||||
}
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
@try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; }
|
||||
@try { [engine stop]; } @catch (NSException* e2) { (void)e2; }
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&g_mon_lock);
|
||||
g_mon_rms = 0.0;
|
||||
pthread_mutex_unlock(&g_mon_lock);
|
||||
|
||||
g_mon_engine = engine;
|
||||
g_mon_running = 1;
|
||||
g_mon_code = aec ? 1 : 2;
|
||||
return (el_val_t)g_mon_code;
|
||||
}
|
||||
|
||||
/* Float in 0..1. Reads the last window the tap computed; never blocks on the
|
||||
* audio thread beyond the mutex, because this is polled inside a turn-taking
|
||||
* loop where a stall IS a missed barge-in. */
|
||||
el_val_t mic_monitor_rms(void) {
|
||||
double rms = 0.0;
|
||||
pthread_mutex_lock(&g_mon_lock);
|
||||
rms = g_mon_rms;
|
||||
pthread_mutex_unlock(&g_mon_lock);
|
||||
return el_from_float(rms);
|
||||
}
|
||||
|
||||
el_val_t mic_monitor_stop(void) {
|
||||
AVAudioEngine* engine = g_mon_engine;
|
||||
g_mon_engine = nil;
|
||||
g_mon_running = 0;
|
||||
g_mon_code = 0;
|
||||
|
||||
if (engine) {
|
||||
@try { [[engine inputNode] removeTapOnBus:0]; } @catch (NSException* e) { (void)e; }
|
||||
@try { [engine stop]; } @catch (NSException* e) { (void)e; }
|
||||
}
|
||||
pthread_mutex_lock(&g_mon_lock);
|
||||
g_mon_rms = 0.0;
|
||||
pthread_mutex_unlock(&g_mon_lock);
|
||||
return (el_val_t)1;
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* CAMERA
|
||||
* ══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static void el_cap_free_bitmap(void* info, const void* data, size_t size) {
|
||||
(void)info; (void)size;
|
||||
free((void*)data);
|
||||
}
|
||||
|
||||
/* CVPixelBuffer -> CGImage, own-core, no CoreImage.
|
||||
*
|
||||
* The output is pinned to 32BGRA at the AVCaptureVideoDataOutput (see below)
|
||||
* precisely so this conversion can be a memcpy and a CGImageCreate. The
|
||||
* alternative — accepting the camera's native 2vuy/420v and colour-converting
|
||||
* here — would mean either pulling in CoreImage or writing a YUV->RGB matrix in
|
||||
* the file that is supposed to contain no arithmetic. Asking the capture output
|
||||
* for BGRA moves that work into AVFoundation, where it is already written and
|
||||
* already hardware-accelerated.
|
||||
*
|
||||
* The rows are copied out rather than aliased because the CVPixelBuffer is
|
||||
* recycled by the capture session the moment the delegate returns; a CGImage
|
||||
* pointing at it would be pointing at the NEXT frame by the time anyone looked. */
|
||||
static CGImageRef el_cap_cgimage_from_pixelbuffer(CVPixelBufferRef pb) {
|
||||
if (!pb) return NULL;
|
||||
if (CVPixelBufferGetPixelFormatType(pb) != kCVPixelFormatType_32BGRA) return NULL;
|
||||
if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return NULL;
|
||||
|
||||
size_t w = CVPixelBufferGetWidth(pb);
|
||||
size_t h = CVPixelBufferGetHeight(pb);
|
||||
size_t src_bpr = CVPixelBufferGetBytesPerRow(pb);
|
||||
const uint8_t* base = (const uint8_t*)CVPixelBufferGetBaseAddress(pb);
|
||||
|
||||
CGImageRef img = NULL;
|
||||
if (base && w > 0 && h > 0 && src_bpr >= w * 4) {
|
||||
size_t dst_bpr = w * 4;
|
||||
uint8_t* copy = (uint8_t*)malloc(dst_bpr * h);
|
||||
if (copy) {
|
||||
for (size_t y = 0; y < h; y++) {
|
||||
memcpy(copy + y * dst_bpr, base + y * src_bpr, dst_bpr);
|
||||
}
|
||||
CGDataProviderRef dp =
|
||||
CGDataProviderCreateWithData(NULL, copy, dst_bpr * h, el_cap_free_bitmap);
|
||||
if (dp) {
|
||||
CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB();
|
||||
if (cs) {
|
||||
img = CGImageCreate(w, h, 8, 32, dst_bpr, cs,
|
||||
(CGBitmapInfo)(kCGBitmapByteOrder32Little |
|
||||
kCGImageAlphaNoneSkipFirst),
|
||||
dp, NULL, false, kCGRenderingIntentDefault);
|
||||
CGColorSpaceRelease(cs);
|
||||
}
|
||||
CGDataProviderRelease(dp); /* provider owns `copy` from here */
|
||||
} else {
|
||||
free(copy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly);
|
||||
return img;
|
||||
}
|
||||
|
||||
/* The frame delegate. AVCaptureVideoDataOutput is used rather than
|
||||
* AVCapturePhotoOutput for the same reason periph.swift used it: the photo path
|
||||
* wants KVO and a session owned by an app object, and this runs in a plain CLI
|
||||
* process with no run loop it can assume. A data output just calls back.
|
||||
*
|
||||
* The first frames are dropped on purpose. A camera that has just been powered
|
||||
* on is still converging exposure and white balance, and the first frame is
|
||||
* reliably darker and greener than the room. El's scene-geometry descriptors
|
||||
* are brightness and mean-colour statistics, so handing up an unsettled frame
|
||||
* would not produce a slightly worse answer, it would produce a confidently
|
||||
* wrong one. */
|
||||
@interface ElCapFrameGrabber : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate> {
|
||||
CGImageRef _img;
|
||||
int _seen;
|
||||
dispatch_semaphore_t _sem;
|
||||
}
|
||||
- (dispatch_semaphore_t)sem;
|
||||
- (CGImageRef)takeImage; /* transfers ownership to the caller */
|
||||
@end
|
||||
|
||||
@implementation ElCapFrameGrabber
|
||||
|
||||
- (instancetype)init {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_img = NULL;
|
||||
_seen = 0;
|
||||
_sem = dispatch_semaphore_create(0);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (dispatch_semaphore_t)sem { return _sem; }
|
||||
|
||||
- (CGImageRef)takeImage {
|
||||
CGImageRef out = _img;
|
||||
_img = NULL;
|
||||
return out;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
if (_img) { CGImageRelease(_img); _img = NULL; }
|
||||
}
|
||||
|
||||
/* Runs on the serial delegate queue, so no lock is needed among callbacks; the
|
||||
* waiter only reads _img after the semaphore has been signalled AND the session
|
||||
* has been stopped, which orders it after the last callback. */
|
||||
- (void)captureOutput:(AVCaptureOutput*)output
|
||||
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
fromConnection:(AVCaptureConnection*)connection {
|
||||
(void)output; (void)connection;
|
||||
_seen++;
|
||||
if (_img != NULL || _seen < 5) return; /* let exposure settle */
|
||||
|
||||
CVImageBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer);
|
||||
if (!pb) return;
|
||||
CGImageRef img = el_cap_cgimage_from_pixelbuffer(pb);
|
||||
if (!img) return;
|
||||
_img = img;
|
||||
dispatch_semaphore_signal(_sem);
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
/* Bring the camera up, take exactly one settled frame, put it back down.
|
||||
* Returns a +1 CGImageRef the caller releases, or NULL. Bounded at 10s: a
|
||||
* camera held by another process, or one whose TCC grant was revoked between
|
||||
* the check and the open, must fail rather than park. */
|
||||
static CGImageRef el_cap_grab_frame(void) {
|
||||
AVCaptureSession* session = nil;
|
||||
AVCaptureVideoDataOutput* output = nil;
|
||||
ElCapFrameGrabber* grabber = nil;
|
||||
CGImageRef img = NULL;
|
||||
|
||||
@try {
|
||||
AVCaptureDevice* dev = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
|
||||
if (!dev) return NULL;
|
||||
|
||||
NSError* err = nil;
|
||||
AVCaptureDeviceInput* in = [AVCaptureDeviceInput deviceInputWithDevice:dev error:&err];
|
||||
if (!in) return NULL;
|
||||
|
||||
session = [[AVCaptureSession alloc] init];
|
||||
session.sessionPreset = AVCaptureSessionPresetPhoto;
|
||||
if (![session canAddInput:in]) return NULL;
|
||||
[session addInput:in];
|
||||
|
||||
output = [[AVCaptureVideoDataOutput alloc] init];
|
||||
output.alwaysDiscardsLateVideoFrames = YES;
|
||||
/* Pin the pixel format so the CGImage conversion above stays a memcpy.
|
||||
* Every macOS capture device advertises 32BGRA. */
|
||||
output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey :
|
||||
@(kCVPixelFormatType_32BGRA) };
|
||||
|
||||
grabber = [[ElCapFrameGrabber alloc] init];
|
||||
dispatch_queue_t q = dispatch_queue_create("el.capture.camera", DISPATCH_QUEUE_SERIAL);
|
||||
[output setSampleBufferDelegate:grabber queue:q];
|
||||
|
||||
if (![session canAddOutput:output]) return NULL;
|
||||
[session addOutput:output];
|
||||
|
||||
el_cap_disclose("CAMERA: opening the camera for one frame (local, never egresses).");
|
||||
[session startRunning];
|
||||
} @catch (NSException* e) {
|
||||
(void)e;
|
||||
@try { [session stopRunning]; } @catch (NSException* e2) { (void)e2; }
|
||||
return NULL;
|
||||
}
|
||||
|
||||
long timed_out = dispatch_semaphore_wait([grabber sem],
|
||||
dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC)));
|
||||
|
||||
/* Stop first, then detach the delegate, then read. In that order the last
|
||||
* callback has already returned by the time anyone touches the image. */
|
||||
@try { [session stopRunning]; } @catch (NSException* e) { (void)e; }
|
||||
@try { [output setSampleBufferDelegate:nil queue:NULL]; } @catch (NSException* e) { (void)e; }
|
||||
|
||||
if (timed_out == 0) img = [grabber takeImage];
|
||||
return img;
|
||||
}
|
||||
|
||||
el_val_t camera_available(void) {
|
||||
if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0;
|
||||
return (el_val_t)(el_cap_device_present(AVMediaTypeVideo) ? 1 : 0);
|
||||
}
|
||||
|
||||
el_val_t camera_request_access(void) {
|
||||
return (el_val_t)(el_cap_request(AVMediaTypeVideo) ? 1 : 0);
|
||||
}
|
||||
|
||||
/* Longest edge of the grid handed to El. 64 is not a resolution, it is a budget:
|
||||
* a 1920x1080 frame is 6.2 MILLION packed RGB ints, and building that as an El
|
||||
* list would cost more time and memory than everything El then does with it.
|
||||
* The descriptors El computes over this — mean colour, brightness, a 3x3
|
||||
* luminance grid — are region statistics, and region statistics do not get
|
||||
* meaningfully better above a 64-wide grid. The TRUE frame dimensions are
|
||||
* reported separately so nothing downstream has to guess what was thrown away. */
|
||||
#define EL_CAP_GRID_MAX 64
|
||||
|
||||
/* One frame as Map{width, height, grid_w, grid_h, pixels:[Int]}.
|
||||
*
|
||||
* "pixels" is packed R,G,B with NO alpha — three ints per grid cell, row-major
|
||||
* from the TOP-LEFT. (CGBitmapContext lays its buffer out top row first and
|
||||
* CGContextDrawImage does the flip, so row 0 here is the top of the frame, the
|
||||
* same convention periph.swift's grid indexing assumed.) Alpha is dropped
|
||||
* because a camera frame has none worth carrying and it would inflate the list
|
||||
* by a third to say "opaque" six thousand times. */
|
||||
el_val_t camera_capture_rgb(void) {
|
||||
if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0;
|
||||
|
||||
CGImageRef img = el_cap_grab_frame();
|
||||
if (!img) return (el_val_t)0;
|
||||
|
||||
size_t w = CGImageGetWidth(img);
|
||||
size_t h = CGImageGetHeight(img);
|
||||
if (w == 0 || h == 0) { CGImageRelease(img); return (el_val_t)0; }
|
||||
|
||||
/* Preserve aspect ratio, longest edge capped. */
|
||||
size_t gw = w, gh = h;
|
||||
size_t longest = w > h ? w : h;
|
||||
if (longest > EL_CAP_GRID_MAX) {
|
||||
double s = (double)EL_CAP_GRID_MAX / (double)longest;
|
||||
gw = (size_t)((double)w * s + 0.5);
|
||||
gh = (size_t)((double)h * s + 0.5);
|
||||
if (gw == 0) gw = 1;
|
||||
if (gh == 0) gh = 1;
|
||||
}
|
||||
|
||||
size_t bpr = gw * 4;
|
||||
uint8_t* buf = (uint8_t*)calloc(1, bpr * gh);
|
||||
if (!buf) { CGImageRelease(img); return (el_val_t)0; }
|
||||
|
||||
CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB();
|
||||
CGContextRef ctx = cs ? CGBitmapContextCreate(buf, gw, gh, 8, bpr, cs,
|
||||
(CGBitmapInfo)kCGImageAlphaPremultipliedLast)
|
||||
: NULL;
|
||||
if (cs) CGColorSpaceRelease(cs);
|
||||
if (!ctx) { free(buf); CGImageRelease(img); return (el_val_t)0; }
|
||||
|
||||
/* Nearest-neighbour. This is a decimation for statistics, not a thumbnail
|
||||
* for a human to look at; smoothing would only cost time and blur the very
|
||||
* region boundaries the grid exists to measure. */
|
||||
CGContextSetInterpolationQuality(ctx, kCGInterpolationNone);
|
||||
CGContextDrawImage(ctx, CGRectMake(0, 0, (CGFloat)gw, (CGFloat)gh), img);
|
||||
CGContextRelease(ctx);
|
||||
CGImageRelease(img);
|
||||
|
||||
el_val_t pixels = el_list_empty();
|
||||
for (size_t y = 0; y < gh; y++) {
|
||||
const uint8_t* row = buf + y * bpr;
|
||||
for (size_t x = 0; x < gw; x++) {
|
||||
const uint8_t* p = row + x * 4; /* RGBA8, premultiplied-last */
|
||||
pixels = el_list_append(pixels, (el_val_t)(int64_t)p[0]);
|
||||
pixels = el_list_append(pixels, (el_val_t)(int64_t)p[1]);
|
||||
pixels = el_list_append(pixels, (el_val_t)(int64_t)p[2]);
|
||||
}
|
||||
}
|
||||
free(buf);
|
||||
|
||||
el_val_t m = el_map_new((el_val_t)0);
|
||||
if (!m) return (el_val_t)0;
|
||||
m = el_map_set(m, EL_STR("width"), (el_val_t)(int64_t)w);
|
||||
m = el_map_set(m, EL_STR("height"), (el_val_t)(int64_t)h);
|
||||
m = el_map_set(m, EL_STR("grid_w"), (el_val_t)(int64_t)gw);
|
||||
m = el_map_set(m, EL_STR("grid_h"), (el_val_t)(int64_t)gh);
|
||||
m = el_map_set(m, EL_STR("pixels"), pixels);
|
||||
return m;
|
||||
}
|
||||
|
||||
/* One frame to disk as JPEG, at FULL resolution — the opposite budget from
|
||||
* camera_capture_rgb, and for the opposite reason. A file is not being walked
|
||||
* element-by-element by an interpreter; it costs one ImageIO call and it is the
|
||||
* artefact a human or a later pass will actually look at. The encoder is
|
||||
* ImageIO's because a JPEG encoder is a codec, and re-implementing one in El
|
||||
* would be a large amount of arithmetic that buys nothing: the point of keeping
|
||||
* work in El is the reasoning, not the entropy coding. */
|
||||
el_val_t camera_capture_jpeg(el_val_t path) {
|
||||
const char* p = EL_CSTR(path);
|
||||
if (!p || !*p) return (el_val_t)0;
|
||||
if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0;
|
||||
|
||||
CGImageRef img = el_cap_grab_frame();
|
||||
if (!img) return (el_val_t)0;
|
||||
|
||||
int ok = 0;
|
||||
@autoreleasepool {
|
||||
NSString* ns = [NSString stringWithUTF8String:p];
|
||||
NSURL* url = ns ? [NSURL fileURLWithPath:ns] : nil;
|
||||
if (url) {
|
||||
CGImageDestinationRef dst =
|
||||
CGImageDestinationCreateWithURL((__bridge CFURLRef)url, CFSTR("public.jpeg"), 1, NULL);
|
||||
if (dst) {
|
||||
CGImageDestinationAddImage(dst, img, NULL);
|
||||
ok = CGImageDestinationFinalize(dst) ? 1 : 0;
|
||||
CFRelease(dst);
|
||||
}
|
||||
}
|
||||
}
|
||||
CGImageRelease(img);
|
||||
return (el_val_t)ok;
|
||||
}
|
||||
|
||||
#endif /* __APPLE__ */
|
||||
@@ -0,0 +1,89 @@
|
||||
/* el_peripheral_null.c — the no-device build of El's I/O organ.
|
||||
*
|
||||
* Every entry point declared in el_runtime.h's "Peripheral" block, implemented
|
||||
* as an honest refusal. This is what a platform without an El audio/capture
|
||||
* realizer links instead of el_audio_darwin.m + el_capture_darwin.m, so an El
|
||||
* program that speaks or listens still COMPILES AND LINKS everywhere.
|
||||
*
|
||||
* The distinction that matters: these do not pretend. speaker_available() and
|
||||
* mic_available() return 0, and every operation returns its failure sentinel.
|
||||
* A program asking "can I speak here?" gets a truthful no, rather than a
|
||||
* silence it would have to infer something from. Silent success is the failure
|
||||
* mode this whole change exists to eliminate — El spent this entire codebase's
|
||||
* history writing WAV files full of zeros and reporting ok=true, and nobody
|
||||
* caught it because nothing ever said "there is no sound here".
|
||||
*
|
||||
* Compiled INSTEAD OF the Darwin realizers, never alongside them — the symbols
|
||||
* are the same by design, which is the point: the El side never branches on
|
||||
* platform, it branches on speaker_available().
|
||||
*/
|
||||
|
||||
#include "el_runtime.h"
|
||||
|
||||
#if !defined(__APPLE__)
|
||||
|
||||
/* ── Speaker ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t speaker_available(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_name(void) { return EL_STR("none"); }
|
||||
|
||||
el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) {
|
||||
(void)samples; (void)sample_rate;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
el_val_t speaker_play_wav(el_val_t path) {
|
||||
(void)path;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) {
|
||||
(void)samples; (void)sample_rate;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
el_val_t speaker_play_wav_async(el_val_t path) {
|
||||
(void)path;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
el_val_t speaker_pause(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_resume(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_playing(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_stop(void) { return (el_val_t)0; }
|
||||
el_val_t speaker_played_frames(void) { return (el_val_t)0; }
|
||||
|
||||
/* WAV geometry is pure parsing and would work fine here, but reporting a
|
||||
* duration for audio this build cannot play would invite a caller to sequence
|
||||
* around a silence. Refuse consistently with the rest of the file. */
|
||||
el_val_t wav_frames(el_val_t path) { (void)path; return (el_val_t)0; }
|
||||
el_val_t wav_rate(el_val_t path) { (void)path; return (el_val_t)0; }
|
||||
|
||||
/* ── Microphone ──────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t mic_available(void) { return (el_val_t)0; }
|
||||
el_val_t mic_request_access(void) { return (el_val_t)0; }
|
||||
|
||||
/* Empty list, not 0: the contract says capture returns samples, and a caller
|
||||
* iterating the result must find nothing rather than dereference a non-list. */
|
||||
el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) {
|
||||
(void)seconds; (void)sample_rate;
|
||||
return el_list_empty();
|
||||
}
|
||||
|
||||
el_val_t mic_monitor_start(void) { return (el_val_t)0; }
|
||||
el_val_t mic_monitor_rms(void) { return el_from_float(0.0); }
|
||||
el_val_t mic_monitor_stop(void) { return (el_val_t)0; }
|
||||
|
||||
/* ── Camera ──────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t camera_available(void) { return (el_val_t)0; }
|
||||
el_val_t camera_request_access(void) { return (el_val_t)0; }
|
||||
el_val_t camera_capture_rgb(void) { return (el_val_t)0; }
|
||||
|
||||
el_val_t camera_capture_jpeg(el_val_t path) {
|
||||
(void)path;
|
||||
return (el_val_t)0;
|
||||
}
|
||||
|
||||
#endif /* !__APPLE__ */
|
||||
@@ -80,6 +80,92 @@ void println(el_val_t s);
|
||||
void print(el_val_t s);
|
||||
el_val_t readline(void);
|
||||
|
||||
/* stderr counterpart of println (defined in el_seed.c). El could write to
|
||||
* stdout and nowhere else, which is right for a program's RESULT and wrong for
|
||||
* everything about how that result was produced. Disclosure especially has to
|
||||
* leave on a stream the caller can separate from the answer: a program that
|
||||
* announces "I am about to open the microphone" on stdout has corrupted its own
|
||||
* output. Flushed on every call, so a disclosure reaches the terminal BEFORE
|
||||
* the device it describes is touched rather than whenever the buffer drains. */
|
||||
void eprintln(el_val_t s);
|
||||
|
||||
/* ── Peripheral: the speaker, the microphone, the camera ─────────────────────
|
||||
*
|
||||
* El's I/O organ. Implemented per platform in its OWN translation unit —
|
||||
* el_audio_darwin.m / el_capture_darwin.m on Darwin, el_peripheral_null.c
|
||||
* everywhere else — so El code that speaks or listens links on every platform
|
||||
* and merely reports having no device where there isn't one. Declared here and
|
||||
* deliberately NOT implemented in el_runtime.c: acquiring a device must not
|
||||
* mean editing the middle of the language, the same rule the realizer registry
|
||||
* follows for modalities.
|
||||
*
|
||||
* These are the ONLY parts of the organ that are not El. Everything above the
|
||||
* sample buffer — WAV encode/decode, LPC autocorrelation, Levinson-Durbin,
|
||||
* formant extraction, source-filter resynthesis, the compact descriptors, the
|
||||
* converse decision loop — is arithmetic, and arithmetic belongs in El. What
|
||||
* remains here is what El cannot express: handing a buffer to the DAC and
|
||||
* waiting for it to drain, and asking the OS for frames off a capture device.
|
||||
*
|
||||
* Local by construction: none of these entry points has a network path. Samples
|
||||
* and pixels go to and from local hardware and nowhere else. Consent is
|
||||
* enforced ABOVE this layer in El (peripheral/src/organ.el) for the Neuron-level
|
||||
* grant, and BELOW it by the OS for TCC; capture fails closed on either. */
|
||||
|
||||
/* Speaker (efferent). speaker_play_pcm16 BLOCKS until the audio has actually
|
||||
* been played rather than merely queued, so a caller can sequence utterances
|
||||
* without guessing durations and without clipping each tail. */
|
||||
el_val_t speaker_available(void); /* 1 if a real speaker backs this build */
|
||||
el_val_t speaker_name(void); /* backend id, e.g. "coreaudio-audioqueue" */
|
||||
el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate); /* [Int] 16-bit mono; 1 ok */
|
||||
el_val_t speaker_play_wav(el_val_t path); /* 16-bit mono RIFF/WAVE; 1 ok */
|
||||
|
||||
/* Asynchronous playback — required by converse, which must keep listening while
|
||||
* it speaks and must be able to stop ON THE SPOT mid-buffer. A blocking play
|
||||
* cannot be interrupted, and "finish the current buffer" is not barge-in.
|
||||
* speaker_stop() halts output immediately; speaker_playing() reports whether
|
||||
* the hardware is still going; speaker_played_frames() is how far it actually
|
||||
* got, which is what makes an interrupted utterance resumable at the sample. */
|
||||
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate);
|
||||
el_val_t speaker_play_wav_async(el_val_t path);
|
||||
el_val_t speaker_pause(void); /* stop AT THIS SAMPLE, keep position */
|
||||
el_val_t speaker_resume(void); /* carry on from exactly there */
|
||||
el_val_t speaker_playing(void);
|
||||
el_val_t speaker_stop(void);
|
||||
el_val_t speaker_played_frames(void);
|
||||
|
||||
/* WAV geometry without playing — wav-info, and the segment duration converse
|
||||
* needs to turn elapsed time into progress. */
|
||||
el_val_t wav_frames(el_val_t path);
|
||||
el_val_t wav_rate(el_val_t path);
|
||||
|
||||
/* Microphone (afferent). Fails CLOSED: returns 0 unless the OS has granted
|
||||
* capture access. mic_capture_pcm16 blocks for `seconds` and returns an [Int]
|
||||
* of 16-bit mono samples at `sample_rate` — the raw stream is handed to El and
|
||||
* never written anywhere by this layer. mic_available() reports device +
|
||||
* permission state without prompting. */
|
||||
el_val_t mic_available(void); /* 1 device present AND OS-authorized */
|
||||
el_val_t mic_request_access(void); /* prompt once; 1 if granted */
|
||||
el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate); /* [Int], empty on refusal */
|
||||
|
||||
/* Live monitoring for full-duplex converse. mic_monitor_start enables the OS
|
||||
* voice-processing unit (acoustic echo cancellation) so the microphone does not
|
||||
* hear the speaker — without AEC, Neuron barges in on its own voice and
|
||||
* turn-taking is unusable in a real room. mic_monitor_rms returns the current
|
||||
* short-window RMS as a Float in 0..1. */
|
||||
el_val_t mic_monitor_start(void); /* 1 ok; 2 = started but AEC unavailable */
|
||||
el_val_t mic_monitor_rms(void); /* Float */
|
||||
el_val_t mic_monitor_stop(void);
|
||||
|
||||
/* Camera (afferent). Fails CLOSED like the microphone. camera_capture_rgb
|
||||
* returns a Map with width/height and the frame as an [Int] of packed RGB
|
||||
* bytes, so the descriptor arithmetic can happen in El rather than here.
|
||||
* camera_capture_jpeg writes an encoded frame via ImageIO, which is a codec and
|
||||
* not something El should re-implement. */
|
||||
el_val_t camera_available(void);
|
||||
el_val_t camera_request_access(void);
|
||||
el_val_t camera_capture_rgb(void); /* Map{width,height,pixels:[Int]} or 0 */
|
||||
el_val_t camera_capture_jpeg(el_val_t path); /* 1 ok */
|
||||
|
||||
/* ── String builtins ─────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_str_concat(el_val_t a, el_val_t b);
|
||||
|
||||
+130
-3
@@ -154,9 +154,18 @@ static void seed_request_start(void) {
|
||||
* file still links on its own. */
|
||||
__attribute__((weak)) void el_str_cache_flush(void);
|
||||
|
||||
/* Byte-buffer capacity registry (defined below, next to the string
|
||||
* primitives). The arena frees the pointers it tracked, so any capacity
|
||||
* entry for those addresses must go with them — otherwise a later malloc
|
||||
* reusing the address would inherit a stale width. */
|
||||
static void seed_cap_drop(const char* p);
|
||||
|
||||
static void seed_request_end(void) {
|
||||
_seed_arena_on = 0;
|
||||
for (size_t i = 0; i < _seed_arena.count; i++) free(_seed_arena.ptrs[i]);
|
||||
for (size_t i = 0; i < _seed_arena.count; i++) {
|
||||
seed_cap_drop(_seed_arena.ptrs[i]);
|
||||
free(_seed_arena.ptrs[i]);
|
||||
}
|
||||
_seed_arena.count = 0;
|
||||
if (el_str_cache_flush) el_str_cache_flush(); /* freed pointers may be reused */
|
||||
}
|
||||
@@ -188,6 +197,114 @@ static char* seed_strbuf(size_t n) {
|
||||
|
||||
static el_val_t seed_wrap_str(char* s) { return EL_STR(s); }
|
||||
|
||||
/* ── Byte-buffer capacity registry ────────────────────────────────────────────
|
||||
* A String produced by __str_alloc is a fixed-size BYTE BUFFER, not text. Its
|
||||
* length is the capacity it was asked for; strlen() is meaningless on it,
|
||||
* because the buffer is zero-filled and binary content (PCM audio, RIFF
|
||||
* headers, image rasters) contains NUL bytes by nature.
|
||||
*
|
||||
* Before this registry existed, __str_set_char bounds-checked the write index
|
||||
* against strlen(p). For a freshly __str_alloc'd buffer strlen(p) == 0, so the
|
||||
* check `idx >= len` rejected EVERY index and the function was a total no-op:
|
||||
* every El program that built bytes this way wrote a file of pure zeros and
|
||||
* still saw a success return. That is why El's own-core WAV writer emitted
|
||||
* 55,244 silent bytes with a correct-looking header length and no header.
|
||||
*
|
||||
* The fix cannot be "trust the index", because that removes the bound. It also
|
||||
* cannot be a length header stored behind the pointer, because __str_set_char
|
||||
* accepts any String — including a string literal in .rodata, where reading the
|
||||
* bytes preceding the pointer is undefined and may fault. So capacity is kept
|
||||
* in a side table keyed by the pointer itself: allocation registers, the arena
|
||||
* sweep unregisters, and anything not registered keeps the exact strlen
|
||||
* behaviour it had before. Text semantics are unchanged; byte buffers gain the
|
||||
* bound they always should have had. */
|
||||
|
||||
typedef struct {
|
||||
char* ptr; /* NULL = empty slot, (char*)1 = tombstone */
|
||||
size_t cap;
|
||||
} SeedCapEntry;
|
||||
|
||||
#define SEED_CAP_TOMB ((char*)1)
|
||||
|
||||
static _Thread_local SeedCapEntry* _seed_cap = NULL;
|
||||
static _Thread_local size_t _seed_cap_mask = 0; /* table size - 1 */
|
||||
static _Thread_local size_t _seed_cap_used = 0; /* live + tombstoned */
|
||||
|
||||
static size_t seed_cap_hash(const char* p) {
|
||||
uintptr_t h = (uintptr_t)p >> 4; /* malloc alignment: low bits are dead */
|
||||
h *= (uintptr_t)0x9E3779B97F4A7C15ull;
|
||||
return (size_t)(h >> 32);
|
||||
}
|
||||
|
||||
static void seed_cap_put(char* p, size_t cap);
|
||||
|
||||
static void seed_cap_grow(void) {
|
||||
size_t old_size = _seed_cap_mask ? _seed_cap_mask + 1 : 0;
|
||||
SeedCapEntry* old = _seed_cap;
|
||||
size_t new_size = old_size ? old_size * 2 : 256;
|
||||
SeedCapEntry* fresh = calloc(new_size, sizeof(SeedCapEntry));
|
||||
if (!fresh) return; /* out of memory: keep old table */
|
||||
_seed_cap = fresh;
|
||||
_seed_cap_mask = new_size - 1;
|
||||
_seed_cap_used = 0;
|
||||
for (size_t i = 0; i < old_size; i++) {
|
||||
if (old[i].ptr && old[i].ptr != SEED_CAP_TOMB) seed_cap_put(old[i].ptr, old[i].cap);
|
||||
}
|
||||
free(old);
|
||||
}
|
||||
|
||||
static void seed_cap_put(char* p, size_t cap) {
|
||||
if (!p) return;
|
||||
if (!_seed_cap || (_seed_cap_used + 1) * 4 >= (_seed_cap_mask + 1) * 3) {
|
||||
seed_cap_grow();
|
||||
if (!_seed_cap) return;
|
||||
}
|
||||
size_t i = seed_cap_hash(p) & _seed_cap_mask;
|
||||
size_t first_free = (size_t)-1;
|
||||
for (;;) {
|
||||
char* e = _seed_cap[i].ptr;
|
||||
if (e == p) { _seed_cap[i].cap = cap; return; } /* address reused */
|
||||
if (e == SEED_CAP_TOMB && first_free == (size_t)-1) first_free = i;
|
||||
if (!e) {
|
||||
if (first_free != (size_t)-1) i = first_free; else _seed_cap_used++;
|
||||
_seed_cap[i].ptr = p;
|
||||
_seed_cap[i].cap = cap;
|
||||
return;
|
||||
}
|
||||
i = (i + 1) & _seed_cap_mask;
|
||||
}
|
||||
}
|
||||
|
||||
/* Capacity of a registered byte buffer, or -1 when the pointer is not one. */
|
||||
static int64_t seed_cap_get(const char* p) {
|
||||
if (!p || !_seed_cap) return -1;
|
||||
size_t i = seed_cap_hash(p) & _seed_cap_mask;
|
||||
for (;;) {
|
||||
char* e = _seed_cap[i].ptr;
|
||||
if (!e) return -1;
|
||||
if (e == (char*)p) return (int64_t)_seed_cap[i].cap;
|
||||
i = (i + 1) & _seed_cap_mask;
|
||||
}
|
||||
}
|
||||
|
||||
static void seed_cap_drop(const char* p) {
|
||||
if (!p || !_seed_cap) return;
|
||||
size_t i = seed_cap_hash(p) & _seed_cap_mask;
|
||||
for (;;) {
|
||||
char* e = _seed_cap[i].ptr;
|
||||
if (!e) return;
|
||||
if (e == (char*)p) { _seed_cap[i].ptr = SEED_CAP_TOMB; return; }
|
||||
i = (i + 1) & _seed_cap_mask;
|
||||
}
|
||||
}
|
||||
|
||||
/* Effective addressable length of a String: its buffer capacity when it is a
|
||||
* byte buffer, otherwise strlen. */
|
||||
static int64_t seed_addressable_len(const char* p) {
|
||||
int64_t cap = seed_cap_get(p);
|
||||
return cap >= 0 ? cap : (int64_t)strlen(p);
|
||||
}
|
||||
|
||||
/* ── String primitives ───────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t __str_len(el_val_t s) {
|
||||
@@ -199,7 +316,7 @@ el_val_t __str_len(el_val_t s) {
|
||||
el_val_t __str_char_at(el_val_t s, el_val_t i) {
|
||||
const char* p = EL_CSTR(s);
|
||||
if (!p) return 0;
|
||||
int64_t len = (int64_t)strlen(p);
|
||||
int64_t len = seed_addressable_len(p); /* capacity for byte buffers */
|
||||
int64_t idx = (int64_t)i;
|
||||
if (idx < 0 || idx >= len) return 0;
|
||||
return (el_val_t)(unsigned char)p[idx];
|
||||
@@ -210,13 +327,14 @@ el_val_t __str_alloc(el_val_t n) {
|
||||
if (sz < 0) sz = 0;
|
||||
char* buf = seed_strbuf((size_t)sz);
|
||||
memset(buf, 0, (size_t)sz + 1);
|
||||
seed_cap_put(buf, (size_t)sz); /* this is a byte buffer of width sz */
|
||||
return seed_wrap_str(buf);
|
||||
}
|
||||
|
||||
el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c) {
|
||||
char* p = (char*)(uintptr_t)s;
|
||||
if (!p) return s;
|
||||
int64_t len = (int64_t)strlen(p);
|
||||
int64_t len = seed_addressable_len(p); /* capacity for byte buffers */
|
||||
int64_t idx = (int64_t)i;
|
||||
if (idx < 0 || idx >= len) return s;
|
||||
p[idx] = (char)(unsigned char)(int64_t)c;
|
||||
@@ -406,6 +524,15 @@ el_val_t __fs_mkdir(el_val_t path) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* stderr counterpart of println. Flushed immediately: a disclosure line is only
|
||||
* worth anything if it lands before the thing it discloses happens. */
|
||||
void eprintln(el_val_t s) {
|
||||
const char* p = EL_CSTR(s);
|
||||
fputs(p ? p : "", stderr);
|
||||
fputc('\n', stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n) {
|
||||
const char* p = EL_CSTR(path);
|
||||
const char* b = EL_CSTR(bytes);
|
||||
|
||||
+165
-61
@@ -1,80 +1,184 @@
|
||||
# peripheral — Neuron's I/O organ (own-core, local, consent-gated)
|
||||
# peripheral — Neuron's I/O organ, in El
|
||||
|
||||
The interface made physical. Two afferent senses in, one efferent voice out —
|
||||
all reached the way the agentic surface reaches any tool.
|
||||
**El speaks.** The engram stores geometry and does not speak; the speaking
|
||||
belongs to the language and its runtime.
|
||||
|
||||
Until this landed, the organ was a 939-line Swift program (`src/periph.swift`)
|
||||
that shelled out to `afplay`. Neuron's mouth and ears were a separate binary
|
||||
standing next to the language, and "speak" meant "ask that binary to speak."
|
||||
That program is now **reference material, not the implementation.**
|
||||
|
||||
```
|
||||
MIC (hear) afferent device -> capture -> descriptor -> ingest -> geometry
|
||||
CAMERA (see) afferent device -> capture -> descriptor -> ingest -> scene-geometry
|
||||
SPEAKER(speak) efferent render WAV -> PLAY ALOUD out the speaker
|
||||
SPEAKER (speak) efferent samples ──────────────► CoreAudio ──► the room
|
||||
MIC (hear) afferent device ──► samples ──► descriptor ──► engram
|
||||
CAMERA (see) afferent device ──► frame ──► descriptor ──► engram
|
||||
```
|
||||
|
||||
Closes the conversational loop: **hear (mic) -> understand (engram) -> speak (speaker)**.
|
||||
## The split, and why it falls where it does
|
||||
|
||||
Exactly **two** things here are not El, and they are the two things El cannot
|
||||
express as arithmetic:
|
||||
|
||||
| Not El (realizers) | Why |
|
||||
|---|---|
|
||||
| `lang/runtime/el_audio_darwin.m` | Handing a buffer to the DAC and waiting for it to drain. There is no way to say "the hardware has now played these samples" in El, and there should not be. |
|
||||
| `lang/runtime/el_capture_darwin.m` | Asking the OS for samples off a microphone or frames off a camera, plus the TCC permission dance. |
|
||||
|
||||
**Everything else is El**, because everything else is arithmetic:
|
||||
|
||||
| In El | Where |
|
||||
|---|---|
|
||||
| WAV encode / decode (chunk-walking, JUNK/FLLR tolerant) | `src/organ_dsp.el`, `elp/src/speech.el` |
|
||||
| LPC autocorrelation + Levinson-Durbin (order 16 @ 16 kHz) | `src/organ_dsp.el` |
|
||||
| Formant extraction off the all-pole spectral envelope | `src/organ_dsp.el` |
|
||||
| Source-filter resynthesis (glottal impulse train through the filter) | `src/organ_dsp.el` |
|
||||
| Audio descriptor `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` | `src/organ_dsp.el` |
|
||||
| Voice descriptor `[F0, F1..F5, bandwidths]` | `src/organ_dsp.el` |
|
||||
| Scene descriptor `[w, h, meanRGB, brightness, 3×3 luminance grid]` | `src/organ.el` |
|
||||
| Consent, disclosure, the voice-from-engram fetch | `src/organ.el` |
|
||||
| Barge-in, yield-or-hold, backchannel, resume | `src/organ_converse.el` |
|
||||
| The command surface | `src/organ_cli.el` |
|
||||
|
||||
Both realizers are their **own translation units**, declared in
|
||||
`lang/runtime/el_runtime.h`, and deliberately **not** patches to
|
||||
`el_runtime.c`. Acquiring a device must not mean editing the middle of the
|
||||
language — the same rule the realizer registry follows for modalities.
|
||||
`lang/runtime/el_peripheral_null.c` provides the identical entry points
|
||||
everywhere else, so El that speaks links on any platform and truthfully reports
|
||||
having no speaker rather than going quietly silent.
|
||||
|
||||
## The voice comes from the engram
|
||||
|
||||
A voice is **geometry in the engram**, not a JSON file next to the code and
|
||||
certainly not constants in a source file. The organ fetches it the way anything
|
||||
retrieves a memory — it asks:
|
||||
|
||||
```el
|
||||
let g: [Int] = organ_voice_fetch("will")
|
||||
// [peripheral] VOICE: fetched 'will' FROM THE ENGRAM —
|
||||
// f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531
|
||||
```
|
||||
|
||||
`organ_voice_fetch` issues an engram query and reads the geometry off the node
|
||||
that comes back. Nothing opens a file. If the region is not in the graph it
|
||||
returns **empty**, not a plausible default — a caller has to be able to tell
|
||||
"this is how they sound" from "I never heard them."
|
||||
|
||||
The reverse direction is `ingest-voice`: an LPC voiceprint becomes a node, and
|
||||
from then on the voice is a memory rather than a measurement someone wrote down.
|
||||
|
||||
## What the organ never does
|
||||
|
||||
**It never learns a word.** Pronunciation, vocabulary and phonemes belong to the
|
||||
language faculty and are already built as ingested geometry — *the engram knows
|
||||
how to pronounce*. The seam is `synth_codes(codes, voice, pmap)`: the codes and
|
||||
the phoneme map arrive from the language side as geometry, and the organ's whole
|
||||
job is turning them into samples and getting the samples out the speaker, plus
|
||||
the same trip in reverse for the senses. There is no lexicon here and no
|
||||
grapheme-to-phoneme rule, by design.
|
||||
|
||||
## Rails
|
||||
- **Own-core.** macOS-native only: AVFoundation (camera/mic), CoreAudio voice-
|
||||
processing (AEC), afplay (speaker), ImageIO/CoreGraphics (frames), hand-rolled
|
||||
DSP (WAV, LPC, formant synthesis). No cloud, no heavy deps.
|
||||
- **Local-only.** Raw streams are written to `out/` and never egress. `.gitignore`
|
||||
keeps captured media out of git.
|
||||
- **Consent-gated (two locks).** A Neuron-level grant (`grant`/`revoke`) *and* the
|
||||
OS TCC permission. Sensitive senses (camera/mic) fail closed without both.
|
||||
- **Disclosed.** Every device touch prints a `[peripheral]` line on stderr.
|
||||
|
||||
- **Own-core.** CoreAudio / AVFoundation / ImageIO — all ship with macOS. No
|
||||
cloud, no model, no heavy dependency. There is **no network code in the organ
|
||||
at all**, by construction.
|
||||
- **Local-only.** Raw streams stay on the machine. What leaves a capture is a
|
||||
descriptor of a few dozen numbers. A 1920×1080 frame becomes 15 integers
|
||||
(~414,000× smaller); three seconds of audio becomes 8.
|
||||
- **Consent, two locks.** A Neuron-level grant **and** the OS TCC permission.
|
||||
Camera and mic **fail closed** without both. The speaker is disclosed but not
|
||||
gated — you cannot secretly speak aloud, and gating it would mean Neuron needs
|
||||
permission to answer.
|
||||
- **Disclosed.** Every device touch prints a `[peripheral]` line on **stderr**
|
||||
(via `eprintln`, flushed immediately), so a disclosure lands before the device
|
||||
is touched and never contaminates the program's stdout.
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
./peripheral/build.sh /tmp/organ
|
||||
```
|
||||
swiftc -O -o bin/periph src/periph.swift \
|
||||
-framework AVFoundation -framework CoreMedia -framework Foundation \
|
||||
-framework CoreGraphics -framework ImageIO -framework CoreImage
|
||||
```
|
||||
|
||||
Concatenates the El modules, compiles with `elc`, links the two realizers.
|
||||
Run it **from the repo root** or the `.psv` phoneme data will not resolve.
|
||||
|
||||
## Commands
|
||||
|
||||
```
|
||||
periph grant|revoke <camera|mic> # Neuron-level consent
|
||||
periph status
|
||||
periph speak <file.wav> # SPEAK ALOUD (efferent)
|
||||
periph tone <out.wav> [hz] [sec] # own-core WAV synth
|
||||
periph listen <sec> <out.wav> # MIC capture (afferent), 16k mono
|
||||
periph see <out.jpg> # CAMERA one frame (afferent)
|
||||
periph feat-audio <wav> | feat-image <jpg> # capture -> compact descriptor
|
||||
periph ingest-audio|ingest-image <file> <engramURL> # descriptor -> engram node (geometry)
|
||||
periph voiceprint <voice.wav> # extract F0 + formants F1-F5
|
||||
periph imitate <voice.wav> <out.wav> # speak back in that voice (LPC resynthesis)
|
||||
periph hear-imitate <sec> <out.wav> # MIC -> signature -> imitate -> SPEAK ALOUD
|
||||
periph converse <manifest.json> [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] [--live-mic]
|
||||
organ grant|revoke <camera|mic> Neuron-level consent
|
||||
organ status consent + device state
|
||||
organ speak <file.wav> play a WAV aloud (efferent)
|
||||
organ tone [hz] [ms] synthesize and play — no file at all
|
||||
organ say <voice> <CODE> [CODE...] fetch voice FROM THE ENGRAM, render, speak
|
||||
organ listen <sec> <out.wav> mic capture 16k mono (afferent)
|
||||
organ see <out.jpg> one camera frame (afferent)
|
||||
organ wav-info <file.wav> WAV geometry
|
||||
organ feat-audio <file.wav> compact audio descriptor (8 numbers)
|
||||
organ feat-image compact scene-geometry from the camera
|
||||
organ voiceprint <voice.wav> F0 + formants F1-F5 (LPC)
|
||||
organ imitate <in.wav> <out.wav> LPC analysis-resynthesis
|
||||
organ hear-imitate <sec> <out.wav> mic -> signature -> imitate -> speak aloud
|
||||
organ ingest-audio <file.wav> descriptor -> engram node (geometry)
|
||||
organ ingest-voice <voice.wav> <n> voiceprint -> engram voice region
|
||||
organ converse <manifest.json> [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume]
|
||||
```
|
||||
|
||||
## The afferent metabolism
|
||||
A capture is never shipped raw. It becomes a **compact descriptor** — the afferent
|
||||
twin of the music instrument-signature:
|
||||
- audio -> `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` (~2400-6000x smaller)
|
||||
- image -> `[w, h, meanRGB, brightness, 3x3 luminance grid]` (~400000x smaller)
|
||||
- voice -> `[F0, F1..F5, bandwidths]` (11 numbers)
|
||||
## Interruptibility
|
||||
|
||||
That descriptor is what the ingest organ (engram `POST /api/nodes`) turns into an
|
||||
embedded node = geometry.
|
||||
`converse` speaks an ordered, salience-tagged **meaning-plan** while listening:
|
||||
|
||||
## Voice by imitation
|
||||
`voiceprint`/`imitate` are own-core LPC (autocorrelation + Levinson-Durbin, order
|
||||
16 @ 16 kHz), formant extraction from the LPC spectral envelope, and source-filter
|
||||
resynthesis (glottal impulse train at F0 through the all-pole formant filter). A
|
||||
voice is grabbed by ear as ~a dozen numbers and spoken back — **no training, no
|
||||
stolen voice.** Measured fidelity on real speech: resynthesized formants match the
|
||||
source within 2-3%. The full phoneme->formant path for *novel* sentences is the
|
||||
speech faculty's seam (`elp` audio surface profile); this engine provides the
|
||||
formant synthesis primitive it renders through.
|
||||
- **barge-in** — output stops at the sample, not at the end of the buffer. The
|
||||
realizer exposes `pause`/`resume` and reports `played_frames` (the real DAC
|
||||
position) precisely so this is possible.
|
||||
- **yield-or-hold** — a decision, not a rule: `hold = salience·0.6 +
|
||||
progress·0.4`, and holding also requires that the interrupter not be
|
||||
high-authority. Otherwise yield, because the polite default is the right one.
|
||||
- **backchannel** — "mm-hm" is brief and low-energy; resume seamlessly.
|
||||
- **resumable** — on yield the remaining plan persists to `.resume.json`;
|
||||
`--resume` picks the thread back up. An interruption should cost a turn, not
|
||||
the content.
|
||||
|
||||
## Interruptibility (native turn-taking)
|
||||
`converse` plays the utterance as an ordered, salience-tagged **meaning-plan**
|
||||
while the mic listens (full-duplex, AEC on so it never barges in on its own voice):
|
||||
- **barge-in**: user speech -> pause on the spot (sample-accurate), not "finish the buffer."
|
||||
- **yield-or-hold**: a decision grounded in the current segment's salience + progress
|
||||
+ the interrupter's authority — YIELD (stop) or HOLD ("hang on, let me finish").
|
||||
- **backchannel** ("mm-hm"): brief/low -> keep going, resume seamlessly.
|
||||
- **resumable**: on yield the remaining plan persists (`.resume.json`); `--resume`
|
||||
picks the thread back up ("as I was saying").
|
||||
Live full-duplex uses `--live-mic` with the OS voice-processing unit (AEC) so
|
||||
Neuron does not barge in on its own voice. `--barge-at` injects the event
|
||||
deterministically for testing.
|
||||
|
||||
Live full-duplex uses `--live-mic` (OS AEC). Injected `--barge-at` drives the
|
||||
decision loop deterministically for testing.
|
||||
```
|
||||
```
|
||||
## Measured against the Swift original
|
||||
|
||||
Same input (`out/mic_room.wav`, 16 kHz mono, 48121 samples), Swift `periph`
|
||||
vs the El organ:
|
||||
|
||||
| | Swift | El |
|
||||
|---|---|---|
|
||||
| seconds | 3.0075625 | 3.0076 |
|
||||
| rms | 0.0047766496761 | 0.004777 |
|
||||
| peak | 0.01806640625 | 0.018066 |
|
||||
| zcr_hz | 416.28395087 | 416.2840 |
|
||||
| centroid_hz | 727.60529169 | 727.6053 |
|
||||
| f0_hz | 400 | 400.0000 |
|
||||
| formants F1–F5 | 1734.375 / 3343.75 / 3875 / 4359.375 / 4468.75 | identical |
|
||||
| bandwidths B1–B5 | 2000 / 2968.75 / 4203.125 / 4687.5 / 5000 | identical |
|
||||
|
||||
Agreement to every printed digit. `imitate` cannot match bit-for-bit because the
|
||||
Swift excites unvoiced frames with `Double.random` — two Swift runs correlate
|
||||
0.957 with **each other**; El correlates **0.958** with Swift. The port is as
|
||||
close to the original as the original is to itself, and the deterministic prefix
|
||||
is bit-identical.
|
||||
|
||||
## Honest status
|
||||
|
||||
- **Works:** speaker (CoreAudio, no `afplay`, no subprocess — verified: zero
|
||||
`afplay`/Swift strings in the binary, no child process during playback), mic
|
||||
capture, camera capture, all descriptors, LPC voiceprint, imitate,
|
||||
hear-imitate, voice fetch/ingest against the engram, converse (yield, hold,
|
||||
yield-to-authority, backchannel, resume — all exercised with real audio).
|
||||
- **Coarse, and labelled so:** a fetched voice is one formant triple with no
|
||||
coarticulation and no prosody. It is an impression, explicitly **not a
|
||||
clone**, and `prov=COARSE` says so on the node.
|
||||
- **Not verified here:** live `--live-mic` barge-in in a real room with a real
|
||||
interrupter. The AEC path is implemented and the deterministic path is proven;
|
||||
the acoustic behaviour is not something a headless run can establish.
|
||||
- **Not in the engram yet:** the structured `Voice` / `VowelTarget` geometry
|
||||
nodes live in the organ's own store and in snapshot files from earlier work,
|
||||
but the **production engram does not carry them**. Getting them there is an
|
||||
ingest, not a code change.
|
||||
- `src/periph.swift` is kept as the reference the port was measured against.
|
||||
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# build.sh — build the El organ.
|
||||
#
|
||||
# El has no import system on this path, so the modules are concatenated in
|
||||
# dependency order (the same thing elp/tests/run.sh does) and handed to elc as
|
||||
# one unit. The two device realizers are then linked in.
|
||||
#
|
||||
# MUST be run from the repo root, or the .psv phoneme geometry will not resolve
|
||||
# and the render silently degrades.
|
||||
set -uo pipefail
|
||||
|
||||
OUT="${1:-./peripheral/organ}"
|
||||
REPO="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO"
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
# Dependency order. The elp modules supply the render (synth_codes) and the
|
||||
# phoneme-geometry read; the organ supplies everything else.
|
||||
cat elp/src/voice-profile.el \
|
||||
elp/src/accent.el \
|
||||
elp/src/voice-ingest.el \
|
||||
elp/src/speech-ingest.el \
|
||||
elp/src/speech.el \
|
||||
peripheral/src/organ.el \
|
||||
peripheral/src/organ_dsp.el \
|
||||
peripheral/src/organ_converse.el \
|
||||
peripheral/src/organ_cli.el \
|
||||
| grep -v '^import ' > "$WORK/organ.el"
|
||||
|
||||
cd "$REPO/lang"
|
||||
./dist/platform/elc "$WORK/organ.el" > "$WORK/organ.c" || { echo "elc failed" >&2; exit 1; }
|
||||
|
||||
SSL_PREFIX="$(brew --prefix openssl@3 2>/dev/null || echo /usr/local)"
|
||||
|
||||
# The peripheral realizers are per-platform: Darwin gets the real devices,
|
||||
# anything else gets el_peripheral_null.c and honestly reports having none.
|
||||
case "$(uname)" in
|
||||
Darwin)
|
||||
# The Objective-C realizers are compiled SEPARATELY, with -fobjc-arc. The
|
||||
# capture realizer is written against ARC (it holds AVFoundation objects);
|
||||
# compiling it MRR silently changes its memory semantics, which on a device
|
||||
# path shows up as a use-after-free under load rather than as an error here.
|
||||
cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_audio_darwin.m -o "$WORK/el_audio.o" || exit 1
|
||||
cc -std=c11 -fobjc-arc -O1 -I runtime -c runtime/el_capture_darwin.m -o "$WORK/el_capture.o" || exit 1
|
||||
PERIPH_SRC="$WORK/el_audio.o $WORK/el_capture.o"
|
||||
PERIPH_LIBS="-framework AudioToolbox -framework AVFoundation -framework CoreMedia
|
||||
-framework CoreVideo -framework CoreGraphics -framework ImageIO
|
||||
-framework Foundation"
|
||||
;;
|
||||
*)
|
||||
PERIPH_SRC="runtime/el_peripheral_null.c"
|
||||
PERIPH_LIBS=""
|
||||
;;
|
||||
esac
|
||||
|
||||
cc -O1 -I runtime -I"$SSL_PREFIX/include" -L"$SSL_PREFIX/lib" \
|
||||
-o "$OUT" "$WORK/organ.c" \
|
||||
runtime/el_runtime.c runtime/el_seed.c \
|
||||
runtime/engram_cognition.c runtime/engram_geometry.c runtime/engram_reason.c \
|
||||
runtime/engram_store.c runtime/engram_verify.c runtime/engram_vindex.c \
|
||||
runtime/eg_cosine_batch.c runtime/eg_cosine_batch_strategy_cpu.c \
|
||||
$PERIPH_SRC $PERIPH_LIBS \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm || { echo "link failed" >&2; exit 1; }
|
||||
|
||||
echo "built: $OUT"
|
||||
@@ -0,0 +1,570 @@
|
||||
// organ.el — Neuron's I/O organ, in El.
|
||||
//
|
||||
// THE PRINCIPLE. El speaks. The engram stores geometry and does not speak.
|
||||
// Before this file the organ was a 939-line Swift program standing next to the
|
||||
// language (peripheral/src/periph.swift): Neuron's mouth and ears were a
|
||||
// separate binary, and "speak" meant "shell out to that binary, which shells
|
||||
// out to afplay." That is not a voice, it is a subprocess. The voice belongs to
|
||||
// the language and its runtime.
|
||||
//
|
||||
// THE SPLIT. Exactly two things here are not El, and they are the two things El
|
||||
// cannot express as arithmetic:
|
||||
//
|
||||
// the speaker — handing a buffer to the DAC and waiting for it to drain
|
||||
// the capture — asking the OS for samples off a mic or frames off a camera
|
||||
//
|
||||
// Those live in lang/runtime/el_audio_darwin.m and el_capture_darwin.m, as
|
||||
// their own translation units, declared in el_runtime.h. Everything ELSE that
|
||||
// the Swift did — WAV encode and decode, LPC autocorrelation, Levinson-Durbin,
|
||||
// formant extraction off the all-pole envelope, source-filter resynthesis, the
|
||||
// compact descriptors, the converse yield-or-hold decision — is arithmetic, and
|
||||
// arithmetic is El's. See organ_dsp.el for that half.
|
||||
//
|
||||
// WHERE THE VOICE COMES FROM. Not from this file, and not from a JSON manifest
|
||||
// on disk. A voice is GEOMETRY IN THE ENGRAM, and the organ goes and gets it by
|
||||
// asking the engram, the same way anything else asks the engram for anything:
|
||||
// a query against the graph, then read the numbers off the node that comes
|
||||
// back. organ_voice_fetch is that. The previous path, load_voice("...json"),
|
||||
// parsed a file — which quietly made the voice a build artifact instead of a
|
||||
// memory. If the region is not in the graph, the honest answer is an empty
|
||||
// result, not a default voice.
|
||||
//
|
||||
// WHAT THE ORGAN NEVER DOES. It never learns a word. Pronunciation, vocabulary
|
||||
// and phonemes are the language faculty's, already built as ingested geometry —
|
||||
// "the engram knows how to pronounce." The seam is synth_codes(codes, voice,
|
||||
// pmap): the codes and the phoneme map arrive from the language side as
|
||||
// geometry, and the organ's whole job is turning them into samples and getting
|
||||
// the samples out the speaker, plus the same trip in reverse for the senses.
|
||||
//
|
||||
// RAILS, all non-negotiable:
|
||||
// own-core — CoreAudio / AVFoundation / ImageIO, all shipped with the OS.
|
||||
// No cloud, no model, no heavy dependency. There is no network
|
||||
// call anywhere in the organ, by construction.
|
||||
// local-only — raw streams stay on the machine. What leaves a capture is a
|
||||
// DESCRIPTOR of a few dozen numbers, never the stream.
|
||||
// consent — two locks on the sensitive senses: a Neuron-level grant AND
|
||||
// the OS TCC permission. Camera and mic FAIL CLOSED without
|
||||
// both. The speaker is disclosed but not gated (see below).
|
||||
// disclosed — every device touch prints a [peripheral] line on stderr.
|
||||
// Nothing here is ever silent about being a device.
|
||||
|
||||
// ── Disclosure ───────────────────────────────────────────────────────────────
|
||||
//
|
||||
// stderr, not stdout: a program that announces "I am opening the microphone" on
|
||||
// stdout has corrupted its own output. And flushed immediately, so the line is
|
||||
// on the terminal BEFORE the device is touched — a disclosure that arrives
|
||||
// after the fact is a log, not a disclosure.
|
||||
|
||||
fn organ_disclose(msg: String) -> Bool {
|
||||
eprintln(" [peripheral] " + msg)
|
||||
return true
|
||||
}
|
||||
|
||||
// ── Consent, the Neuron-level lock ───────────────────────────────────────────
|
||||
//
|
||||
// The OS has its own lock (TCC) and it is not enough on its own: TCC grants the
|
||||
// TERMINAL access to the microphone, once, more or less forever. That says the
|
||||
// user trusts the app. It does not say the user consents to THIS program
|
||||
// listening THIS time. So Neuron keeps its own grant, revocable, on the same
|
||||
// footing — and both must be open for a sensitive sense to work.
|
||||
//
|
||||
// Stored next to the organ rather than in the engram deliberately: consent must
|
||||
// be inspectable and revocable without a running graph, and a permission that
|
||||
// can only be revoked by the system it governs is not a permission.
|
||||
|
||||
fn organ_consent_path() -> String {
|
||||
let home: String = env("PERIPH_HOME")
|
||||
if str_eq(home, "") {
|
||||
return "peripheral/.consent.json"
|
||||
}
|
||||
return home + "/.consent.json"
|
||||
}
|
||||
|
||||
fn organ_consent_granted(device: String) -> Bool {
|
||||
let raw: String = fs_read(organ_consent_path())
|
||||
if str_eq(raw, "") {
|
||||
return false
|
||||
}
|
||||
// A device is granted only on an explicit true. Anything unparseable,
|
||||
// missing or malformed reads as NOT granted — the failure direction for a
|
||||
// permission file is always closed.
|
||||
let key: String = "\"" + device + "\""
|
||||
let at: Int = str_index_of(raw, key)
|
||||
if at < 0 {
|
||||
return false
|
||||
}
|
||||
let tail: String = str_slice(raw, at, str_len(raw))
|
||||
let t: Int = str_index_of(tail, "true")
|
||||
let f: Int = str_index_of(tail, "false")
|
||||
if t < 0 {
|
||||
return false
|
||||
}
|
||||
if f < 0 {
|
||||
return true
|
||||
}
|
||||
// whichever token appears first after the key is this device's value
|
||||
if t < f {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
fn organ_consent_write(camera: Bool, mic: Bool) -> Bool {
|
||||
let c: String = "false"
|
||||
if camera {
|
||||
c = "true"
|
||||
}
|
||||
let m: String = "false"
|
||||
if mic {
|
||||
m = "true"
|
||||
}
|
||||
return fs_write(organ_consent_path(), "{\"camera\": " + c + ", \"mic\": " + m + "}\n")
|
||||
}
|
||||
|
||||
fn organ_grant(device: String) -> Bool {
|
||||
let cam: Bool = organ_consent_granted("camera")
|
||||
let mic: Bool = organ_consent_granted("mic")
|
||||
if str_eq(device, "camera") {
|
||||
cam = true
|
||||
}
|
||||
if str_eq(device, "mic") {
|
||||
mic = true
|
||||
}
|
||||
let ok: Bool = organ_consent_write(cam, mic)
|
||||
organ_disclose("granted '" + device + "' (Neuron-level) — raw stream stays local, never egresses.")
|
||||
return ok
|
||||
}
|
||||
|
||||
fn organ_revoke(device: String) -> Bool {
|
||||
let cam: Bool = organ_consent_granted("camera")
|
||||
let mic: Bool = organ_consent_granted("mic")
|
||||
if str_eq(device, "camera") {
|
||||
cam = false
|
||||
}
|
||||
if str_eq(device, "mic") {
|
||||
mic = false
|
||||
}
|
||||
let ok: Bool = organ_consent_write(cam, mic)
|
||||
organ_disclose("revoked '" + device + "' (Neuron-level).")
|
||||
return ok
|
||||
}
|
||||
|
||||
fn organ_consent_status() -> String {
|
||||
let cam: String = "denied"
|
||||
if organ_consent_granted("camera") {
|
||||
cam = "granted"
|
||||
}
|
||||
let mic: String = "denied"
|
||||
if organ_consent_granted("mic") {
|
||||
mic = "granted"
|
||||
}
|
||||
return "camera=" + cam + " mic=" + mic
|
||||
}
|
||||
|
||||
// Both locks, in order, with a disclosure for each outcome. Returns false and
|
||||
// says exactly which lock is shut — a refusal that does not say why is
|
||||
// indistinguishable from a bug.
|
||||
fn organ_may_listen() -> Bool {
|
||||
if organ_consent_granted("mic") == false {
|
||||
organ_disclose("CONSENT DENIED for 'mic' (Neuron-level). Run: organ grant mic")
|
||||
return false
|
||||
}
|
||||
if mic_available() == 0 {
|
||||
organ_disclose("CONSENT DENIED for 'mic' (OS/TCC), or no input device. Grant microphone access to this terminal in System Settings > Privacy.")
|
||||
return false
|
||||
}
|
||||
organ_disclose("consent OK (Neuron + OS) for 'mic' — local only, never egresses.")
|
||||
return true
|
||||
}
|
||||
|
||||
fn organ_may_see() -> Bool {
|
||||
if organ_consent_granted("camera") == false {
|
||||
organ_disclose("CONSENT DENIED for 'camera' (Neuron-level). Run: organ grant camera")
|
||||
return false
|
||||
}
|
||||
if camera_available() == 0 {
|
||||
organ_disclose("CONSENT DENIED for 'camera' (OS/TCC), or no capture device. Grant camera access to this terminal in System Settings > Privacy.")
|
||||
return false
|
||||
}
|
||||
organ_disclose("consent OK (Neuron + OS) for 'camera' — local only, never egresses.")
|
||||
return true
|
||||
}
|
||||
|
||||
// ── SPEAKER (efferent) ───────────────────────────────────────────────────────
|
||||
//
|
||||
// Not consent-gated, and that is a deliberate asymmetry rather than an
|
||||
// oversight. The microphone and camera take information OFF the user without
|
||||
// them necessarily knowing; the speaker puts information INTO a room the user
|
||||
// is in, audibly, which is self-disclosing by its nature — you cannot secretly
|
||||
// speak aloud. So the speaker is DISCLOSED (every utterance announces itself on
|
||||
// stderr) but not gated. Gating it would mean Neuron needs permission to answer.
|
||||
|
||||
fn organ_speak_samples(samples: [Int], sr: Int) -> Bool {
|
||||
let n: Int = native_list_len(samples)
|
||||
if n <= 0 {
|
||||
organ_disclose("SPEAKER: nothing to say (0 samples) — not touching the device.")
|
||||
return false
|
||||
}
|
||||
if speaker_available() == 0 {
|
||||
organ_disclose("SPEAKER: no audio output on this build (" + speaker_name() + ") — cannot speak.")
|
||||
return false
|
||||
}
|
||||
let secs: Int = n * 1000 / sr
|
||||
organ_disclose("SPEAKER: playing " + int_to_str(n) + " samples (" + int_to_str(secs) + " ms @ " + int_to_str(sr) + " Hz) ALOUD via " + speaker_name() + " (efferent).")
|
||||
let ok: Int = speaker_play_pcm16(samples, sr)
|
||||
if ok == 1 {
|
||||
organ_disclose("SPEAKER: done — Neuron spoke aloud.")
|
||||
return true
|
||||
}
|
||||
organ_disclose("SPEAKER: playback FAILED.")
|
||||
return false
|
||||
}
|
||||
|
||||
fn organ_speak_wav(path: String) -> Bool {
|
||||
if speaker_available() == 0 {
|
||||
organ_disclose("SPEAKER: no audio output on this build — cannot speak.")
|
||||
return false
|
||||
}
|
||||
if fs_exists(path) == false {
|
||||
organ_disclose("SPEAKER: no such file: " + path)
|
||||
return false
|
||||
}
|
||||
organ_disclose("SPEAKER: playing '" + path + "' ALOUD via " + speaker_name() + " (efferent).")
|
||||
let ok: Int = speaker_play_wav(path)
|
||||
if ok == 1 {
|
||||
organ_disclose("SPEAKER: done — Neuron spoke aloud.")
|
||||
return true
|
||||
}
|
||||
organ_disclose("SPEAKER: playback FAILED.")
|
||||
return false
|
||||
}
|
||||
|
||||
// ── The voice, fetched FROM THE ENGRAM ───────────────────────────────────────
|
||||
//
|
||||
// This is the part that matters most and is easiest to get subtly wrong. A
|
||||
// voice is not a constant in code and it is not a JSON file next to the code —
|
||||
// it is a region of the graph, put there by having heard someone, and the organ
|
||||
// retrieves it the way anything retrieves a memory: by asking.
|
||||
//
|
||||
// The node content is the geometry, in the engram's own flat key=value form:
|
||||
// voice will | f0=137 f0_end=116 kf=1269 f1=500 f2=2093 f3=3531 ...
|
||||
// so the read is: query the graph, take the returned node, pull the numbers off
|
||||
// it. Nothing here opens a file.
|
||||
//
|
||||
// Returns [f0, f0_end, kf, f1, f2, f3], or an EMPTY list when the region is not
|
||||
// in the graph. Empty is the honest answer — a caller that gets no voice must
|
||||
// not be handed a plausible default and left unable to tell the difference
|
||||
// between "this is how they sound" and "I never heard them."
|
||||
|
||||
// Read an unsigned integer that follows `key` in `s`. Stops at the first
|
||||
// non-digit, returns 0 when the key is absent.
|
||||
fn organ_int_after(s: String, key: String) -> Int {
|
||||
let at: Int = str_index_of(s, key)
|
||||
if at < 0 {
|
||||
return 0
|
||||
}
|
||||
let i: Int = at + str_len(key)
|
||||
let n: Int = str_len(s)
|
||||
let v: Int = 0
|
||||
let seen: Int = 0
|
||||
while i < n {
|
||||
let c: Int = str_char_code(s, i)
|
||||
if c < 48 {
|
||||
i = n
|
||||
} else {
|
||||
if c > 57 {
|
||||
i = n
|
||||
} else {
|
||||
v = v * 10 + (c - 48)
|
||||
seen = seen + 1
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
if seen == 0 {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// Ask the engram for a named voice region and read its geometry back.
|
||||
fn organ_voice_fetch(name: String) -> [Int] {
|
||||
let out: [Int] = native_list_empty()
|
||||
let marker: String = "voice " + name + " |"
|
||||
// The graph is asked by MEANING, not by id or by path.
|
||||
let hits: String = engram_search_json("voice " + name + " f0 formants", 12)
|
||||
let at: Int = str_index_of(hits, marker)
|
||||
if at < 0 {
|
||||
// Fall back to a scan of the resident graph before giving up: search is
|
||||
// geometric and a small graph may not rank the region first.
|
||||
let scan: String = engram_scan_nodes_json(500, 0)
|
||||
at = str_index_of(scan, marker)
|
||||
if at < 0 {
|
||||
organ_disclose("VOICE: no region for '" + name + "' in the engram — nothing to speak with.")
|
||||
return out
|
||||
}
|
||||
hits = scan
|
||||
}
|
||||
let win: String = str_slice(hits, at, at + 240)
|
||||
out = native_list_append(out, organ_int_after(win, "f0="))
|
||||
out = native_list_append(out, organ_int_after(win, "f0_end="))
|
||||
out = native_list_append(out, organ_int_after(win, "kf="))
|
||||
out = native_list_append(out, organ_int_after(win, "f1="))
|
||||
out = native_list_append(out, organ_int_after(win, "f2="))
|
||||
out = native_list_append(out, organ_int_after(win, "f3="))
|
||||
organ_disclose("VOICE: fetched '" + name + "' FROM THE ENGRAM — f0=" + int_to_str(native_list_get(out, 0)) + " f0_end=" + int_to_str(native_list_get(out, 1)) + " kf=" + int_to_str(native_list_get(out, 2)) + " f1=" + int_to_str(native_list_get(out, 3)) + " f2=" + int_to_str(native_list_get(out, 4)) + " f3=" + int_to_str(native_list_get(out, 5)))
|
||||
return out
|
||||
}
|
||||
|
||||
// Put a measured voice INTO the engram as geometry. This is the afferent end of
|
||||
// the same wire: a voiceprint (organ_dsp.el's LPC analysis) becomes a node, and
|
||||
// from then on the voice is a memory rather than a measurement someone happened
|
||||
// to write down. `prov` carries the honesty: COARSE means one formant triple, no
|
||||
// coarticulation, no prosody — an impression, explicitly not a clone.
|
||||
fn organ_voice_ingest(name: String, f0: Int, f0_end: Int, kf: Int, f1: Int, f2: Int, f3: Int, src: String, prov: String) -> String {
|
||||
let hub: String = engram_node("voice-signature-set " + name + " grounding=measured src=" + src, "VoiceSet", 90)
|
||||
let body: String = "voice " + name + " | f0=" + int_to_str(f0) + " f0_end=" + int_to_str(f0_end) + " kf=" + int_to_str(kf) + " f1=" + int_to_str(f1) + " f2=" + int_to_str(f2) + " f3=" + int_to_str(f3) + " grounding=measured src=" + src + " prov=" + prov
|
||||
let vid: String = engram_node(body, "Voice", 90)
|
||||
engram_connect(hub, vid, 90, "has-signature")
|
||||
organ_disclose("VOICE: ingested '" + name + "' into the engram as geometry (node " + vid + ").")
|
||||
return vid
|
||||
}
|
||||
|
||||
// Turn the fetched geometry into the voice slot-map the render consumes. Kept
|
||||
// separate from the fetch so the organ never invents a voice: if the fetch came
|
||||
// back empty this returns empty too, and the caller has to deal with it.
|
||||
//
|
||||
// The slot-map is built here rather than by calling the render's own
|
||||
// constructor, so the organ carries NO dependency on the language faculty's
|
||||
// modules — it only has to agree with them about a wire format, which is the
|
||||
// looser and more honest coupling. (The layout is the same key/value [String]
|
||||
// convention lang_get / surface_get / voice_get all read.)
|
||||
fn organ_voice_profile(name: String, g: [Int]) -> [String] {
|
||||
let r: [String] = native_list_empty()
|
||||
if native_list_len(g) < 6 {
|
||||
return r
|
||||
}
|
||||
r = native_list_append(r, "name")
|
||||
r = native_list_append(r, name)
|
||||
r = native_list_append(r, "f0")
|
||||
r = native_list_append(r, int_to_str(native_list_get(g, 0)))
|
||||
r = native_list_append(r, "f0_end")
|
||||
r = native_list_append(r, int_to_str(native_list_get(g, 1)))
|
||||
r = native_list_append(r, "kf")
|
||||
r = native_list_append(r, int_to_str(native_list_get(g, 2)))
|
||||
r = native_list_append(r, "dur")
|
||||
r = native_list_append(r, "1000")
|
||||
r = native_list_append(r, "tilt")
|
||||
r = native_list_append(r, "1000")
|
||||
r = native_list_append(r, "breath")
|
||||
r = native_list_append(r, "8")
|
||||
return r
|
||||
}
|
||||
|
||||
// ── Scene geometry (afferent, camera) ────────────────────────────────────────
|
||||
//
|
||||
// The image half of the afferent metabolism, and the same principle as the
|
||||
// audio descriptor: a frame is never handed on raw. The realizer returns a
|
||||
// small pixel grid; THIS computes the descriptor, in El, because averaging
|
||||
// pixels is arithmetic and arithmetic is not a device concern.
|
||||
//
|
||||
// Returns 15 numbers — [w, h, meanR, meanG, meanB, brightness_pm, and a 3x3
|
||||
// luminance grid] — standing in for a multi-megapixel frame. The 3x3 grid is
|
||||
// the smallest thing that still says WHERE the light is, which is most of what
|
||||
// makes a scene comparable to another scene; a single brightness average would
|
||||
// make a lamp on the left indistinguishable from a lamp on the right.
|
||||
//
|
||||
// Luminance is Rec. 601 (0.299R + 0.587G + 0.114B), in integer per-mille, so
|
||||
// the descriptor is reproducible rather than subject to float drift.
|
||||
fn organ_image_descriptor() -> [Int] {
|
||||
let out: [Int] = native_list_empty()
|
||||
let frame: Any = camera_capture_rgb()
|
||||
if frame == 0 {
|
||||
return out
|
||||
}
|
||||
let w: Int = el_map_get(frame, "width")
|
||||
let h: Int = el_map_get(frame, "height")
|
||||
let gw: Int = el_map_get(frame, "grid_w")
|
||||
let gh: Int = el_map_get(frame, "grid_h")
|
||||
let px: [Int] = el_map_get(frame, "pixels")
|
||||
let np: Int = native_list_len(px)
|
||||
if np < 3 {
|
||||
return out
|
||||
}
|
||||
let count: Int = np / 3
|
||||
let rsum: Int = 0
|
||||
let gsum: Int = 0
|
||||
let bsum: Int = 0
|
||||
// 3x3 accumulators, row-major
|
||||
let cell: [Int] = native_list_empty()
|
||||
let cn: [Int] = native_list_empty()
|
||||
let z: Int = 0
|
||||
while z < 9 {
|
||||
cell = native_list_append(cell, 0)
|
||||
cn = native_list_append(cn, 0)
|
||||
z = z + 1
|
||||
}
|
||||
// El has no list-set, so the cells are summed into parallel scalars and
|
||||
// reassembled — nine explicit accumulators would be worse to read than one
|
||||
// pass per cell over a grid this small.
|
||||
let c0: Int = 0
|
||||
let c1: Int = 0
|
||||
let c2: Int = 0
|
||||
let c3: Int = 0
|
||||
let c4: Int = 0
|
||||
let c5: Int = 0
|
||||
let c6: Int = 0
|
||||
let c7: Int = 0
|
||||
let c8: Int = 0
|
||||
let n0: Int = 0
|
||||
let n1: Int = 0
|
||||
let n2: Int = 0
|
||||
let n3: Int = 0
|
||||
let n4: Int = 0
|
||||
let n5: Int = 0
|
||||
let n6: Int = 0
|
||||
let n7: Int = 0
|
||||
let n8: Int = 0
|
||||
let i: Int = 0
|
||||
while i < count {
|
||||
let r: Int = native_list_get(px, i * 3)
|
||||
let g: Int = native_list_get(px, i * 3 + 1)
|
||||
let b: Int = native_list_get(px, i * 3 + 2)
|
||||
rsum = rsum + r
|
||||
gsum = gsum + g
|
||||
bsum = bsum + b
|
||||
let lum: Int = (299 * r + 587 * g + 114 * b) / 1000
|
||||
let x: Int = i - (i / gw) * gw
|
||||
let y: Int = i / gw
|
||||
let cx: Int = x * 3 / gw
|
||||
let cy: Int = y * 3 / gh
|
||||
if cx > 2 {
|
||||
cx = 2
|
||||
}
|
||||
if cy > 2 {
|
||||
cy = 2
|
||||
}
|
||||
let idx: Int = cy * 3 + cx
|
||||
if idx == 0 {
|
||||
c0 = c0 + lum
|
||||
n0 = n0 + 1
|
||||
}
|
||||
if idx == 1 {
|
||||
c1 = c1 + lum
|
||||
n1 = n1 + 1
|
||||
}
|
||||
if idx == 2 {
|
||||
c2 = c2 + lum
|
||||
n2 = n2 + 1
|
||||
}
|
||||
if idx == 3 {
|
||||
c3 = c3 + lum
|
||||
n3 = n3 + 1
|
||||
}
|
||||
if idx == 4 {
|
||||
c4 = c4 + lum
|
||||
n4 = n4 + 1
|
||||
}
|
||||
if idx == 5 {
|
||||
c5 = c5 + lum
|
||||
n5 = n5 + 1
|
||||
}
|
||||
if idx == 6 {
|
||||
c6 = c6 + lum
|
||||
n6 = n6 + 1
|
||||
}
|
||||
if idx == 7 {
|
||||
c7 = c7 + lum
|
||||
n7 = n7 + 1
|
||||
}
|
||||
if idx == 8 {
|
||||
c8 = c8 + lum
|
||||
n8 = n8 + 1
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
let rA: Int = rsum / count
|
||||
let gA: Int = gsum / count
|
||||
let bA: Int = bsum / count
|
||||
let bright: Int = (299 * rA + 587 * gA + 114 * bA) / 255
|
||||
out = native_list_append(out, w)
|
||||
out = native_list_append(out, h)
|
||||
out = native_list_append(out, rA)
|
||||
out = native_list_append(out, gA)
|
||||
out = native_list_append(out, bA)
|
||||
out = native_list_append(out, bright)
|
||||
if n0 < 1 {
|
||||
n0 = 1
|
||||
}
|
||||
if n1 < 1 {
|
||||
n1 = 1
|
||||
}
|
||||
if n2 < 1 {
|
||||
n2 = 1
|
||||
}
|
||||
if n3 < 1 {
|
||||
n3 = 1
|
||||
}
|
||||
if n4 < 1 {
|
||||
n4 = 1
|
||||
}
|
||||
if n5 < 1 {
|
||||
n5 = 1
|
||||
}
|
||||
if n6 < 1 {
|
||||
n6 = 1
|
||||
}
|
||||
if n7 < 1 {
|
||||
n7 = 1
|
||||
}
|
||||
if n8 < 1 {
|
||||
n8 = 1
|
||||
}
|
||||
out = native_list_append(out, c0 / n0)
|
||||
out = native_list_append(out, c1 / n1)
|
||||
out = native_list_append(out, c2 / n2)
|
||||
out = native_list_append(out, c3 / n3)
|
||||
out = native_list_append(out, c4 / n4)
|
||||
out = native_list_append(out, c5 / n5)
|
||||
out = native_list_append(out, c6 / n6)
|
||||
out = native_list_append(out, c7 / n7)
|
||||
out = native_list_append(out, c8 / n8)
|
||||
organ_disclose("FEAT(image): 15-number scene-geometry vs " + int_to_str(w * h * 3) + " pixel-channels — the descriptor travels, the frame does not.")
|
||||
return out
|
||||
}
|
||||
|
||||
// ── Own-core tone ────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The smallest possible proof that the organ owns its medium end to end: a sine
|
||||
// with a gentle attack and release, computed here, played by us, no file and no
|
||||
// library anywhere in the path.
|
||||
fn organ_tone(hz: Int, ms: Int, sr: Int) -> [Int] {
|
||||
let n: Int = sr * ms / 1000
|
||||
let out: [Int] = native_list_empty()
|
||||
let two_pi: Float = 6.283185307
|
||||
let srf: Float = int_to_float(sr)
|
||||
let hzf: Float = int_to_float(hz)
|
||||
let i: Int = 0
|
||||
// 20 ms of ramp at each end; a square-edged tone clicks, and a click is the
|
||||
// organ announcing that it does not understand envelopes.
|
||||
let ramp: Int = sr / 50
|
||||
if ramp < 1 {
|
||||
ramp = 1
|
||||
}
|
||||
while i < n {
|
||||
let t: Float = int_to_float(i) / srf
|
||||
let s: Float = math_sin(two_pi * hzf * t)
|
||||
let env: Int = 32767
|
||||
if i < ramp {
|
||||
env = 32767 * i / ramp
|
||||
}
|
||||
let tail: Int = n - i
|
||||
if tail < ramp {
|
||||
env = 32767 * tail / ramp
|
||||
}
|
||||
let v: Int = float_to_int(s * 9000.0) * env / 32767
|
||||
out = native_list_append(out, v)
|
||||
i = i + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
// organ_cli.el — the organ's command surface. main() lives here.
|
||||
//
|
||||
// One binary, the same verbs the Swift program had, and nothing behind them
|
||||
// except El and two thin device realizers. This file is the proof surface: if
|
||||
// `organ speak` makes a sound and no Swift binary is in the process tree, the
|
||||
// claim in organ.el's header is true.
|
||||
//
|
||||
// Verbs, and what each one demonstrates:
|
||||
//
|
||||
// grant/revoke/status the Neuron-level consent lock, inspectable
|
||||
// speak <wav> efferent — audio out of El's own speaker
|
||||
// tone <hz> <ms> own-core synthesis: computed in El, played by El,
|
||||
// never touching the disk
|
||||
// say <name> <codes...> fetch a VOICE FROM THE ENGRAM and render through it
|
||||
// listen <sec> <out> afferent — mic capture, consent-gated, fails closed
|
||||
// see <out.jpg> afferent — one camera frame, same two locks
|
||||
// wav-info <wav> WAV geometry, parsed in El
|
||||
// feat-audio <wav> capture -> compact descriptor (8 numbers)
|
||||
// feat-image <jpg> frame -> compact scene-geometry
|
||||
// voiceprint <wav> F0 + formants F1-F5 by LPC, in El
|
||||
// imitate <in> <out> LPC analysis-resynthesis, in El
|
||||
// hear-imitate <sec> the closed loop: hear a voice, take its signature,
|
||||
// speak back in it
|
||||
// ingest-audio <wav> descriptor -> engram node (the capture becomes geometry)
|
||||
// ingest-voice <wav> <n> voiceprint -> engram voice region (how a voice is learned)
|
||||
// converse <manifest> full-duplex interruptible utterance
|
||||
//
|
||||
// The descriptors are the point of the afferent half. A capture is NEVER handed
|
||||
// on raw: a three-second recording is ~48,000 samples and what leaves this
|
||||
// process is eight numbers. That is both the privacy rail (the stream stays
|
||||
// local because only its shape travels) and the reason the engram can hold a
|
||||
// perception at all — geometry is storable, a waveform is not.
|
||||
|
||||
fn cli_usage() -> Bool {
|
||||
println("organ — Neuron's I/O organ, native El (own-core, local, consent-gated)")
|
||||
println(" grant|revoke <camera|mic> Neuron-level consent")
|
||||
println(" status consent + device state")
|
||||
println(" speak <file.wav> play a WAV aloud (efferent)")
|
||||
println(" tone [hz] [ms] synthesize and play, no file at all")
|
||||
println(" say <voice> <CODE> [CODE...] fetch voice FROM THE ENGRAM, render, speak")
|
||||
println(" listen <sec> <out.wav> mic capture 16k mono (afferent)")
|
||||
println(" see <out.jpg> one camera frame (afferent)")
|
||||
println(" wav-info <file.wav> WAV geometry")
|
||||
println(" feat-audio <file.wav> compact audio descriptor (8 numbers)")
|
||||
println(" feat-image compact scene-geometry from the camera")
|
||||
println(" voiceprint <voice.wav> F0 + formants F1-F5 (LPC)")
|
||||
println(" imitate <in.wav> <out.wav> LPC analysis-resynthesis")
|
||||
println(" hear-imitate <sec> <out.wav> mic -> signature -> imitate -> speak aloud")
|
||||
println(" ingest-audio <file.wav> descriptor -> engram node (geometry)")
|
||||
println(" ingest-voice <voice.wav> <n> voiceprint -> engram voice region")
|
||||
println(" converse <manifest.json> [--authority PM] [--barge-at MS[:kind]] [--live-mic] [--resume]")
|
||||
return true
|
||||
}
|
||||
|
||||
// The engram the organ reads and writes. Its own store, never production's.
|
||||
fn cli_engram_dir() -> String {
|
||||
let d: String = env("ORGAN_ENGRAM")
|
||||
if str_eq(d, "") {
|
||||
return "peripheral/.engram"
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
fn cli_open_engram() -> Bool {
|
||||
let dir: String = cli_engram_dir()
|
||||
fs_mkdir(dir)
|
||||
let ok: Int = engram_store_boot(dir)
|
||||
if ok == 1 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ── formatting helpers ───────────────────────────────────────────────────────
|
||||
|
||||
fn cli_f(v: Float, dec: Int) -> String {
|
||||
return format_float(v, dec)
|
||||
}
|
||||
|
||||
// ── the descriptor, printed and ingested ─────────────────────────────────────
|
||||
//
|
||||
// [seconds, sr, ch, rms, peak, zcr, centroid, f0] — the same eight numbers the
|
||||
// Swift produced, computed in El, and the compression ratio is the headline:
|
||||
// a few dozen bytes standing in for a few hundred kilobytes.
|
||||
fn cli_audio_descriptor_text(v: [Float], path: String) -> String {
|
||||
let secs: Float = native_list_get(v, 0)
|
||||
let sr: Float = native_list_get(v, 1)
|
||||
let ch: Float = native_list_get(v, 2)
|
||||
let rms: Float = native_list_get(v, 3)
|
||||
let peak: Float = native_list_get(v, 4)
|
||||
let zcr: Float = native_list_get(v, 5)
|
||||
let cen: Float = native_list_get(v, 6)
|
||||
let f0: Float = native_list_get(v, 7)
|
||||
return "Heard sound (afferent, mic): " + cli_f(secs, 2) + "s at " + cli_f(sr, 0) + "Hz. RMS energy " + cli_f(rms, 4) + ", peak " + cli_f(peak, 4) + ", zero-crossing rate " + cli_f(zcr, 0) + "Hz, spectral centroid " + cli_f(cen, 0) + "Hz, estimated voice pitch F0 " + cli_f(f0, 0) + "Hz. Compact voice/sound signature (8 numbers) — phonetic geometry seed."
|
||||
}
|
||||
|
||||
fn cli_feat_audio(path: String) -> Bool {
|
||||
let v: [Float] = dsp_compute_audio(path)
|
||||
if native_list_len(v) < 8 {
|
||||
println("{\"ok\": false, \"op\": \"feat-audio\", \"error\": \"cannot read PCM\"}")
|
||||
return false
|
||||
}
|
||||
organ_disclose("FEAT(audio): 8-number signature vs " + int_to_str(float_to_int(native_list_get(v, 0) * native_list_get(v, 1))) + " raw samples.")
|
||||
println("{\"ok\": true, \"op\": \"feat-audio\", \"file\": \"" + path + "\", \"seconds\": " + cli_f(native_list_get(v, 0), 4) + ", \"sample_rate\": " + cli_f(native_list_get(v, 1), 0) + ", \"channels\": " + cli_f(native_list_get(v, 2), 0) + ", \"rms\": " + cli_f(native_list_get(v, 3), 6) + ", \"peak\": " + cli_f(native_list_get(v, 4), 6) + ", \"zcr_hz\": " + cli_f(native_list_get(v, 5), 4) + ", \"centroid_hz\": " + cli_f(native_list_get(v, 6), 4) + ", \"f0_hz\": " + cli_f(native_list_get(v, 7), 4) + "}")
|
||||
return true
|
||||
}
|
||||
|
||||
fn cli_voiceprint(path: String) -> Bool {
|
||||
let v: [Float] = dsp_voiceprint(path)
|
||||
if native_list_len(v) < 4 {
|
||||
println("{\"ok\": false, \"op\": \"voiceprint\", \"error\": \"cannot read speech\"}")
|
||||
return false
|
||||
}
|
||||
let nf: Int = float_to_int(native_list_get(v, 3))
|
||||
let fs: String = ""
|
||||
let bs: String = ""
|
||||
let i: Int = 0
|
||||
while i < nf {
|
||||
if i > 0 {
|
||||
fs = fs + ", "
|
||||
bs = bs + ", "
|
||||
}
|
||||
fs = fs + cli_f(native_list_get(v, 4 + i * 2), 3)
|
||||
bs = bs + cli_f(native_list_get(v, 5 + i * 2), 3)
|
||||
i = i + 1
|
||||
}
|
||||
println("{\"ok\": true, \"op\": \"voiceprint\", \"file\": \"" + path + "\", \"f0_hz\": " + cli_f(native_list_get(v, 0), 4) + ", \"f0_range\": [" + cli_f(native_list_get(v, 1), 4) + ", " + cli_f(native_list_get(v, 2), 4) + "], \"formants_hz\": [" + fs + "], \"bandwidths_hz\": [" + bs + "]}")
|
||||
return true
|
||||
}
|
||||
|
||||
// ── main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn main() {
|
||||
let a: [String] = args()
|
||||
let n: Int = native_list_len(a)
|
||||
if n < 1 {
|
||||
cli_usage()
|
||||
return
|
||||
}
|
||||
let cmd: String = native_list_get(a, 0)
|
||||
|
||||
// ---- consent -----------------------------------------------------------
|
||||
if str_eq(cmd, "grant") {
|
||||
if n < 2 {
|
||||
println("grant needs a device")
|
||||
return
|
||||
}
|
||||
organ_grant(native_list_get(a, 1))
|
||||
println("{\"ok\": true, \"op\": \"grant\", \"consent\": \"" + organ_consent_status() + "\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "revoke") {
|
||||
if n < 2 {
|
||||
println("revoke needs a device")
|
||||
return
|
||||
}
|
||||
organ_revoke(native_list_get(a, 1))
|
||||
println("{\"ok\": true, \"op\": \"revoke\", \"consent\": \"" + organ_consent_status() + "\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "status") {
|
||||
println("{\"ok\": true, \"op\": \"status\", \"consent\": \"" + organ_consent_status() + "\", \"speaker\": \"" + speaker_name() + "\", \"speaker_available\": " + int_to_str(speaker_available()) + ", \"mic_os_authorized\": " + int_to_str(mic_available()) + ", \"camera_os_authorized\": " + int_to_str(camera_available()) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- efferent ----------------------------------------------------------
|
||||
if str_eq(cmd, "speak") {
|
||||
if n < 2 {
|
||||
println("speak needs a wav")
|
||||
return
|
||||
}
|
||||
let ok: Bool = organ_speak_wav(native_list_get(a, 1))
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"speak\", \"played_aloud\": " + bool_to_str(ok) + "}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "tone") {
|
||||
let hz: Int = 220
|
||||
let ms: Int = 1000
|
||||
if n >= 2 {
|
||||
hz = str_to_int(native_list_get(a, 1))
|
||||
}
|
||||
if n >= 3 {
|
||||
ms = str_to_int(native_list_get(a, 2))
|
||||
}
|
||||
let s: [Int] = organ_tone(hz, ms, 16000)
|
||||
let ok: Bool = organ_speak_samples(s, 16000)
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"tone\", \"hz\": " + int_to_str(hz) + ", \"ms\": " + int_to_str(ms) + ", \"samples\": " + int_to_str(native_list_len(s)) + ", \"file\": null}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- the voice, from the engram ----------------------------------------
|
||||
if str_eq(cmd, "say") {
|
||||
if n < 3 {
|
||||
println("say needs <voice> <CODE> [CODE...]")
|
||||
return
|
||||
}
|
||||
cli_open_engram()
|
||||
let vname: String = native_list_get(a, 1)
|
||||
let g: [Int] = organ_voice_fetch(vname)
|
||||
if native_list_len(g) < 6 {
|
||||
println("{\"ok\": false, \"op\": \"say\", \"error\": \"no voice region '" + vname + "' in the engram\"}")
|
||||
return
|
||||
}
|
||||
// Codes and the phoneme map come from the LANGUAGE side. The organ does
|
||||
// not know what a word is and never looks one up.
|
||||
let pmap: [String] = ingest_phonetics("elp/data/phonetics.psv")
|
||||
let codes: [String] = native_list_empty()
|
||||
let i: Int = 2
|
||||
while i < n {
|
||||
codes = native_list_append(codes, native_list_get(a, i))
|
||||
i = i + 1
|
||||
}
|
||||
let voice: [String] = organ_voice_profile(vname, g)
|
||||
let s: [Int] = synth_codes(codes, voice, pmap)
|
||||
let ok: Bool = organ_speak_samples(s, 16000)
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"say\", \"voice\": \"" + vname + "\", \"f0\": " + int_to_str(native_list_get(g, 0)) + ", \"kf\": " + int_to_str(native_list_get(g, 2)) + ", \"codes\": " + int_to_str(native_list_len(codes)) + ", \"samples\": " + int_to_str(native_list_len(s)) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- afferent ----------------------------------------------------------
|
||||
if str_eq(cmd, "listen") {
|
||||
if n < 3 {
|
||||
println("listen needs <sec> <out.wav>")
|
||||
return
|
||||
}
|
||||
let secs: Int = str_to_int(native_list_get(a, 1))
|
||||
let out: String = native_list_get(a, 2)
|
||||
if organ_may_listen() == false {
|
||||
println("{\"ok\": false, \"op\": \"listen\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
organ_disclose("MIC: capturing " + int_to_str(secs) + "s (16 kHz mono, LOCAL, never egresses).")
|
||||
let s: [Int] = mic_capture_pcm16(secs, 16000)
|
||||
let got: Int = native_list_len(s)
|
||||
if got <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"listen\", \"error\": \"capture returned nothing\"}")
|
||||
return
|
||||
}
|
||||
let ok: Bool = write_wav(s, 16000, out)
|
||||
organ_disclose("MIC: captured " + int_to_str(got) + " frames — ready to hand to the ingest organ.")
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"listen\", \"file\": \"" + out + "\", \"frames\": " + int_to_str(got) + ", \"sample_rate\": 16000}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "see") {
|
||||
if n < 2 {
|
||||
println("see needs an out path")
|
||||
return
|
||||
}
|
||||
if organ_may_see() == false {
|
||||
println("{\"ok\": false, \"op\": \"see\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
organ_disclose("CAMERA: capturing one frame (LOCAL, never egresses).")
|
||||
let ok: Int = camera_capture_jpeg(native_list_get(a, 1))
|
||||
println("{\"ok\": " + int_to_str(ok) + ", \"op\": \"see\", \"file\": \"" + native_list_get(a, 1) + "\"}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- descriptors -------------------------------------------------------
|
||||
if str_eq(cmd, "wav-info") {
|
||||
if n < 2 {
|
||||
println("wav-info needs a wav")
|
||||
return
|
||||
}
|
||||
let p: String = native_list_get(a, 1)
|
||||
let w: [Float] = dsp_read_wav(p)
|
||||
if dsp_wav_n(w) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"wav-info\"}")
|
||||
return
|
||||
}
|
||||
println("{\"ok\": true, \"op\": \"wav-info\", \"sample_rate\": " + int_to_str(dsp_wav_sr(w)) + ", \"channels\": " + int_to_str(dsp_wav_ch(w)) + ", \"frames\": " + int_to_str(dsp_wav_n(w)) + "}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "feat-audio") {
|
||||
if n < 2 {
|
||||
println("feat-audio needs a wav")
|
||||
return
|
||||
}
|
||||
cli_feat_audio(native_list_get(a, 1))
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "feat-image") {
|
||||
if organ_may_see() == false {
|
||||
println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
let f: [Int] = organ_image_descriptor()
|
||||
if native_list_len(f) < 15 {
|
||||
println("{\"ok\": false, \"op\": \"feat-image\", \"error\": \"no frame\"}")
|
||||
return
|
||||
}
|
||||
let grid: String = ""
|
||||
let i: Int = 6
|
||||
while i < 15 {
|
||||
if i > 6 {
|
||||
grid = grid + ", "
|
||||
}
|
||||
grid = grid + int_to_str(native_list_get(f, i))
|
||||
i = i + 1
|
||||
}
|
||||
println("{\"ok\": true, \"op\": \"feat-image\", \"width\": " + int_to_str(native_list_get(f, 0)) + ", \"height\": " + int_to_str(native_list_get(f, 1)) + ", \"mean_rgb\": [" + int_to_str(native_list_get(f, 2)) + ", " + int_to_str(native_list_get(f, 3)) + ", " + int_to_str(native_list_get(f, 4)) + "], \"brightness_pm\": " + int_to_str(native_list_get(f, 5)) + ", \"luma_grid\": [" + grid + "]}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "voiceprint") {
|
||||
if n < 2 {
|
||||
println("voiceprint needs a wav")
|
||||
return
|
||||
}
|
||||
cli_voiceprint(native_list_get(a, 1))
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "imitate") {
|
||||
if n < 3 {
|
||||
println("imitate needs <in.wav> <out.wav>")
|
||||
return
|
||||
}
|
||||
let s: [Int] = dsp_imitate(native_list_get(a, 1))
|
||||
if native_list_len(s) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"imitate\"}")
|
||||
return
|
||||
}
|
||||
let ok: Bool = write_wav(s, 16000, native_list_get(a, 2))
|
||||
organ_disclose("IMITATE: rebuilt the voice from its own LPC signature (own-core, no training, no stolen voice).")
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"imitate\", \"out\": \"" + native_list_get(a, 2) + "\", \"samples\": " + int_to_str(native_list_len(s)) + ", \"method\": \"LPC analysis-resynthesis\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "hear-imitate") {
|
||||
if n < 3 {
|
||||
println("hear-imitate needs <sec> <out.wav>")
|
||||
return
|
||||
}
|
||||
let secs: Int = str_to_int(native_list_get(a, 1))
|
||||
let out: String = native_list_get(a, 2)
|
||||
if organ_may_listen() == false {
|
||||
println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"consent denied (fails closed)\"}")
|
||||
return
|
||||
}
|
||||
let heard: String = out + ".heard.wav"
|
||||
organ_disclose("HEAR-IMITATE: open the ear, listen " + int_to_str(secs) + "s, take the voice, speak it back.")
|
||||
let s: [Int] = mic_capture_pcm16(secs, 16000)
|
||||
if native_list_len(s) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"capture returned nothing\"}")
|
||||
return
|
||||
}
|
||||
write_wav(s, 16000, heard)
|
||||
let re: [Int] = dsp_imitate(heard)
|
||||
if native_list_len(re) <= 0 {
|
||||
println("{\"ok\": false, \"op\": \"hear-imitate\", \"error\": \"could not model the voice\"}")
|
||||
return
|
||||
}
|
||||
write_wav(re, 16000, out)
|
||||
let ok: Bool = organ_speak_samples(re, 16000)
|
||||
println("{\"ok\": " + bool_to_str(ok) + ", \"op\": \"hear-imitate\", \"heard\": \"" + heard + "\", \"out\": \"" + out + "\", \"spoke_aloud\": " + bool_to_str(ok) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- the afferent wire: descriptor -> geometry --------------------------
|
||||
if str_eq(cmd, "ingest-audio") {
|
||||
if n < 2 {
|
||||
println("ingest-audio needs a wav")
|
||||
return
|
||||
}
|
||||
let p: String = native_list_get(a, 1)
|
||||
let v: [Float] = dsp_compute_audio(p)
|
||||
if native_list_len(v) < 8 {
|
||||
println("{\"ok\": false, \"op\": \"ingest-audio\"}")
|
||||
return
|
||||
}
|
||||
cli_open_engram()
|
||||
let content: String = cli_audio_descriptor_text(v, p)
|
||||
let id: String = engram_node(content, "Observation", 70)
|
||||
engram_store_checkpoint()
|
||||
organ_disclose("INGEST: the capture is now GEOMETRY in the engram (node " + id + ") — the descriptor travelled, the stream did not.")
|
||||
println("{\"ok\": true, \"op\": \"ingest-audio\", \"node_id\": \"" + id + "\", \"content\": \"" + content + "\"}")
|
||||
return
|
||||
}
|
||||
if str_eq(cmd, "ingest-voice") {
|
||||
if n < 3 {
|
||||
println("ingest-voice needs <voice.wav> <name>")
|
||||
return
|
||||
}
|
||||
let p: String = native_list_get(a, 1)
|
||||
let name: String = native_list_get(a, 2)
|
||||
let v: [Float] = dsp_voiceprint(p)
|
||||
if native_list_len(v) < 10 {
|
||||
println("{\"ok\": false, \"op\": \"ingest-voice\", \"error\": \"no voiced frames\"}")
|
||||
return
|
||||
}
|
||||
cli_open_engram()
|
||||
let f0: Int = float_to_int(native_list_get(v, 0))
|
||||
let f1: Int = float_to_int(native_list_get(v, 4))
|
||||
let f2: Int = float_to_int(native_list_get(v, 6))
|
||||
let f3: Int = float_to_int(native_list_get(v, 8))
|
||||
// kf is the vocal-tract scale: this speaker's F1 against the nominal
|
||||
// /AA/ F1 of 730 Hz. One number standing for a tract length.
|
||||
let kf: Int = 1000 * f1 / 730
|
||||
let f0e: Int = f0 * 85 / 100
|
||||
let id: String = organ_voice_ingest(name, f0, f0e, kf, f1, f2, f3, "el-organ-lpc-voiceprint", "COARSE")
|
||||
engram_store_checkpoint()
|
||||
println("{\"ok\": true, \"op\": \"ingest-voice\", \"node_id\": \"" + id + "\", \"name\": \"" + name + "\", \"f0\": " + int_to_str(f0) + ", \"kf\": " + int_to_str(kf) + ", \"f1\": " + int_to_str(f1) + ", \"f2\": " + int_to_str(f2) + ", \"f3\": " + int_to_str(f3) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
// ---- converse ----------------------------------------------------------
|
||||
if str_eq(cmd, "converse") {
|
||||
if n < 2 {
|
||||
println("converse needs a manifest")
|
||||
return
|
||||
}
|
||||
let mf: String = native_list_get(a, 1)
|
||||
let authority: Int = 500
|
||||
let barge: Int = 0 - 1
|
||||
let kind: String = "bargein"
|
||||
let live: Bool = false
|
||||
let resume: Bool = false
|
||||
let i: Int = 2
|
||||
while i < n {
|
||||
let f: String = native_list_get(a, i)
|
||||
if str_eq(f, "--authority") {
|
||||
if i + 1 < n {
|
||||
authority = str_to_int(native_list_get(a, i + 1))
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
if str_eq(f, "--barge-at") {
|
||||
if i + 1 < n {
|
||||
let spec: String = native_list_get(a, i + 1)
|
||||
let c: Int = str_index_of(spec, ":")
|
||||
if c < 0 {
|
||||
barge = str_to_int(spec)
|
||||
} else {
|
||||
barge = str_to_int(str_slice(spec, 0, c))
|
||||
kind = str_slice(spec, c + 1, str_len(spec))
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
if str_eq(f, "--live-mic") {
|
||||
live = true
|
||||
}
|
||||
if str_eq(f, "--resume") {
|
||||
resume = true
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
let plan: [String] = conv_load_manifest(mf)
|
||||
if resume {
|
||||
plan = conv_load_resume()
|
||||
organ_disclose("CONVERSE: resuming — \"as I was saying...\" (" + int_to_str(plan_count(plan)) + " segments left).")
|
||||
} else {
|
||||
organ_disclose("CONVERSE: utterance = \"" + conv_utterance(mf) + "\" (" + int_to_str(plan_count(plan)) + " segments).")
|
||||
}
|
||||
let stopped: Int = conv_run(plan, authority, barge, kind, live)
|
||||
println("{\"ok\": true, \"op\": \"converse\", \"stopped_at\": " + int_to_str(stopped) + ", \"complete\": " + bool_to_str(stopped < 0) + "}")
|
||||
return
|
||||
}
|
||||
|
||||
cli_usage()
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
// organ_converse.el — full-duplex, interruptible speech. The turn-taking organ.
|
||||
//
|
||||
// WHAT THIS IS FOR. A system that plays an utterance to completion and only
|
||||
// then listens is not conversational, it is a loudspeaker with a queue. Being
|
||||
// interruptible is not a feature bolted onto speech; it is most of what makes
|
||||
// speech social. So the utterance is not a blob of audio — it is an ordered,
|
||||
// SALIENCE-TAGGED MEANING-PLAN, and the organ speaks it while listening, decides
|
||||
// what to do when interrupted, and can pick the thread back up afterwards.
|
||||
//
|
||||
// THREE THINGS HAVE TO BE TRUE, and each one is a place naive implementations
|
||||
// go wrong:
|
||||
//
|
||||
// Barge-in is AT THE SAMPLE. When the mic hears speech, output stops on the
|
||||
// spot — not at the end of the current buffer, not at the end of the segment.
|
||||
// A listener experiences even a fifth of a second of continued talking as
|
||||
// being talked over. This is why the speaker realizer has pause/resume and
|
||||
// reports played_frames: "finish the buffer" is not barge-in.
|
||||
//
|
||||
// Yield-or-hold is a DECISION, not a rule. Stopping every time anyone makes a
|
||||
// noise is its own failure — it means Neuron can never finish a sentence that
|
||||
// matters. So the choice is grounded: how salient is what I am mid-saying,
|
||||
// how close am I to done, and how much authority does the interrupter have.
|
||||
// Holding the floor is justified when what I am saying matters AND finishing
|
||||
// is cheap AND the interrupter is not high-priority. Otherwise yield, because
|
||||
// the polite default is the right default.
|
||||
//
|
||||
// A backchannel is NOT an interruption. "mm-hm" means keep going. Treating it
|
||||
// as a barge-in makes the system stop every three seconds during ordinary
|
||||
// listening behaviour, which is worse than not listening at all. It is
|
||||
// distinguished by being brief and low-energy: sample again shortly after
|
||||
// onset, and if the speech already died away it was a backchannel.
|
||||
//
|
||||
// AND THE UTTERANCE SURVIVES. On yield, the remaining plan is persisted, so
|
||||
// Neuron can resume — "as I was saying" — instead of losing the thought. An
|
||||
// interruption should cost a turn, not the content.
|
||||
//
|
||||
// The AEC rail: the microphone runs with the OS voice-processing unit enabled
|
||||
// so it does not hear our own speaker. Without it Neuron barges in on its own
|
||||
// voice on the first syllable and the whole loop is unusable in a real room.
|
||||
//
|
||||
// Note what is NOT here: nothing about words. A segment carries a `text` field
|
||||
// purely as a label for disclosure. The organ speaks pre-rendered audio and
|
||||
// never inspects language — that is the language faculty's, and the seam holds.
|
||||
|
||||
// ── The meaning-plan ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Stored as a flat [String] with stride 3 — file, salience-per-mille, text —
|
||||
// because El has no record type and parallel lists drift out of step under
|
||||
// editing. Salience is an integer per-mille rather than a Float so the decision
|
||||
// arithmetic stays exact and reproducible; a turn-taking decision that varies
|
||||
// with floating-point rounding is not one you can debug.
|
||||
|
||||
fn plan_new() -> [String] {
|
||||
return native_list_empty()
|
||||
}
|
||||
|
||||
fn plan_add(plan: [String], file: String, salience_pm: Int, text: String) -> [String] {
|
||||
let p: [String] = plan
|
||||
p = native_list_append(p, file)
|
||||
p = native_list_append(p, int_to_str(salience_pm))
|
||||
p = native_list_append(p, text)
|
||||
return p
|
||||
}
|
||||
|
||||
fn plan_count(plan: [String]) -> Int {
|
||||
return native_list_len(plan) / 3
|
||||
}
|
||||
|
||||
fn plan_file(plan: [String], i: Int) -> String {
|
||||
return native_list_get(plan, i * 3)
|
||||
}
|
||||
|
||||
fn plan_salience(plan: [String], i: Int) -> Int {
|
||||
return str_to_int(native_list_get(plan, i * 3 + 1))
|
||||
}
|
||||
|
||||
fn plan_text(plan: [String], i: Int) -> String {
|
||||
return native_list_get(plan, i * 3 + 2)
|
||||
}
|
||||
|
||||
// ── Manifest ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// {"utterance": "...", "segments": [{"file":..., "salience":0.9, "text":"..."}]}
|
||||
// Salience arrives as a 0..1 float in the manifest and is converted once, here,
|
||||
// at the edge — the same discipline the runtime uses for wire encodings.
|
||||
|
||||
fn conv_salience_pm(raw: String) -> Int {
|
||||
// "0.85" -> 850. Parsed by hand rather than through a float so a manifest
|
||||
// typo degrades to a visible number instead of a silent 0.0.
|
||||
let dot: Int = str_index_of(raw, ".")
|
||||
if dot < 0 {
|
||||
let whole: Int = str_to_int(raw)
|
||||
return whole * 1000
|
||||
}
|
||||
let ip: Int = str_to_int(str_slice(raw, 0, dot))
|
||||
let frac: String = str_slice(raw, dot + 1, str_len(raw))
|
||||
let pm: Int = 0
|
||||
let scale: Int = 100
|
||||
let i: Int = 0
|
||||
while i < 3 {
|
||||
let d: Int = 0
|
||||
if i < str_len(frac) {
|
||||
let c: Int = str_char_code(frac, i)
|
||||
if c >= 48 {
|
||||
if c <= 57 {
|
||||
d = c - 48
|
||||
}
|
||||
}
|
||||
}
|
||||
pm = pm + d * scale
|
||||
scale = scale / 10
|
||||
i = i + 1
|
||||
}
|
||||
return ip * 1000 + pm
|
||||
}
|
||||
|
||||
fn conv_load_manifest(path: String) -> [String] {
|
||||
let plan: [String] = plan_new()
|
||||
let raw: String = fs_read(path)
|
||||
if str_eq(raw, "") {
|
||||
organ_disclose("CONVERSE: cannot read manifest " + path)
|
||||
return plan
|
||||
}
|
||||
let segs: String = json_get_raw(raw, "segments")
|
||||
let n: Int = json_array_len(segs)
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let seg: String = json_array_get(segs, i)
|
||||
let file: String = json_get_string(seg, "file")
|
||||
let text: String = json_get_string(seg, "text")
|
||||
let sal: String = json_get_raw(seg, "salience")
|
||||
let pm: Int = conv_salience_pm(sal)
|
||||
if pm <= 0 {
|
||||
pm = 500
|
||||
}
|
||||
plan = plan_add(plan, file, pm, text)
|
||||
i = i + 1
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
fn conv_utterance(path: String) -> String {
|
||||
let raw: String = fs_read(path)
|
||||
return json_get_string(raw, "utterance")
|
||||
}
|
||||
|
||||
// ── The decision ─────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Returns: 0 = backchannel, carry on seamlessly
|
||||
// 1 = hold the floor ("hang on, let me finish this thought")
|
||||
// 2 = yield (stop, let them in)
|
||||
//
|
||||
// All arguments are per-mille integers. Holding requires BOTH that the material
|
||||
// is worth finishing AND that the interrupter is not high-authority — either
|
||||
// condition alone is not enough, because "what I'm saying is important" is
|
||||
// exactly the reasoning that produces a system nobody can get a word in against.
|
||||
fn conv_decide(salience_pm: Int, progress_pm: Int, authority_pm: Int, is_backchannel: Bool) -> Int {
|
||||
if is_backchannel {
|
||||
return 0
|
||||
}
|
||||
let hold_score: Int = (salience_pm * 6 + progress_pm * 4) / 10
|
||||
if hold_score >= 600 {
|
||||
if authority_pm < 800 {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
// ── Resume ───────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The remaining plan, written where a later run can find it. This is what turns
|
||||
// an interruption into a pause rather than a loss.
|
||||
|
||||
fn conv_resume_path() -> String {
|
||||
let home: String = env("PERIPH_HOME")
|
||||
if str_eq(home, "") {
|
||||
return "peripheral/.resume.json"
|
||||
}
|
||||
return home + "/.resume.json"
|
||||
}
|
||||
|
||||
// Minimal JSON string escaping. Written here rather than reached for from the
|
||||
// runtime because the organ needs exactly two escapes and no dependency: a
|
||||
// segment label containing a quote or a backslash must not be able to produce a
|
||||
// resume file that fails to parse and silently loses the thread.
|
||||
fn conv_escape(s: String) -> String {
|
||||
let n: Int = str_len(s)
|
||||
let out: String = ""
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let c: Int = str_char_code(s, i)
|
||||
if c == 34 {
|
||||
out = out + "\\\""
|
||||
} else {
|
||||
if c == 92 {
|
||||
out = out + "\\\\"
|
||||
} else {
|
||||
if c >= 32 {
|
||||
out = out + str_slice(s, i, i + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
fn conv_persist_resume(plan: [String], start_at: Int, reason: String) -> Bool {
|
||||
let n: Int = plan_count(plan)
|
||||
let body: String = "{\"resume_from\": " + int_to_str(start_at) + ", \"reason\": \"" + reason + "\", \"segments\": ["
|
||||
let i: Int = start_at
|
||||
let first: Bool = true
|
||||
while i < n {
|
||||
if first == false {
|
||||
body = body + ", "
|
||||
}
|
||||
body = body + "{\"file\": \"" + plan_file(plan, i) + "\", \"salience\": " + int_to_str(plan_salience(plan, i)) + ", \"text\": \"" + conv_escape(plan_text(plan, i)) + "\"}"
|
||||
first = false
|
||||
i = i + 1
|
||||
}
|
||||
body = body + "]}\n"
|
||||
let ok: Bool = fs_write(conv_resume_path(), body)
|
||||
organ_disclose("CONVERSE: meaning-plan persisted (" + int_to_str(n - start_at) + " segments remain) — Neuron can resume the thread.")
|
||||
return ok
|
||||
}
|
||||
|
||||
fn conv_clear_resume() -> Bool {
|
||||
return fs_write(conv_resume_path(), "")
|
||||
}
|
||||
|
||||
// Read a persisted plan back. Salience is already per-mille here (we wrote it),
|
||||
// so it is NOT re-scaled — the manifest and the resume file are different
|
||||
// formats on purpose, and conflating them silently divides every salience by a
|
||||
// thousand.
|
||||
fn conv_load_resume() -> [String] {
|
||||
let plan: [String] = plan_new()
|
||||
let raw: String = fs_read(conv_resume_path())
|
||||
if str_eq(raw, "") {
|
||||
return plan
|
||||
}
|
||||
let segs: String = json_get_raw(raw, "segments")
|
||||
let n: Int = json_array_len(segs)
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let seg: String = json_array_get(segs, i)
|
||||
plan = plan_add(plan, json_get_string(seg, "file"), json_get_int(seg, "salience"), json_get_string(seg, "text"))
|
||||
i = i + 1
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
// ── The loop ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// live_mic : open the microphone with AEC and let real speech drive barge-in.
|
||||
// barge_ms : if >= 0, inject a barge event at that offset into the utterance
|
||||
// instead. Deterministic, so the decision paths can be exercised
|
||||
// without a room and a person — the same reason periph.swift has it.
|
||||
// kind : "backchannel" or "bargein", for the injected case.
|
||||
// authority : interrupter authority, per-mille.
|
||||
//
|
||||
// Returns the index the utterance stopped at, or -1 if it completed.
|
||||
|
||||
fn conv_run(plan: [String], authority_pm: Int, barge_ms: Int, kind: String, live_mic: Bool) -> Int {
|
||||
let n: Int = plan_count(plan)
|
||||
if n <= 0 {
|
||||
organ_disclose("CONVERSE: nothing to say.")
|
||||
return 0 - 1
|
||||
}
|
||||
if speaker_available() == 0 {
|
||||
organ_disclose("CONVERSE: no speaker on this build — cannot hold a conversation.")
|
||||
return 0 - 1
|
||||
}
|
||||
|
||||
let mic_live: Bool = false
|
||||
if live_mic {
|
||||
if organ_may_listen() {
|
||||
let m: Int = mic_monitor_start()
|
||||
if m == 1 {
|
||||
organ_disclose("CONVERSE: full-duplex — mic listening WHILE speaking, AEC on (won't self-interrupt).")
|
||||
mic_live = true
|
||||
}
|
||||
if m == 2 {
|
||||
organ_disclose("CONVERSE: full-duplex — mic listening, but AEC UNAVAILABLE; raising the VAD floor so we do not barge in on ourselves.")
|
||||
mic_live = true
|
||||
}
|
||||
if m == 0 {
|
||||
organ_disclose("CONVERSE: could not open the mic monitor — falling back to injected events.")
|
||||
}
|
||||
}
|
||||
}
|
||||
if mic_live == false {
|
||||
organ_disclose("CONVERSE: deterministic mode (live mic off).")
|
||||
}
|
||||
|
||||
// Without AEC the mic hears the speaker, so the threshold has to sit above
|
||||
// our own output. This is a mitigation and not a fix: the honest note is
|
||||
// that barge-in is markedly less sensitive in this mode.
|
||||
let vad_pm: Int = 20
|
||||
if mic_live {
|
||||
if mic_monitor_start() == 2 {
|
||||
vad_pm = 60
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed_ms: Int = 0
|
||||
let prior_ms: Int = 0
|
||||
let handled: Bool = false
|
||||
// An injected barge is ONE event, not a condition that stays true. Without
|
||||
// this the deadline re-fires on every poll after a backchannel resume, and
|
||||
// the utterance live-locks: paused, resumed, paused again, forever.
|
||||
let injected_fired: Bool = false
|
||||
let i: Int = 0
|
||||
|
||||
while i < n {
|
||||
let file: String = plan_file(plan, i)
|
||||
let sal: Int = plan_salience(plan, i)
|
||||
let frames: Int = wav_frames(file)
|
||||
let rate: Int = wav_rate(file)
|
||||
if frames <= 0 {
|
||||
organ_disclose("CONVERSE: missing or unreadable segment '" + file + "', skipping.")
|
||||
i = i + 1
|
||||
} else {
|
||||
let dur_ms: Int = frames * 1000 / rate
|
||||
organ_disclose("CONVERSE: speaking segment " + int_to_str(i + 1) + "/" + int_to_str(n) + " (salience " + int_to_str(sal) + "/1000) — \"" + plan_text(plan, i) + "\"")
|
||||
let started: Int = speaker_play_wav_async(file)
|
||||
if started == 0 {
|
||||
organ_disclose("CONVERSE: could not start playback for '" + file + "'.")
|
||||
i = i + 1
|
||||
} else {
|
||||
let seg_ms: Int = 0
|
||||
let done: Bool = false
|
||||
let interrupted: Bool = false
|
||||
let speech_ticks: Int = 0
|
||||
|
||||
while done == false {
|
||||
sleep_ms(10)
|
||||
seg_ms = seg_ms + 10
|
||||
|
||||
if speaker_playing() == 0 {
|
||||
done = true
|
||||
} else {
|
||||
// The tick counter is an approximation — each pass costs
|
||||
// more than the sleep it asked for. The DAC position is
|
||||
// the truth, so drive the injected deadline off THAT and
|
||||
// an injected barge lands where it was asked to land.
|
||||
let pos_ms: Int = speaker_played_frames() * 1000 / rate
|
||||
elapsed_ms = prior_ms + pos_ms
|
||||
// --- onset detection: real speech, or an injected event ---
|
||||
let onset: Bool = false
|
||||
if mic_live {
|
||||
let rms: Float = mic_monitor_rms()
|
||||
let rms_pm: Int = float_to_int(rms * 1000.0)
|
||||
if rms_pm > vad_pm {
|
||||
speech_ticks = speech_ticks + 1
|
||||
} else {
|
||||
speech_ticks = 0
|
||||
}
|
||||
// ~60ms of continuous voice: short enough to feel
|
||||
// instant, long enough that a door closing is not a turn.
|
||||
if speech_ticks >= 3 {
|
||||
if handled == false {
|
||||
onset = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if barge_ms >= 0 {
|
||||
if injected_fired == false {
|
||||
if elapsed_ms >= barge_ms {
|
||||
onset = true
|
||||
injected_fired = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if onset {
|
||||
handled = true
|
||||
// (1) BARGE-IN — pause on the spot.
|
||||
speaker_pause()
|
||||
let played: Int = speaker_played_frames()
|
||||
let at_ms: Int = played * 1000 / rate
|
||||
let progress_pm: Int = at_ms * 1000 / dur_ms
|
||||
if progress_pm > 1000 {
|
||||
progress_pm = 1000
|
||||
}
|
||||
organ_disclose("CONVERSE: << user speech at " + int_to_str(at_ms) + "ms into segment " + int_to_str(i + 1) + " — PAUSED instantly >>")
|
||||
|
||||
// (2) backchannel or real barge-in?
|
||||
let is_bc: Bool = false
|
||||
if barge_ms >= 0 {
|
||||
if str_eq(kind, "backchannel") {
|
||||
is_bc = true
|
||||
}
|
||||
} else {
|
||||
// Live: look again ~250ms after onset. If the
|
||||
// energy has already collapsed it was "mm-hm".
|
||||
sleep_ms(250)
|
||||
let r2: Float = mic_monitor_rms()
|
||||
if float_to_int(r2 * 1000.0) < 15 {
|
||||
is_bc = true
|
||||
}
|
||||
}
|
||||
|
||||
// (3) yield, hold, or carry on
|
||||
let d: Int = conv_decide(sal, progress_pm, authority_pm, is_bc)
|
||||
if d == 0 {
|
||||
organ_disclose("CONVERSE: read as BACKCHANNEL (\"mm-hm\") — keep going, resume seamlessly.")
|
||||
handled = false
|
||||
speech_ticks = 0
|
||||
speaker_resume()
|
||||
}
|
||||
if d == 1 {
|
||||
organ_disclose("CONVERSE: HOLD the floor — \"hang on, let me finish this thought.\" (salience " + int_to_str(sal) + ", progress " + int_to_str(progress_pm) + ")")
|
||||
speaker_resume()
|
||||
// Finish THIS segment, then yield the remainder:
|
||||
// holding is a request for a moment, not a claim
|
||||
// on the rest of the conversation.
|
||||
while speaker_playing() == 1 {
|
||||
sleep_ms(20)
|
||||
}
|
||||
speaker_stop()
|
||||
conv_persist_resume(plan, i + 1, "held-then-yield")
|
||||
if mic_live {
|
||||
mic_monitor_stop()
|
||||
}
|
||||
return i + 1
|
||||
}
|
||||
if d == 2 {
|
||||
organ_disclose("CONVERSE: YIELD — stop, let them in. Remembering where I was (resumable).")
|
||||
speaker_stop()
|
||||
conv_persist_resume(plan, i, "yield")
|
||||
if mic_live {
|
||||
mic_monitor_stop()
|
||||
}
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if interrupted == false {
|
||||
prior_ms = prior_ms + dur_ms
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
conv_clear_resume()
|
||||
organ_disclose("CONVERSE: utterance complete (uninterrupted).")
|
||||
if mic_live {
|
||||
mic_monitor_stop()
|
||||
}
|
||||
return 0 - 1
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user