/* el_capture_darwin.m — the MICROPHONE and CAMERA realizers. El's afferent * organ on Darwin: the two entry points through which the world gets in. * * WHY THIS FILE EXISTS, AND WHY IT IS A REALIZER RATHER THAN PURE EL. * * el_audio_darwin.m argued the efferent half of this: El can compute a sound * but it cannot make one, because "the DAC has now played these samples" is not * a fact any amount of arithmetic can produce. This file is the same argument * run backwards. El can compute *about* a sound — it can window it, take its * autocorrelation, run Levinson-Durbin over that, find the formant peaks in the * resulting all-pole envelope, and hand back a voiceprint — but it cannot ASK. * There is no expression in El, and there must not be, whose value is "the next * 1024 frames the microphone hears" or "what the camera is pointed at right * now." Those are not computed; they are *requested*, from an operating system * that owns the device, mediates consent for it, and delivers the answer on a * thread of its choosing whenever it feels like it. Asking is the one primitive * operation here. Everything else in this file is bookkeeping around the ask. * * So the line is drawn exactly where el_audio_darwin.m drew it, at the sample * buffer, and it is drawn on purpose: * * BELOW the line (here): open the device, honour the OS permission gate, * install a tap or a frame delegate, convert whatever the hardware happens to * emit into the one shape El asked for, and hand it up. No opinions about * content. No analysis. No decisions. * * ABOVE the line (El): energy, zero-crossing rate, spectral centroid, F0 by * autocorrelation, LPC, formants F1-F5, the compact descriptors, the * scene-geometry grid, the yield-or-hold turn-taking decision. All of it is * arithmetic over a buffer, all of it belongs in El, and none of it appears * below. The reference this file ports — peripheral/src/periph.swift — held * both halves, and that was the problem worth fixing: the descriptors were * trapped in a 939-line binary standing next to the language instead of being * written in it. Porting the *whole* of periph.swift down here would have * reproduced that mistake in C. Only the ask came down. * * The precedent for the file's SHAPE is eg_cosine_batch_strategy_metal_hand.m: * a platform-bound capability compiled as its own translation unit, declared in * el_runtime.h, linked in where the platform supports it, and deliberately NOT * a patch to the middle of el_runtime.c. Acquiring a device must not mean * editing the language, for the same reason acquiring a modality must not (see * the realizer registry: organs are resolved by name). el_peripheral_null.c is * the same entry points everywhere else, so El code that listens still links on * every platform and merely reports having no ear. * * FAIL CLOSED, ALWAYS. * * A capture path that returns plausible-looking zeros when it was denied is * worse than one that returns nothing, because the caller cannot tell the * difference between a silent room and a refused microphone. Every entry point * here checks AVCaptureDevice's authorization status BEFORE touching hardware * and returns the empty value — an empty list, a 0 map — on anything short of * .authorized. mic_available() and camera_available() report that state WITHOUT * prompting, so El can ask "may I?" without the act of asking being a prompt. * * NEVER HANG. * * Every wait in this file is bounded: 30s on a permission prompt (the user has * to walk to the dialog), seconds+5 on a capture of `seconds`, 10s on a camera * frame. An organ that wedges the program holding it is not an organ, it is a * fault. Every path also tears the device down on the way out, including the * failure paths, so a timed-out capture does not leave the mic light on. * * OWN-CORE AND LOCAL BY CONSTRUCTION. * * AVFoundation, CoreVideo, CoreGraphics and ImageIO ship with macOS. There is * no third-party library here, no model, and — the part that matters — no * network path of any kind. Samples and pixels move from local hardware into an * El value and stop. periph.swift had a URLSession in it; this file has no * socket, no URL, and nothing that could grow one without being obvious in * review. Consent is enforced above this layer in El and below it by the OS; * this layer's whole contribution to that is refusing to proceed. * * DISCLOSURE. * * Every actual device touch writes one line to stderr and flushes it, before * the device opens. stderr and not stdout: a program that announces "I am about * to open the microphone" on stdout has corrupted its own output, and the * caller must be able to separate the answer from how it was obtained. One line * per touch, no more — a disclosure rail that spams is a rail people learn to * ignore. */ #if defined(__APPLE__) #import #import #import #import #import #import #include #include #include #include #include #include #include "el_runtime.h" /* ── Disclosure ────────────────────────────────────────────────────────────── * One flushed line per real device touch, on stderr. Flushed rather than * buffered so the line reaches the terminal BEFORE the mic light comes on * rather than whenever the buffer happens to drain. */ static void el_cap_disclose(const char* what) { fprintf(stderr, " [peripheral] %s\n", what); fflush(stderr); } /* ── Permission ────────────────────────────────────────────────────────────── * authorizationStatus is a pure read of the TCC database: it never prompts and * never blocks, which is what lets mic_available()/camera_available() answer * honestly without the question itself becoming an event. requestAccess DOES * prompt, so it lives behind its own entry point and nothing calls it * implicitly. */ static int el_cap_authorized(AVMediaType media) { @try { return [AVCaptureDevice authorizationStatusForMediaType:media] == AVAuthorizationStatusAuthorized ? 1 : 0; } @catch (NSException* e) { (void)e; return 0; } } static int el_cap_device_present(AVMediaType media) { @try { return [AVCaptureDevice defaultDeviceWithMediaType:media] != nil ? 1 : 0; } @catch (NSException* e) { (void)e; return 0; } } /* Prompt once and wait, bounded. 30 seconds is the same budget periph.swift * used: long enough for a human to notice a dialog and decide, short enough * that an unattended run fails rather than parks forever. A timeout is reported * as "not granted", which is the safe reading — we genuinely do not know that * it was. */ static int el_cap_request(AVMediaType media) { __block int granted = 0; dispatch_semaphore_t sem = dispatch_semaphore_create(0); @try { [AVCaptureDevice requestAccessForMediaType:media completionHandler:^(BOOL ok) { granted = ok ? 1 : 0; dispatch_semaphore_signal(sem); }]; } @catch (NSException* e) { (void)e; return 0; } if (dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (int64_t)(30 * NSEC_PER_SEC))) != 0) { return 0; /* timed out — treat as refused */ } return granted; } /* ════════════════════════════════════════════════════════════════════════════ * MICROPHONE — one-shot capture * ══════════════════════════════════════════════════════════════════════════ */ /* The sink the tap block writes into. An object rather than a static so two * captures can never share state, and so ARC keeps it alive for exactly as long * as the block that captured it. The lock is real, not decorative: the tap runs * on an AVAudioEngine-internal thread and the waiter runs on the caller's. */ @interface ElCapMicSink : NSObject @property (nonatomic, strong) NSMutableData* pcm; @property (nonatomic, strong) NSLock* lock; @end @implementation ElCapMicSink - (instancetype)init { self = [super init]; if (self) { _pcm = [NSMutableData data]; _lock = [[NSLock alloc] init]; } return self; } @end /* Capture `seconds` of mono 16-bit PCM at `sample_rate`. * * The hardware format is NOT assumed. A built-in mic will typically hand back * float32 at 44.1 or 48 kHz, an aggregate device may be 8 channels at 96 kHz, * and a caller asking for 16 kHz mono (which is what the formant path wants) * gets 16 kHz mono either way. AVAudioConverter does the rate conversion and * the downmix; doing it by hand would mean writing a resampler in the one file * that is supposed to contain no arithmetic. * * The converter is built ONCE, outside the tap, because a sample-rate converter * carries filter state across buffers — rebuilding it per callback would put a * discontinuity at every buffer boundary, which is audible and which would then * show up in El's spectral descriptors as energy that was never in the room. */ el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { el_val_t empty = el_list_empty(); if (!el_cap_authorized(AVMediaTypeAudio)) return empty; int64_t secs = (int64_t)seconds; int64_t sr = (int64_t)sample_rate; if (secs <= 0 || sr <= 0) return empty; /* Bound the ask. A caller that asks for a year of audio has made a mistake, * and honouring it would mean an unkillable capture and an OOM. */ if (secs > 300) secs = 300; if (sr > 384000) sr = 384000; __block AVAudioEngine* engine = nil; AVAudioInputNode* input = nil; AVAudioFormat* hwFmt = nil; int tapped = 0; @try { engine = [[AVAudioEngine alloc] init]; input = [engine inputNode]; hwFmt = [input inputFormatForBus:0]; } @catch (NSException* e) { (void)e; return empty; } if (!input || !hwFmt || hwFmt.sampleRate <= 0 || hwFmt.channelCount == 0) { return empty; } /* Preferred target: mono int16 at the requested rate. If the converter * refuses that pairing (some exotic input layouts will not downmix), fall * back to keeping the hardware's channel count and taking channel 0 on the * way out — the organ is mono by design and inventing a downmix here would * be an opinion about content this layer is not entitled to have. */ AVAudioFormat* outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:(double)sr channels:1 interleaved:YES]; AVAudioConverter* conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil; AVAudioChannelCount outCh = 1; if (!conv) { outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:(double)sr channels:hwFmt.channelCount interleaved:YES]; conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil; outCh = hwFmt.channelCount; } if (!conv || !outFmt) return empty; ElCapMicSink* sink = [[ElCapMicSink alloc] init]; const int64_t want = secs * sr; /* frames we are waiting for */ { char msg[192]; snprintf(msg, sizeof(msg), "MIC: opening the microphone for %llds -> %lld Hz mono PCM " "(local, never egresses).", (long long)secs, (long long)sr); el_cap_disclose(msg); } const double ratio = (double)sr / hwFmt.sampleRate; @try { [input installTapOnBus:0 bufferSize:4096 format:hwFmt block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { (void)when; if (!buf || buf.frameLength == 0) return; AVAudioFrameCount cap = (AVAudioFrameCount)((double)buf.frameLength * ratio) + 1024; AVAudioPCMBuffer* out = [[AVAudioPCMBuffer alloc] initWithPCMFormat:outFmt frameCapacity:cap]; if (!out) return; __block BOOL fed = NO; AVAudioConverterInputBlock feed = ^AVAudioBuffer* _Nullable (AVAudioPacketCount need, AVAudioConverterInputStatus* _Nonnull status) { (void)need; if (fed) { *status = AVAudioConverterInputStatus_NoDataNow; return nil; } fed = YES; *status = AVAudioConverterInputStatus_HaveData; return buf; }; NSError* err = nil; AVAudioConverterOutputStatus st = [conv convertToBuffer:out error:&err withInputFromBlock:feed]; if (st == AVAudioConverterOutputStatus_Error || out.frameLength == 0) return; const int16_t* src = out.int16ChannelData ? out.int16ChannelData[0] : NULL; if (!src) return; NSUInteger n = (NSUInteger)out.frameLength; [sink.lock lock]; if (outCh == 1) { [sink.pcm appendBytes:src length:n * sizeof(int16_t)]; } else { /* Interleaved: stride to channel 0. */ for (NSUInteger i = 0; i < n; i++) { int16_t v = src[i * outCh]; [sink.pcm appendBytes:&v length:sizeof(int16_t)]; } } [sink.lock unlock]; }]; tapped = 1; [engine prepare]; NSError* startErr = nil; if (![engine startAndReturnError:&startErr]) { [input removeTapOnBus:0]; return empty; } } @catch (NSException* e) { (void)e; @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } return empty; } /* Wait for `want` frames, bounded by the material's own duration plus a * margin. A device that stops producing must not become a hang. */ const int64_t deadline_us = (secs + 5) * 1000000; int64_t waited_us = 0; const int64_t tick_us = 5000; for (;;) { [sink.lock lock]; int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); [sink.lock unlock]; if (have >= want || waited_us >= deadline_us) break; usleep((useconds_t)tick_us); waited_us += tick_us; } @try { [input removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } @try { [engine stop]; } @catch (NSException* e) { (void)e; } engine = nil; /* Hand up exactly what was asked for, or everything we got if the device * came up short. Never padded: silence we invented is indistinguishable * from silence we heard, and El has no way to tell them apart afterwards. */ [sink.lock lock]; int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); int64_t n = have < want ? have : want; const int16_t* pcm = (const int16_t*)[sink.pcm bytes]; el_val_t list = empty; for (int64_t i = 0; i < n; i++) { list = el_list_append(list, (el_val_t)(int64_t)pcm[i]); } [sink.lock unlock]; return list; } el_val_t mic_available(void) { if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; return (el_val_t)(el_cap_device_present(AVMediaTypeAudio) ? 1 : 0); } el_val_t mic_request_access(void) { return (el_val_t)(el_cap_request(AVMediaTypeAudio) ? 1 : 0); } /* ════════════════════════════════════════════════════════════════════════════ * MICROPHONE — live monitor (the full-duplex ear) * * converse needs to keep listening WHILE it speaks, which means the mic is open * at the same time as the speaker. In a real room that is a feedback path: * without cancellation Neuron hears its own voice, decides someone is talking, * and barges in on itself. setVoiceProcessingEnabled: hands the input node to * the OS voice-processing unit, which subtracts the known output signal from * the input — the single thing that makes barge-in work outside a headset. * * It is not always available (some aggregate and virtual devices refuse it), so * failure to enable it is reported as a DISTINCT return value (2) rather than * folded into success. The caller needs to know, because the correct response * is to raise the VAD floor, and a caller that thinks AEC is on will set that * floor far too low. * * The tap keeps only a running short-window RMS in a static behind a mutex. * Deliberately not a queue of samples: this path is polled at ~50 Hz by a loop * that only ever asks "is someone talking", and buffering audio nobody reads * would be an unbounded allocation in the middle of a conversation. * ══════════════════════════════════════════════════════════════════════════ */ static AVAudioEngine* g_mon_engine = nil; static int g_mon_running = 0; static int g_mon_code = 0; /* what the successful start reported */ static double g_mon_rms = 0.0; static pthread_mutex_t g_mon_lock = PTHREAD_MUTEX_INITIALIZER; el_val_t mic_monitor_start(void) { /* Idempotent, and it re-reports the ORIGINAL code rather than a bare 1: a * caller that starts twice must not be told AEC is on when the first start * already discovered it was not. */ if (g_mon_running) return (el_val_t)g_mon_code; if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; AVAudioEngine* engine = nil; AVAudioInputNode* input = nil; AVAudioFormat* fmt = nil; int aec = 0; int tapped = 0; @try { engine = [[AVAudioEngine alloc] init]; input = [engine inputNode]; } @catch (NSException* e) { (void)e; return (el_val_t)0; } if (!input) return (el_val_t)0; /* Enable AEC BEFORE reading the format: the voice-processing unit imposes * its own input format, and a tap installed with the pre-VP format would be * rejected at start. */ @try { NSError* vpErr = nil; if ([input respondsToSelector:@selector(setVoiceProcessingEnabled:error:)]) { aec = [input setVoiceProcessingEnabled:YES error:&vpErr] ? 1 : 0; } } @catch (NSException* e) { (void)e; aec = 0; } @try { fmt = [input inputFormatForBus:0]; } @catch (NSException* e) { (void)e; return (el_val_t)0; } if (!fmt || fmt.sampleRate <= 0 || fmt.channelCount == 0) return (el_val_t)0; el_cap_disclose(aec ? "MIC: opening the microphone for live monitoring, echo-cancelled (local)." : "MIC: opening the microphone for live monitoring, NO echo cancellation (local)."); @try { [input installTapOnBus:0 bufferSize:1024 format:fmt block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { (void)when; if (!buf) return; AVAudioFrameCount n = buf.frameLength; if (n == 0) return; double sum = 0.0; /* Whatever the VP unit hands back — float32 is the norm, int16 and * int32 are possible on odd hardware — normalise to -1..1 so the * Float El sees means the same thing on every device. */ if (buf.floatChannelData) { const float* ch = buf.floatChannelData[0]; for (AVAudioFrameCount i = 0; i < n; i++) sum += (double)ch[i] * (double)ch[i]; } else if (buf.int16ChannelData) { const int16_t* ch = buf.int16ChannelData[0]; for (AVAudioFrameCount i = 0; i < n; i++) { double v = (double)ch[i] / 32768.0; sum += v * v; } } else if (buf.int32ChannelData) { const int32_t* ch = buf.int32ChannelData[0]; for (AVAudioFrameCount i = 0; i < n; i++) { double v = (double)ch[i] / 2147483648.0; sum += v * v; } } else { return; } double rms = sqrt(sum / (double)n); if (rms < 0.0) rms = 0.0; if (rms > 1.0) rms = 1.0; pthread_mutex_lock(&g_mon_lock); g_mon_rms = rms; pthread_mutex_unlock(&g_mon_lock); }]; tapped = 1; [engine prepare]; NSError* startErr = nil; if (![engine startAndReturnError:&startErr]) { [input removeTapOnBus:0]; return (el_val_t)0; } } @catch (NSException* e) { (void)e; @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } return (el_val_t)0; } pthread_mutex_lock(&g_mon_lock); g_mon_rms = 0.0; pthread_mutex_unlock(&g_mon_lock); g_mon_engine = engine; g_mon_running = 1; g_mon_code = aec ? 1 : 2; return (el_val_t)g_mon_code; } /* Float in 0..1. Reads the last window the tap computed; never blocks on the * audio thread beyond the mutex, because this is polled inside a turn-taking * loop where a stall IS a missed barge-in. */ el_val_t mic_monitor_rms(void) { double rms = 0.0; pthread_mutex_lock(&g_mon_lock); rms = g_mon_rms; pthread_mutex_unlock(&g_mon_lock); return el_from_float(rms); } el_val_t mic_monitor_stop(void) { AVAudioEngine* engine = g_mon_engine; g_mon_engine = nil; g_mon_running = 0; g_mon_code = 0; if (engine) { @try { [[engine inputNode] removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } @try { [engine stop]; } @catch (NSException* e) { (void)e; } } pthread_mutex_lock(&g_mon_lock); g_mon_rms = 0.0; pthread_mutex_unlock(&g_mon_lock); return (el_val_t)1; } /* ════════════════════════════════════════════════════════════════════════════ * CAMERA * ══════════════════════════════════════════════════════════════════════════ */ static void el_cap_free_bitmap(void* info, const void* data, size_t size) { (void)info; (void)size; free((void*)data); } /* CVPixelBuffer -> CGImage, own-core, no CoreImage. * * The output is pinned to 32BGRA at the AVCaptureVideoDataOutput (see below) * precisely so this conversion can be a memcpy and a CGImageCreate. The * alternative — accepting the camera's native 2vuy/420v and colour-converting * here — would mean either pulling in CoreImage or writing a YUV->RGB matrix in * the file that is supposed to contain no arithmetic. Asking the capture output * for BGRA moves that work into AVFoundation, where it is already written and * already hardware-accelerated. * * The rows are copied out rather than aliased because the CVPixelBuffer is * recycled by the capture session the moment the delegate returns; a CGImage * pointing at it would be pointing at the NEXT frame by the time anyone looked. */ static CGImageRef el_cap_cgimage_from_pixelbuffer(CVPixelBufferRef pb) { if (!pb) return NULL; if (CVPixelBufferGetPixelFormatType(pb) != kCVPixelFormatType_32BGRA) return NULL; if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return NULL; size_t w = CVPixelBufferGetWidth(pb); size_t h = CVPixelBufferGetHeight(pb); size_t src_bpr = CVPixelBufferGetBytesPerRow(pb); const uint8_t* base = (const uint8_t*)CVPixelBufferGetBaseAddress(pb); CGImageRef img = NULL; if (base && w > 0 && h > 0 && src_bpr >= w * 4) { size_t dst_bpr = w * 4; uint8_t* copy = (uint8_t*)malloc(dst_bpr * h); if (copy) { for (size_t y = 0; y < h; y++) { memcpy(copy + y * dst_bpr, base + y * src_bpr, dst_bpr); } CGDataProviderRef dp = CGDataProviderCreateWithData(NULL, copy, dst_bpr * h, el_cap_free_bitmap); if (dp) { CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); if (cs) { img = CGImageCreate(w, h, 8, 32, dst_bpr, cs, (CGBitmapInfo)(kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipFirst), dp, NULL, false, kCGRenderingIntentDefault); CGColorSpaceRelease(cs); } CGDataProviderRelease(dp); /* provider owns `copy` from here */ } else { free(copy); } } } CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); return img; } /* The frame delegate. AVCaptureVideoDataOutput is used rather than * AVCapturePhotoOutput for the same reason periph.swift used it: the photo path * wants KVO and a session owned by an app object, and this runs in a plain CLI * process with no run loop it can assume. A data output just calls back. * * The first frames are dropped on purpose. A camera that has just been powered * on is still converging exposure and white balance, and the first frame is * reliably darker and greener than the room. El's scene-geometry descriptors * are brightness and mean-colour statistics, so handing up an unsettled frame * would not produce a slightly worse answer, it would produce a confidently * wrong one. */ @interface ElCapFrameGrabber : NSObject { CGImageRef _img; int _seen; dispatch_semaphore_t _sem; } - (dispatch_semaphore_t)sem; - (CGImageRef)takeImage; /* transfers ownership to the caller */ @end @implementation ElCapFrameGrabber - (instancetype)init { self = [super init]; if (self) { _img = NULL; _seen = 0; _sem = dispatch_semaphore_create(0); } return self; } - (dispatch_semaphore_t)sem { return _sem; } - (CGImageRef)takeImage { CGImageRef out = _img; _img = NULL; return out; } - (void)dealloc { if (_img) { CGImageRelease(_img); _img = NULL; } } /* Runs on the serial delegate queue, so no lock is needed among callbacks; the * waiter only reads _img after the semaphore has been signalled AND the session * has been stopped, which orders it after the last callback. */ - (void)captureOutput:(AVCaptureOutput*)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection*)connection { (void)output; (void)connection; _seen++; if (_img != NULL || _seen < 5) return; /* let exposure settle */ CVImageBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer); if (!pb) return; CGImageRef img = el_cap_cgimage_from_pixelbuffer(pb); if (!img) return; _img = img; dispatch_semaphore_signal(_sem); } @end /* Bring the camera up, take exactly one settled frame, put it back down. * Returns a +1 CGImageRef the caller releases, or NULL. Bounded at 10s: a * camera held by another process, or one whose TCC grant was revoked between * the check and the open, must fail rather than park. */ static CGImageRef el_cap_grab_frame(void) { AVCaptureSession* session = nil; AVCaptureVideoDataOutput* output = nil; ElCapFrameGrabber* grabber = nil; CGImageRef img = NULL; @try { AVCaptureDevice* dev = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; if (!dev) return NULL; NSError* err = nil; AVCaptureDeviceInput* in = [AVCaptureDeviceInput deviceInputWithDevice:dev error:&err]; if (!in) return NULL; session = [[AVCaptureSession alloc] init]; session.sessionPreset = AVCaptureSessionPresetPhoto; if (![session canAddInput:in]) return NULL; [session addInput:in]; output = [[AVCaptureVideoDataOutput alloc] init]; output.alwaysDiscardsLateVideoFrames = YES; /* Pin the pixel format so the CGImage conversion above stays a memcpy. * Every macOS capture device advertises 32BGRA. */ output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA) }; grabber = [[ElCapFrameGrabber alloc] init]; dispatch_queue_t q = dispatch_queue_create("el.capture.camera", DISPATCH_QUEUE_SERIAL); [output setSampleBufferDelegate:grabber queue:q]; if (![session canAddOutput:output]) return NULL; [session addOutput:output]; el_cap_disclose("CAMERA: opening the camera for one frame (local, never egresses)."); [session startRunning]; } @catch (NSException* e) { (void)e; @try { [session stopRunning]; } @catch (NSException* e2) { (void)e2; } return NULL; } long timed_out = dispatch_semaphore_wait([grabber sem], dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC))); /* Stop first, then detach the delegate, then read. In that order the last * callback has already returned by the time anyone touches the image. */ @try { [session stopRunning]; } @catch (NSException* e) { (void)e; } @try { [output setSampleBufferDelegate:nil queue:NULL]; } @catch (NSException* e) { (void)e; } if (timed_out == 0) img = [grabber takeImage]; return img; } el_val_t camera_available(void) { if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; return (el_val_t)(el_cap_device_present(AVMediaTypeVideo) ? 1 : 0); } el_val_t camera_request_access(void) { return (el_val_t)(el_cap_request(AVMediaTypeVideo) ? 1 : 0); } /* Longest edge of the grid handed to El. 64 is not a resolution, it is a budget: * a 1920x1080 frame is 6.2 MILLION packed RGB ints, and building that as an El * list would cost more time and memory than everything El then does with it. * The descriptors El computes over this — mean colour, brightness, a 3x3 * luminance grid — are region statistics, and region statistics do not get * meaningfully better above a 64-wide grid. The TRUE frame dimensions are * reported separately so nothing downstream has to guess what was thrown away. */ #define EL_CAP_GRID_MAX 64 /* One frame as Map{width, height, grid_w, grid_h, pixels:[Int]}. * * "pixels" is packed R,G,B with NO alpha — three ints per grid cell, row-major * from the TOP-LEFT. (CGBitmapContext lays its buffer out top row first and * CGContextDrawImage does the flip, so row 0 here is the top of the frame, the * same convention periph.swift's grid indexing assumed.) Alpha is dropped * because a camera frame has none worth carrying and it would inflate the list * by a third to say "opaque" six thousand times. */ el_val_t camera_capture_rgb(void) { if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; CGImageRef img = el_cap_grab_frame(); if (!img) return (el_val_t)0; size_t w = CGImageGetWidth(img); size_t h = CGImageGetHeight(img); if (w == 0 || h == 0) { CGImageRelease(img); return (el_val_t)0; } /* Preserve aspect ratio, longest edge capped. */ size_t gw = w, gh = h; size_t longest = w > h ? w : h; if (longest > EL_CAP_GRID_MAX) { double s = (double)EL_CAP_GRID_MAX / (double)longest; gw = (size_t)((double)w * s + 0.5); gh = (size_t)((double)h * s + 0.5); if (gw == 0) gw = 1; if (gh == 0) gh = 1; } size_t bpr = gw * 4; uint8_t* buf = (uint8_t*)calloc(1, bpr * gh); if (!buf) { CGImageRelease(img); return (el_val_t)0; } CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); CGContextRef ctx = cs ? CGBitmapContextCreate(buf, gw, gh, 8, bpr, cs, (CGBitmapInfo)kCGImageAlphaPremultipliedLast) : NULL; if (cs) CGColorSpaceRelease(cs); if (!ctx) { free(buf); CGImageRelease(img); return (el_val_t)0; } /* Nearest-neighbour. This is a decimation for statistics, not a thumbnail * for a human to look at; smoothing would only cost time and blur the very * region boundaries the grid exists to measure. */ CGContextSetInterpolationQuality(ctx, kCGInterpolationNone); CGContextDrawImage(ctx, CGRectMake(0, 0, (CGFloat)gw, (CGFloat)gh), img); CGContextRelease(ctx); CGImageRelease(img); el_val_t pixels = el_list_empty(); for (size_t y = 0; y < gh; y++) { const uint8_t* row = buf + y * bpr; for (size_t x = 0; x < gw; x++) { const uint8_t* p = row + x * 4; /* RGBA8, premultiplied-last */ pixels = el_list_append(pixels, (el_val_t)(int64_t)p[0]); pixels = el_list_append(pixels, (el_val_t)(int64_t)p[1]); pixels = el_list_append(pixels, (el_val_t)(int64_t)p[2]); } } free(buf); el_val_t m = el_map_new((el_val_t)0); if (!m) return (el_val_t)0; m = el_map_set(m, EL_STR("width"), (el_val_t)(int64_t)w); m = el_map_set(m, EL_STR("height"), (el_val_t)(int64_t)h); m = el_map_set(m, EL_STR("grid_w"), (el_val_t)(int64_t)gw); m = el_map_set(m, EL_STR("grid_h"), (el_val_t)(int64_t)gh); m = el_map_set(m, EL_STR("pixels"), pixels); return m; } /* One frame to disk as JPEG, at FULL resolution — the opposite budget from * camera_capture_rgb, and for the opposite reason. A file is not being walked * element-by-element by an interpreter; it costs one ImageIO call and it is the * artefact a human or a later pass will actually look at. The encoder is * ImageIO's because a JPEG encoder is a codec, and re-implementing one in El * would be a large amount of arithmetic that buys nothing: the point of keeping * work in El is the reasoning, not the entropy coding. */ el_val_t camera_capture_jpeg(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return (el_val_t)0; if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; CGImageRef img = el_cap_grab_frame(); if (!img) return (el_val_t)0; int ok = 0; @autoreleasepool { NSString* ns = [NSString stringWithUTF8String:p]; NSURL* url = ns ? [NSURL fileURLWithPath:ns] : nil; if (url) { CGImageDestinationRef dst = CGImageDestinationCreateWithURL((__bridge CFURLRef)url, CFSTR("public.jpeg"), 1, NULL); if (dst) { CGImageDestinationAddImage(dst, img, NULL); ok = CGImageDestinationFinalize(dst) ? 1 : 0; CFRelease(dst); } } } CGImageRelease(img); return (el_val_t)ok; } #endif /* __APPLE__ */