diff --git a/peripheral/.gitignore b/peripheral/.gitignore new file mode 100644 index 0000000..075d90b --- /dev/null +++ b/peripheral/.gitignore @@ -0,0 +1,8 @@ +# Build + runtime artifacts — never committed. +bin/ +# Captured media (camera frames, mic audio) and syntheses. Raw streams stay +# LOCAL and never egress — including into git. +out/ +# Runtime consent + resume state (local, per-machine). +.consent.json +.resume.json diff --git a/peripheral/README.md b/peripheral/README.md new file mode 100644 index 0000000..765fd35 --- /dev/null +++ b/peripheral/README.md @@ -0,0 +1,80 @@ +# peripheral — Neuron's I/O organ (own-core, local, consent-gated) + +The interface made physical. Two afferent senses in, one efferent voice out — +all reached the way the agentic surface reaches any tool. + +``` +MIC (hear) afferent device -> capture -> descriptor -> ingest -> geometry +CAMERA (see) afferent device -> capture -> descriptor -> ingest -> scene-geometry +SPEAKER(speak) efferent render WAV -> PLAY ALOUD out the speaker +``` + +Closes the conversational loop: **hear (mic) -> understand (engram) -> speak (speaker)**. + +## Rails +- **Own-core.** macOS-native only: AVFoundation (camera/mic), CoreAudio voice- + processing (AEC), afplay (speaker), ImageIO/CoreGraphics (frames), hand-rolled + DSP (WAV, LPC, formant synthesis). No cloud, no heavy deps. +- **Local-only.** Raw streams are written to `out/` and never egress. `.gitignore` + keeps captured media out of git. +- **Consent-gated (two locks).** A Neuron-level grant (`grant`/`revoke`) *and* the + OS TCC permission. Sensitive senses (camera/mic) fail closed without both. +- **Disclosed.** Every device touch prints a `[peripheral]` line on stderr. + +## Build +``` +swiftc -O -o bin/periph src/periph.swift \ + -framework AVFoundation -framework CoreMedia -framework Foundation \ + -framework CoreGraphics -framework ImageIO -framework CoreImage +``` + +## Commands +``` +periph grant|revoke # Neuron-level consent +periph status +periph speak # SPEAK ALOUD (efferent) +periph tone [hz] [sec] # own-core WAV synth +periph listen # MIC capture (afferent), 16k mono +periph see # CAMERA one frame (afferent) +periph feat-audio | feat-image # capture -> compact descriptor +periph ingest-audio|ingest-image # descriptor -> engram node (geometry) +periph voiceprint # extract F0 + formants F1-F5 +periph imitate # speak back in that voice (LPC resynthesis) +periph hear-imitate # MIC -> signature -> imitate -> SPEAK ALOUD +periph converse [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] [--live-mic] +``` + +## The afferent metabolism +A capture is never shipped raw. It becomes a **compact descriptor** — the afferent +twin of the music instrument-signature: +- audio -> `[seconds, sr, ch, rms, peak, zcr, centroid, F0]` (~2400-6000x smaller) +- image -> `[w, h, meanRGB, brightness, 3x3 luminance grid]` (~400000x smaller) +- voice -> `[F0, F1..F5, bandwidths]` (11 numbers) + +That descriptor is what the ingest organ (engram `POST /api/nodes`) turns into an +embedded node = geometry. + +## Voice by imitation +`voiceprint`/`imitate` are own-core LPC (autocorrelation + Levinson-Durbin, order +16 @ 16 kHz), formant extraction from the LPC spectral envelope, and source-filter +resynthesis (glottal impulse train at F0 through the all-pole formant filter). A +voice is grabbed by ear as ~a dozen numbers and spoken back — **no training, no +stolen voice.** Measured fidelity on real speech: resynthesized formants match the +source within 2-3%. The full phoneme->formant path for *novel* sentences is the +speech faculty's seam (`elp` audio surface profile); this engine provides the +formant synthesis primitive it renders through. + +## Interruptibility (native turn-taking) +`converse` plays the utterance as an ordered, salience-tagged **meaning-plan** +while the mic listens (full-duplex, AEC on so it never barges in on its own voice): +- **barge-in**: user speech -> pause on the spot (sample-accurate), not "finish the buffer." +- **yield-or-hold**: a decision grounded in the current segment's salience + progress + + the interrupter's authority — YIELD (stop) or HOLD ("hang on, let me finish"). +- **backchannel** ("mm-hm"): brief/low -> keep going, resume seamlessly. +- **resumable**: on yield the remaining plan persists (`.resume.json`); `--resume` + picks the thread back up ("as I was saying"). + +Live full-duplex uses `--live-mic` (OS AEC). Injected `--barge-at` drives the +decision loop deterministically for testing. +``` +``` diff --git a/peripheral/src/periph.swift b/peripheral/src/periph.swift new file mode 100644 index 0000000..d81c543 --- /dev/null +++ b/peripheral/src/periph.swift @@ -0,0 +1,939 @@ +// periph.swift — Neuron's PERIPHERAL I/O organ (own-core, LOCAL, CONSENT-GATED). +// +// The interface made physical: +// MIC (hear) = afferent : device -> capture -> [ingest -> geometry] +// CAMERA (see) = afferent : device -> capture -> [ingest -> scene-geometry] +// SPEAKER(speak) = efferent : [render WAV] -> PLAY ALOUD out the speaker +// +// Rails: own-core (AVFoundation / CoreAudio / afplay — all ship with macOS), +// no cloud, no heavy deps, raw streams stay LOCAL and never egress, +// every device access is CONSENT-GATED and DISCLOSED. +// +// Full-duplex CONVERSE mode implements native interruptibility: while the +// speaker plays the utterance (a persistent, segmented meaning-plan), the mic +// listens; on user speech it interrupts instantly, then DECIDES yield-or-hold +// grounded in the salience of what it is mid-saying, and can RESUME the thread. +// +// Build: swiftc -O -o peripheral/bin/periph peripheral/src/periph.swift \ +// -framework AVFoundation -framework CoreMedia -framework Foundation + +import Foundation +import AVFoundation +import CoreMedia +import CoreGraphics +import ImageIO +import CoreImage + +// ---------------------------------------------------------------------------- +// Disclosure — every peripheral touch is announced on stderr. Nothing is silent. +// ---------------------------------------------------------------------------- +func disclose(_ msg: String) { + FileHandle.standardError.write(" [peripheral] \(msg)\n".data(using: .utf8)!) +} +func emit(_ obj: [String: Any]) { // machine-readable event on stdout (JSON line) + if let d = try? JSONSerialization.data(withJSONObject: obj), + let s = String(data: d, encoding: .utf8) { + print(s) + } +} +func die(_ msg: String) -> Never { + disclose("ERROR: \(msg)") + emit(["ok": false, "error": msg]) + exit(1) +} + +// ---------------------------------------------------------------------------- +// Consent store — Neuron's OWN gate, on top of the OS (TCC) gate. Two locks on +// the sensitive senses. Persisted locally next to the binary's organ dir. +// ---------------------------------------------------------------------------- +struct Consent { + static let path: String = { + let dir = ProcessInfo.processInfo.environment["PERIPH_HOME"] + ?? FileManager.default.currentDirectoryPath + "/peripheral" + return dir + "/.consent.json" + }() + + static func load() -> [String: Bool] { + guard let d = FileManager.default.contents(atPath: path), + let o = try? JSONSerialization.jsonObject(with: d) as? [String: Bool] + else { return ["camera": false, "mic": false] } + return o + } + static func save(_ g: [String: Bool]) { + let d = try! JSONSerialization.data(withJSONObject: g, options: [.prettyPrinted]) + try? d.write(to: URL(fileURLWithPath: path)) + } + // Neuron-level gate. Sensitive senses (camera/mic) require an explicit grant. + static func require(_ device: String) { + let g = load() + if g[device] != true { + die("CONSENT DENIED for '\(device)'. The user has not granted this sense. " + + "Run: periph grant \(device) (raw streams stay local, never egress).") + } + disclose("consent OK (Neuron-level) for '\(device)' — local only, never egresses.") + } +} + +// ---------------------------------------------------------------------------- +// OS (TCC) permission — the second lock. AVFoundation prompts the user the first +// time; if denied, we fail cleanly rather than hang. +// ---------------------------------------------------------------------------- +func requireOSAccess(_ media: AVMediaType, _ label: String) { + let status = AVCaptureDevice.authorizationStatus(for: media) + switch status { + case .authorized: + disclose("consent OK (OS/TCC) for \(label).") + return + case .notDetermined: + disclose("requesting OS permission for \(label) (first use) — user must grant...") + let sem = DispatchSemaphore(value: 0) + var ok = false + AVCaptureDevice.requestAccess(for: media) { granted in ok = granted; sem.signal() } + _ = sem.wait(timeout: .now() + 30) + if !ok { die("OS permission for \(label) was not granted.") } + disclose("consent OK (OS/TCC) for \(label).") + case .denied, .restricted: + die("OS permission for \(label) is DENIED in System Settings > Privacy. " + + "Grant it to the controlling terminal/app, then retry.") + @unknown default: + die("unknown OS permission state for \(label).") + } +} + +// ---------------------------------------------------------------------------- +// Own-core WAV writer (16-bit PCM). No library — proves we own the medium. +// ---------------------------------------------------------------------------- +func writeWav(_ url: URL, samples: [Int16], sampleRate: Int, channels: Int = 1) { + var data = Data() + func u32(_ v: UInt32) { var x = v.littleEndian; data.append(Data(bytes: &x, count: 4)) } + func u16(_ v: UInt16) { var x = v.littleEndian; data.append(Data(bytes: &x, count: 2)) } + let bytesPerSample = 2 + let dataBytes = samples.count * bytesPerSample + let byteRate = sampleRate * channels * bytesPerSample + data.append("RIFF".data(using: .ascii)!); u32(UInt32(36 + dataBytes)) + data.append("WAVE".data(using: .ascii)!) + data.append("fmt ".data(using: .ascii)!); u32(16); u16(1); u16(UInt16(channels)) + u32(UInt32(sampleRate)); u32(UInt32(byteRate)) + u16(UInt16(channels * bytesPerSample)); u16(16) + data.append("data".data(using: .ascii)!); u32(UInt32(dataBytes)) + for s in samples { var x = s.littleEndian; data.append(Data(bytes: &x, count: 2)) } + try? data.write(to: url) +} + +// Read a WAV's basic geometry (own-core header parse). Walks chunks to find +// 'fmt ' and 'data' — robust to JUNK/FLLR padding chunks (AVAudioRecorder emits them). +func wavInfo(_ path: String) -> (sampleRate: Int, channels: Int, bits: Int, frames: Int)? { + guard let d = FileManager.default.contents(atPath: path), d.count > 44 else { return nil } + func rd16(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1]) << 8) } + func rd32(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1])<<8) | (Int(d[o+2])<<16) | (Int(d[o+3])<<24) } + var channels = 0, sampleRate = 0, bits = 0, dataSize = 0 + var o = 12 + while o + 8 <= d.count { + let id = String(bytes: d[o.. 0 && bits > 0) ? dataSize / (channels * bits/8) : 0 + return (sampleRate, channels, bits, frames) +} + +// ---------------------------------------------------------------------------- +// SPEAKER (efferent) — play a WAV ALOUD. Own-core: afplay ships with macOS. +// ---------------------------------------------------------------------------- +func speak(_ wavPath: String) { + guard FileManager.default.fileExists(atPath: wavPath) else { die("no such file: \(wavPath)") } + disclose("SPEAKER: playing '\(wavPath)' ALOUD out the local speaker (efferent).") + let p = Process() + p.executableURL = URL(fileURLWithPath: "/usr/bin/afplay") + p.arguments = [wavPath] + try? p.run(); p.waitUntilExit() + let ok = p.terminationStatus == 0 + disclose(ok ? "SPEAKER: done — Neuron spoke aloud." : "SPEAKER: afplay failed.") + if let i = wavInfo(wavPath) { + emit(["ok": ok, "op": "speak", "file": wavPath, "played_aloud": ok, + "sample_rate": i.sampleRate, "channels": i.channels, + "seconds": Double(i.frames)/Double(max(i.sampleRate,1))]) + } else { + emit(["ok": ok, "op": "speak", "file": wavPath, "played_aloud": ok]) + } +} + +// ---------------------------------------------------------------------------- +// MIC (afferent) — capture N seconds -> 16k mono 16-bit WAV (formant-ready). +// ---------------------------------------------------------------------------- +func listen(seconds: Double, out: String) { + Consent.require("mic") + requireOSAccess(.audio, "microphone") + disclose("MIC: capturing \(seconds)s -> '\(out)' (16 kHz mono, LOCAL, never egresses).") + + let url = URL(fileURLWithPath: out) + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: 16000.0, + AVNumberOfChannelsKey: 1, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + ] + guard let rec = try? AVAudioRecorder(url: url, settings: settings) else { + die("could not open the microphone recorder.") + } + rec.record() + Thread.sleep(forTimeInterval: seconds) + rec.stop() + // let the file flush + Thread.sleep(forTimeInterval: 0.1) + if let i = wavInfo(out) { + disclose("MIC: captured \(i.frames) frames @ \(i.sampleRate)Hz — ready to hand to the ingest organ.") + emit(["ok": true, "op": "listen", "file": out, "sample_rate": i.sampleRate, + "channels": i.channels, "frames": i.frames, + "seconds": Double(i.frames)/Double(max(i.sampleRate,1)), + "next": "ingest -> phonetic/voice geometry"]) + } else { + die("mic capture produced no readable WAV.") + } +} + +// ---------------------------------------------------------------------------- +// CAMERA (afferent) — capture ONE frame -> JPEG on disk. +// ---------------------------------------------------------------------------- +// Grab one video frame via AVCaptureVideoDataOutput (CLI-safe; no KVO/photo classes). +final class FrameGrabber: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { + let sem = DispatchSemaphore(value: 0) + var cgImage: CGImage? + var seen = 0 + let cictx = CIContext(options: nil) + func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection) { + seen += 1 + if cgImage != nil || seen < 5 { return } // let exposure settle a few frames + guard let pb = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } + let ci = CIImage(cvPixelBuffer: pb) + cgImage = cictx.createCGImage(ci, from: ci.extent) + sem.signal() + } +} +func see(out: String) { + Consent.require("camera") + requireOSAccess(.video, "camera") + disclose("CAMERA: capturing one frame -> '\(out)' (LOCAL, never egresses).") + + let session = AVCaptureSession() + session.sessionPreset = .photo + guard let device = AVCaptureDevice.default(for: .video), + let input = try? AVCaptureDeviceInput(device: device), + session.canAddInput(input) else { die("no camera device available.") } + session.addInput(input) + let output = AVCaptureVideoDataOutput() + output.alwaysDiscardsLateVideoFrames = true + let grabber = FrameGrabber() + output.setSampleBufferDelegate(grabber, queue: DispatchQueue(label: "periph.cam")) + guard session.canAddOutput(output) else { die("cannot add video output.") } + session.addOutput(output) + session.startRunning() + + if grabber.sem.wait(timeout: .now() + 10) == .timedOut { session.stopRunning(); die("camera capture timed out.") } + session.stopRunning() + + guard let cg = grabber.cgImage, + let dst = CGImageDestinationCreateWithURL(URL(fileURLWithPath: out) as CFURL, + "public.jpeg" as CFString, 1, nil) + else { die("camera returned no frame.") } + CGImageDestinationAddImage(dst, cg, nil) + guard CGImageDestinationFinalize(dst) else { die("could not write JPEG.") } + let bytes = ((try? FileManager.default.attributesOfItem(atPath: out))?[.size] as? Int) ?? 0 + disclose("CAMERA: wrote \(cg.width)x\(cg.height) frame (\(bytes) bytes) — ready for scene-geometry ingest.") + emit(["ok": true, "op": "see", "file": out, "width": cg.width, "height": cg.height, + "bytes": bytes, "next": "ingest -> scene-geometry"]) +} + +// ============================================================================ +// FEAT — the afferent METABOLISM: a raw capture becomes a COMPACT descriptor +// (a few dozen numbers), the mirror of the efferent signature. This is what +// gets handed to the ingest organ as geometry — NOT the raw stream. Own-core. +// ============================================================================ + +// Read all 16-bit PCM samples from a WAV (own-core). +func readWavSamples(_ path: String) -> (samples: [Double], sr: Int, ch: Int)? { + guard let d = FileManager.default.contents(atPath: path), d.count > 44 else { return nil } + func rd16(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1]) << 8) } + func rd32(_ o: Int) -> Int { Int(d[o]) | (Int(d[o+1])<<8) | (Int(d[o+2])<<16) | (Int(d[o+3])<<24) } + var ch = 0, sr = 0, bits = 0 + var o = 12 + while o + 8 <= d.count { + let id = String(bytes: d[o.. 0 else { return nil } + var samples = [Double](); let start = o + 8 + let end = min(start + sz, d.count - 1) + var i = start + while i + 1 < end { + var v = Int(rd16(i)); if v >= 32768 { v -= 65536 } + samples.append(Double(v) / 32768.0) + i += 2 * ch // take channel 0 if stereo + } + return (samples, sr, ch) + } + o += 8 + sz + (sz & 1) + } + return nil +} + +// Audio descriptor = compact sound/voice signature (energy, ZCR, centroid, F0). +// The seed for phonetic geometry + the hear->imitate voice-signature. +func computeAudio(_ path: String) -> (content: String, vector: [Double], extra: [String: Any]) { + guard let (s, sr, ch) = readWavSamples(path), !s.isEmpty else { die("cannot read PCM from \(path)") } + let n = s.count + let seconds = Double(n) / Double(sr) + var sumsq = 0.0, peak = 0.0, zc = 0.0 + for i in 0.. 0 && (s[i-1] < 0) != (s[i] < 0) { zc += 1 } + } + let rms = (sumsq / Double(n)).squareRoot() + let zcr = zc / Double(n) * Double(sr) // ~2*dominant freq for tonal + // Spectral centroid via a coarse DFT on a mid window (own-core). + let W = min(2048, n); let off = max(0, (n - W)/2) + var num = 0.0, den = 0.0 + let bins = 64 + for k in 1.. 0 ? num/den : 0 + // F0 via autocorrelation (voice pitch) over plausible speech range 70-400 Hz. + var bestLag = 0; var bestCorr = 0.0 + let lagMin = sr/400, lagMax = min(sr/70, n-1) + if lagMax > lagMin { + for lag in lagMin...lagMax { + var c = 0.0 + var i = 0; while i + lag < min(n, off+W) { c += s[off+i]*s[off+i+lag]; i += 1 } + if c > bestCorr { bestCorr = c; bestLag = lag } + } + } + let f0 = bestLag > 0 ? Double(sr)/Double(bestLag) : 0 + let vector: [Double] = [seconds, Double(sr), Double(ch), rms, peak, zcr, centroid, f0] + let content = String(format: + "Heard sound (afferent, mic): %.2fs at %dHz. RMS energy %.3f, peak %.3f, " + + "zero-crossing rate %.0fHz, spectral centroid %.0fHz, estimated voice pitch F0 %.0fHz. " + + "Compact voice/sound signature (%d numbers) — phonetic geometry + hear-to-imitate seed.", + seconds, sr, rms, peak, zcr, centroid, f0, vector.count) + disclose("FEAT(audio): \(vector.count)-number signature vs \(n) raw samples (~\(n/max(vector.count,1))x compression).") + return (content, vector, ["f0_hz": f0, "centroid_hz": centroid, "zcr_hz": zcr, + "rms": rms, "seconds": seconds, "raw_samples": n]) +} +func featAudio(_ path: String) { + let r = computeAudio(path) + var out: [String: Any] = ["ok": true, "op": "feat-audio", "file": path, + "vector": r.vector, "content": r.content, + "ingest": ["node_type": "Observation", "tier": "Episodic", "content": r.content]] + r.extra.forEach { out[$0] = $1 } + emit(out) +} + +// Image descriptor = compact scene-geometry (dims, brightness, region grid). +func computeImage(_ path: String) -> (content: String, vector: [Double], extra: [String: Any]) { + guard let src = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil), + let img = CGImageSourceCreateImageAtIndex(src, 0, nil) else { die("cannot decode image \(path)") } + let w = img.width, h = img.height + let cs = CGColorSpaceCreateDeviceRGB() + let bpr = w * 4 + var buf = [UInt8](repeating: 0, count: h * bpr) + guard let ctx = CGContext(data: &buf, width: w, height: h, bitsPerComponent: 8, + bytesPerRow: bpr, space: cs, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { + die("cannot rasterize image") + } + ctx.draw(img, in: CGRect(x: 0, y: 0, width: w, height: h)) + // 3x3 region average luminance + overall average color. + var rAvg = 0.0, gAvg = 0.0, bAvg = 0.0 + var grid = [Double](repeating: 0, count: 9); var gridN = [Int](repeating: 0, count: 9) + let step = max(1, (w*h)/40000) // subsample for speed + var count = 0; var idx = 0 + while idx < w*h { + let x = idx % w, y = idx / w + let p = y*bpr + x*4 + let r = Double(buf[p]), g = Double(buf[p+1]), b = Double(buf[p+2]) + rAvg += r; gAvg += g; bAvg += b; count += 1 + let cell = (min(2, y*3/h))*3 + min(2, x*3/w) + grid[cell] += 0.299*r + 0.587*g + 0.114*b; gridN[cell] += 1 + idx += step + } + if count == 0 { die("no pixels sampled") } + rAvg /= Double(count); gAvg /= Double(count); bAvg /= Double(count) + for i in 0..<9 { grid[i] = gridN[i] > 0 ? grid[i]/Double(gridN[i]) : 0 } + let bright = (0.299*rAvg + 0.587*gAvg + 0.114*bAvg)/255.0 + let vector = [Double(w), Double(h), rAvg/255, gAvg/255, bAvg/255, bright] + grid.map { $0/255 } + let content = String(format: + "Saw scene (afferent, camera): %dx%d frame. Mean color rgb(%.0f,%.0f,%.0f), " + + "brightness %.2f. 3x3 luminance grid [%.0f %.0f %.0f / %.0f %.0f %.0f / %.0f %.0f %.0f]. " + + "Compact scene-geometry (%d numbers) vs %d pixel-channels.", + w, h, rAvg, gAvg, bAvg, bright, + grid[0],grid[1],grid[2],grid[3],grid[4],grid[5],grid[6],grid[7],grid[8], + vector.count, w*h*3) + disclose("FEAT(image): \(vector.count)-number scene-geometry vs \(w*h*3) pixel-channels (~\(w*h*3/max(vector.count,1))x).") + return (content, vector, ["width": w, "height": h, "brightness": bright]) +} +func featImage(_ path: String) { + let r = computeImage(path) + var out: [String: Any] = ["ok": true, "op": "feat-image", "file": path, + "vector": r.vector, "content": r.content, + "ingest": ["node_type": "Observation", "tier": "Episodic", "content": r.content]] + r.extra.forEach { out[$0] = $1 } + emit(out) +} + +// The afferent WIRE — hand a capture's descriptor to the ingest organ (engram), +// where it becomes an embedded node = GEOMETRY. Own-core URLSession POST. +// LOCAL only: point at a local engram; raw stream never leaves the machine. +func postNode(engramURL: String, content: String, label: String, tags: [String]) -> String? { + guard let url = URL(string: engramURL + "/api/nodes") else { return nil } + let body: [String: Any] = ["content": content, "node_type": "Observation", + "label": label, "tier": "Episodic", + "salience": 0.7, "importance": 0.6, "confidence": 0.9, + "tags": tags] + var req = URLRequest(url: url); req.httpMethod = "POST" + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + req.httpBody = try? JSONSerialization.data(withJSONObject: body) + let sem = DispatchSemaphore(value: 0); var out: String? + URLSession.shared.dataTask(with: req) { data, _, _ in + if let d = data { out = String(data: d, encoding: .utf8) } + sem.signal() + }.resume() + _ = sem.wait(timeout: .now() + 15) + return out +} +func ingest(_ path: String, kind: String, engramURL: String) { + let r = kind == "audio" ? computeAudio(path) : computeImage(path) + let label = kind == "audio" ? "heard:mic" : "saw:camera" + disclose("INGEST: handing \(kind) descriptor to the ingest organ at \(engramURL) (LOCAL) -> geometry.") + guard let resp = postNode(engramURL: engramURL, content: r.content, label: label, + tags: ["peripheral", kind == "audio" ? "afferent-mic" : "afferent-camera"]) else { + die("ingest POST failed (no local engram at \(engramURL)?)") + } + // pull the node id out of the response (own-core, tolerant) + var nodeId = "" + if let d = resp.data(using: .utf8), + let o = try? JSONSerialization.jsonObject(with: d) as? [String: Any] { + nodeId = (o["id"] as? String) ?? (o["node_id"] as? String) ?? "" + } + disclose("INGEST: landed as node \(nodeId.isEmpty ? "(see response)" : nodeId) — the capture is now geometry in the engram.") + emit(["ok": !nodeId.isEmpty, "op": "ingest-\(kind)", "file": path, + "node_id": nodeId, "engram_response": resp, "content": r.content, + "vector": r.vector]) +} + +// ============================================================================ +// VOICE BY IMITATION — hear a voice, grab its compact SIGNATURE (pitch + +// formants F1-F5 via LPC), and speak back in that voice by source-filter +// resynthesis. Own-core DSP (physics), no training, no stolen voice. The +// afferent twin of the music instrument-signature: a voice = a few dozen +// numbers, not a corpus. +// ============================================================================ + +func hamming(_ x: [Double]) -> [Double] { + let n = x.count; if n < 2 { return x } + return (0.. [Double] { + var r = [Double](repeating: 0, count: p+1) + for lag in 0...p { var s = 0.0; var i = lag; while i < x.count { s += x[i]*x[i-lag]; i += 1 }; r[lag] = s } + return r +} +// Levinson-Durbin -> LPC coeffs a[0..p] (A(z)=1+sum a[k]z^-k) and residual energy. +func levinson(_ r: [Double], _ p: Int) -> (a: [Double], err: Double) { + var a = [Double](repeating: 0, count: p+1); a[0] = 1 + var err = r[0] + if err <= 0 { return (a, 0) } + for i in 1...p { + var acc = r[i] + if i > 1 { for j in 1.. 1 { for j in 1.. [(f: Double, bw: Double)] { + let p = a.count - 1 + let steps = 512 + var mag = [Double](repeating: 0, count: steps) + for s in 0.. 0..sr/2 + var re = 0.0, im = 0.0 + for k in 0...p { re += a[k]*cos(w*Double(k)); im -= a[k]*sin(w*Double(k)) } + mag[s] = 1.0 / max((re*re+im*im).squareRoot(), 1e-9) + } + var peaks: [(f: Double, bw: Double)] = [] + for s in 1..<(steps-1) where mag[s] > mag[s-1] && mag[s] >= mag[s+1] { + let f = Double(s) * Double(sr) / 2 / Double(steps) + if f > 150 && f < 5200 { + // crude bandwidth: width where magnitude falls to peak/sqrt(2) + let thr = mag[s]/1.4142 + var lo = s; while lo > 0 && mag[lo] > thr { lo -= 1 } + var hi = s; while hi < steps-1 && mag[hi] > thr { hi += 1 } + let bw = Double(hi-lo) * Double(sr) / 2 / Double(steps) + peaks.append((f, bw)) + } + } + return Array(peaks.prefix(5)) +} +func pitchOf(_ frame: [Double], sr: Int) -> Double { + let n = frame.count + let lagMin = sr/400, lagMax = min(sr/70, n-1) + if lagMax <= lagMin { return 0 } + var r0 = 0.0; for v in frame { r0 += v*v } + if r0 < 1e-5 { return 0 } + var bestLag = 0; var best = 0.0 + for lag in lagMin...lagMax { var c = 0.0; var i = lag; while i < n { c += frame[i]*frame[i-lag]; i += 1 }; if c > best { best = c; bestLag = lag } } + return (best / r0 > 0.30 && bestLag > 0) ? Double(sr)/Double(bestLag) : 0 // voiced? +} + +let LPC_ORDER = 16 +let FRAME = 400 // 25ms @16k +let HOP = 160 // 10ms + +// Extract Will's voice-signature: averaged F0 + formants over voiced frames. +func voiceprint(_ path: String) -> (f0: Double, f0lo: Double, f0hi: Double, formants: [(Double,Double)], content: String) { + guard let (x, sr, _) = readWavSamples(path), x.count > FRAME else { die("cannot read speech from \(path)") } + var f0s: [Double] = [] + var fbank: [[Double]] = [[],[],[],[],[]] + var bbank: [[Double]] = [[],[],[],[],[]] + var pos = 0 + while pos + FRAME <= x.count { + let raw = Array(x[pos.. 0 { // voiced frame only + f0s.append(f0) + let r = autocorr(hamming(raw), LPC_ORDER) + if r[0] > 1e-6 { + let (a, _) = levinson(r, LPC_ORDER) + let fs = formants(a, sr: sr) + for (i, fm) in fs.enumerated() where i < 5 { fbank[i].append(fm.f); bbank[i].append(fm.bw) } + } + } + pos += HOP + } + func med(_ v: [Double]) -> Double { v.isEmpty ? 0 : v.sorted()[v.count/2] } + let f0med = med(f0s) + let f0lo = f0s.isEmpty ? 0 : f0s.sorted().first! + let f0hi = f0s.isEmpty ? 0 : f0s.sorted().last! + var forms: [(Double,Double)] = [] + for i in 0..<5 where !fbank[i].isEmpty { forms.append((med(fbank[i]), med(bbank[i]))) } + let fstr = forms.map { String(format:"%.0f", $0.0) }.joined(separator: "/") + let content = String(format: + "Voice-signature (afferent, heard a voice): pitch F0 %.0fHz (range %.0f-%.0fHz), " + + "formants F1-F5 = %@ Hz. Compact voiceprint (%d numbers) — grabbed by ear for imitation, not trained.", + f0med, f0lo, f0hi, fstr, 1 + forms.count*2) + return (f0med, f0lo, f0hi, forms, content) +} + +// IMITATE: LPC analysis-resynthesis. Reconstruct the heard voice from its +// per-frame filter model + pitch — the voice rebuilt from its signature. +func imitate(inPath: String, outPath: String) { + guard let (x, sr, _) = readWavSamples(inPath), x.count > FRAME else { die("cannot read speech from \(inPath)") } + var out = [Double](repeating: 0, count: x.count) + var state = [Double](repeating: 0, count: LPC_ORDER) // past outputs + var phase = 0.0 + var lastF0 = 0.0 + var pos = 0 + while pos + FRAME <= x.count { + let raw = Array(x[pos.. 0 ? f0 : (lastF0 > 0 ? lastF0 : 0) + lastF0 = f0 + for i in 0..= x.count { break } + var e = 0.0 + if useF0 > 0 { // voiced: glottal impulse train + phase += useF0/Double(sr) + if phase >= 1.0 { phase -= 1.0; e = sqrt(Double(sr)/useF0) } // energy-normalized impulse + } else { // unvoiced: noise + e = Double.random(in: -1...1) + } + var y = gain * e + for k in 1...LPC_ORDER { y -= a[k]*state[k-1] } + for k in stride(from: LPC_ORDER-1, through: 1, by: -1) { state[k] = state[k-1] } + state[0] = y + out[idx] = y + } + pos += HOP + } + // normalize to peak 0.9 + let peak = out.map { abs($0) }.max() ?? 1 + let scale = peak > 1e-9 ? 0.9/peak : 1 + let samples = out.map { Int16(max(-32767, min(32767, $0*scale*32767))) } + writeWav(URL(fileURLWithPath: outPath), samples: samples, sampleRate: sr) + let vp = voiceprint(inPath) + disclose(String(format: "IMITATE: rebuilt the voice from its signature (F0 %.0fHz, formants %@) -> %@", + vp.f0, vp.formants.map{String(format:"%.0f",$0.0)}.joined(separator:"/"), outPath)) + emit(["ok": true, "op": "imitate", "in": inPath, "out": outPath, + "f0_hz": vp.f0, "f0_range": [vp.f0lo, vp.f0hi], + "formants_hz": vp.formants.map { $0.0 }, + "method": "LPC analysis-resynthesis (own-core, no training, no stolen voice)"]) +} + +// ============================================================================ +// CONVERSE (full-duplex) — the interruptible conversational loop. +// The utterance is a persistent, ordered meaning-plan of SEGMENTS, each with +// a salience. The speaker plays them; the mic listens concurrently. On user +// speech: pause INSTANTLY, classify (backchannel vs barge-in), then DECIDE +// yield-or-hold from the salience of the current segment + the social read. +// Yielded utterances persist their remaining plan so Neuron can RESUME. +// ============================================================================ +struct Segment { let file: String; let salience: Double; let text: String } + +enum Decision { case backchannelContinue, hold, yield } + +// The yield-or-hold DECISION — grounded, contextual. Not a fixed rule. +func decide(currentSalience: Double, progress: Double, + interrupterAuthority: Double, isBackchannel: Bool) -> Decision { + if isBackchannel { return .backchannelContinue } // "mm-hm" => keep going + // Holding the floor is justified when what I'm saying matters AND I'm nearly + // done (cheap to finish) AND the interrupter isn't high-priority. + let holdScore = currentSalience * 0.6 + progress * 0.4 + if holdScore >= 0.6 && interrupterAuthority < 0.8 { return .hold } + return .yield // default: be polite, let them in +} + +final class Conversation { + let engine = AVAudioEngine() + let player = AVAudioPlayerNode() + var micLive = false + // VAD state (shared with the audio tap thread) + let lock = NSLock() + var micRMS: Float = 0 + var speechFrames = 0 // consecutive above-threshold frames + var onsetHandled = false + + let resumePath: String + init(resumePath: String) { self.resumePath = resumePath } + + // Try to bring the mic up as a live VAD. Returns false if unavailable/denied. + func startMic() -> Bool { + let status = AVCaptureDevice.authorizationStatus(for: .audio) + if Consent.load()["mic"] != true || status != .authorized { + disclose("CONVERSE: live mic not available (consent/OS) — using injected barge events for the proof.") + return false + } + let input = engine.inputNode + // Acoustic echo cancellation: the OS voice-processing unit subtracts our + // own speaker output from the mic so Neuron does NOT hear itself and + // barge in on its own voice. This is what makes real-room barge-in work. + do { try input.setVoiceProcessingEnabled(true); disclose("CONVERSE: AEC on (echo-cancelled mic — won't self-interrupt).") } + catch { disclose("CONVERSE: AEC unavailable (\(error)); raising VAD floor instead.") } + let fmt = input.inputFormat(forBus: 0) + if fmt.sampleRate == 0 { return false } + input.installTap(onBus: 0, bufferSize: 1024, format: fmt) { [weak self] buf, _ in + guard let self = self, let ch = buf.floatChannelData?[0] else { return } + let n = Int(buf.frameLength) + var sum: Float = 0 + for i in 0.. 0 ? (sum / Float(n)).squareRoot() : 0 + self.lock.lock(); self.micRMS = rms; self.lock.unlock() + } + micLive = true + disclose("CONVERSE: full-duplex — mic listening WHILE speaking (barge-in armed).") + return true + } + + func run(_ segs: [Segment], interrupterAuthority: Double, + injectBargeAt: Double?, injectKind: String, startIndex: Int, liveMic: Bool) { + engine.attach(player) + let firstFmt = (try? AVAudioFile(forReading: URL(fileURLWithPath: segs[startIndex].file)))?.processingFormat + ?? AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1)! + engine.connect(player, to: engine.mainMixerNode, format: firstFmt) + if liveMic { _ = startMic() } + else { disclose("CONVERSE: deterministic mode (live mic off) — barge events \(injectBargeAt != nil ? "injected" : "none").") } + do { try engine.start() } catch { die("audio engine failed to start: \(error)") } + player.play() + + let injectDeadline = injectBargeAt.map { Date().addingTimeInterval($0) } + var injectedFired = false + var idx = startIndex + + segmentLoop: while idx < segs.count { + let seg = segs[idx] + guard let f = try? AVAudioFile(forReading: URL(fileURLWithPath: seg.file)) else { + disclose("CONVERSE: missing segment '\(seg.file)', skipping."); idx += 1; continue + } + let dur = Double(f.length) / f.processingFormat.sampleRate + disclose(String(format: "CONVERSE: speaking segment %d/%d (salience %.2f) — \"%@\"", + idx+1, segs.count, seg.salience, seg.text)) + emit(["op": "converse", "event": "speaking", "segment": idx, + "salience": seg.salience, "text": seg.text]) + + let done = DispatchSemaphore(value: 0) + // .dataPlayedBack: completion fires only after the audio has actually + // played OUT the DAC (not merely been consumed) — so the tail is never + // clipped and playback always runs the FULL file length. + player.scheduleFile(f, at: nil, completionCallbackType: .dataPlayedBack) { _ in done.signal() } + player.play() + + // Monitor this segment: poll VAD / injected event until it finishes. + let segStart = Date() + while done.wait(timeout: .now() + 0.02) == .timedOut { + let elapsed = Date().timeIntervalSince(segStart) + let progress = min(elapsed / max(dur, 0.001), 1.0) + + // --- detect an onset (live mic OR injected) --- + var onset = false + if micLive { + lock.lock(); let rms = micRMS; lock.unlock() + if rms > 0.02 { speechFrames += 1 } else { speechFrames = 0 } + if speechFrames >= 3 && !onsetHandled { onset = true } // ~60ms of voice + } + if let dl = injectDeadline, !injectedFired, Date() >= dl, !onsetHandled { onset = true; injectedFired = true } + + if onset { + onsetHandled = true + // (1) BARGE-IN: pause INSTANTLY, on the spot. + player.pause() + let tBarge = Date().timeIntervalSince(segStart) + disclose(String(format: "CONVERSE: << user speech at %.2fs into segment %d — PAUSED instantly >>", tBarge, idx+1)) + emit(["op": "converse", "event": "barge_in", "segment": idx, + "at_seconds": tBarge, "progress": progress]) + + // (2) classify backchannel vs real barge-in + let isBackchannel = classifyBackchannel(injected: injectDeadline != nil, + kind: injectKind) + let d = decide(currentSalience: seg.salience, progress: progress, + interrupterAuthority: interrupterAuthority, + isBackchannel: isBackchannel) + switch d { + case .backchannelContinue: + disclose("CONVERSE: read as BACKCHANNEL (\"mm-hm\") — keep going, resume seamlessly.") + emit(["op": "converse", "event": "backchannel_continue", "segment": idx]) + onsetHandled = false; speechFrames = 0 + player.play() // seamless resume + case .hold: + disclose("CONVERSE: HOLD the floor — \"hang on, let me finish this thought.\" (high salience, nearly done)") + emit(["op": "converse", "event": "hold_floor", "segment": idx, + "salience": seg.salience, "progress": progress]) + onsetHandled = false; speechFrames = 0 + player.play() // finish the segment, THEN yield + // after this segment completes we yield the remainder + _ = done.wait(timeout: .now() + dur + 1.0) + persistResume(segs: segs, from: idx + 1, reason: "held-then-yield") + finish(); return + case .yield: + disclose("CONVERSE: YIELD — stop, let them in. Remembering where I was (resumable).") + player.stop() + persistResume(segs: segs, from: idx, reason: "yield") + emit(["op": "converse", "event": "yield", "interrupted_segment": idx, + "resume_from": idx]) + finish(); return + } + } + } + emit(["op": "converse", "event": "segment_done", "segment": idx]) + idx += 1 + } + // whole utterance completed uninterrupted + clearResume() + disclose("CONVERSE: utterance complete (uninterrupted).") + emit(["ok": true, "op": "converse", "event": "complete", "segments": segs.count]) + finish() + } + + // A backchannel is brief/low. Injected kind lets us prove both paths headlessly; + // the live path would measure post-onset duration & energy. + func classifyBackchannel(injected: Bool, kind: String) -> Bool { + if injected { return kind == "backchannel" } + // live: sample ~250ms after onset; if speech already died away, it was a backchannel + Thread.sleep(forTimeInterval: 0.25) + lock.lock(); let rms = micRMS; lock.unlock() + return rms < 0.015 + } + + func persistResume(segs: [Segment], from: Int, reason: String) { + let remaining = segs[from...].map { ["file": $0.file, "salience": $0.salience, "text": $0.text] as [String: Any] } + let state: [String: Any] = ["resume_from": from, "reason": reason, + "remaining": remaining, "ts": Date().timeIntervalSince1970] + if let d = try? JSONSerialization.data(withJSONObject: state, options: [.prettyPrinted]) { + try? d.write(to: URL(fileURLWithPath: resumePath)) + } + disclose("CONVERSE: meaning-plan persisted (\(remaining.count) segments remain) — Neuron can resume the thread.") + } + func clearResume() { try? FileManager.default.removeItem(atPath: resumePath) } + func finish() { player.stop(); if micLive { engine.inputNode.removeTap(onBus: 0) }; engine.stop() } +} + +// ---------------------------------------------------------------------------- +// CLI +// ---------------------------------------------------------------------------- +func loadManifest(_ path: String) -> (segs: [Segment], utterance: String) { + guard let d = FileManager.default.contents(atPath: path), + let o = try? JSONSerialization.jsonObject(with: d) as? [String: Any], + let arr = o["segments"] as? [[String: Any]] else { die("bad manifest: \(path)") } + let segs = arr.map { Segment(file: $0["file"] as? String ?? "", + salience: ($0["salience"] as? NSNumber)?.doubleValue ?? 0.5, + text: $0["text"] as? String ?? "") } + return (segs, o["utterance"] as? String ?? "") +} + +let args = CommandLine.arguments +guard args.count >= 2 else { + print(""" + periph — Neuron peripheral I/O (own-core, local, consent-gated) + grant grant a sensitive sense (Neuron-level consent) + revoke revoke it + status show consent state + speak SPEAK ALOUD (efferent) via the speaker + tone [hz] [sec] own-core synth a test WAV (no deps) + listen MIC capture (afferent) 16k mono + see CAMERA one frame (afferent) + feat-audio extract compact voice/sound signature (for ingest) + feat-image extract compact scene-geometry (for ingest) + ingest-audio capture -> descriptor -> engram node (geometry) + ingest-image capture -> descriptor -> engram node (geometry) + voiceprint extract voice-signature (F0 + formants F1-F5) + imitate speak back in that voice (LPC analysis-resynthesis) + hear-imitate MIC -> extract signature -> imitate -> SPEAK ALOUD + wav-info print WAV geometry + converse [--authority F] [--barge-at S[:backchannel|:bargein]] [--resume] + full-duplex interruptible utterance + """) + exit(0) +} + +switch args[1] { +case "grant": + guard args.count >= 3 else { die("grant needs a device") } + var g = Consent.load(); g[args[2]] = true; Consent.save(g) + disclose("granted '\(args[2])' — the user consents; raw stream stays local, never egresses.") + emit(["ok": true, "op": "grant", "device": args[2], "consent": g]) +case "revoke": + guard args.count >= 3 else { die("revoke needs a device") } + var g = Consent.load(); g[args[2]] = false; Consent.save(g) + emit(["ok": true, "op": "revoke", "device": args[2], "consent": g]) +case "status": + emit(["ok": true, "op": "status", "consent": Consent.load()]) +case "speak": + guard args.count >= 3 else { die("speak needs a wav") } + speak(args[2]) +case "tone": + guard args.count >= 3 else { die("tone needs an out path") } + let hz = args.count >= 4 ? Double(args[3]) ?? 220 : 220 + let sec = args.count >= 5 ? Double(args[4]) ?? 1.0 : 1.0 + let sr = 16000 + var s = [Int16](); s.reserveCapacity(Int(Double(sr)*sec)) + for i in 0..= 4 else { die("listen needs ") } + listen(seconds: Double(args[2]) ?? 3.0, out: args[3]) +case "see": + guard args.count >= 3 else { die("see needs an out path") } + see(out: args[2]) +case "feat-audio": + guard args.count >= 3 else { die("feat-audio needs a wav") } + featAudio(args[2]) +case "feat-image": + guard args.count >= 3 else { die("feat-image needs an image") } + featImage(args[2]) +case "ingest-audio": + guard args.count >= 4 else { die("ingest-audio needs ") } + ingest(args[2], kind: "audio", engramURL: args[3]) +case "ingest-image": + guard args.count >= 4 else { die("ingest-image needs ") } + ingest(args[2], kind: "image", engramURL: args[3]) +case "voiceprint": + guard args.count >= 3 else { die("voiceprint needs a wav") } + let vp = voiceprint(args[2]) + disclose("VOICEPRINT: \(vp.content)") + emit(["ok": true, "op": "voiceprint", "file": args[2], "f0_hz": vp.f0, + "f0_range": [vp.f0lo, vp.f0hi], "formants_hz": vp.formants.map { $0.0 }, + "bandwidths_hz": vp.formants.map { $0.1 }, "content": vp.content, + "ingest": ["node_type": "Observation", "tier": "Episodic", "content": vp.content]]) +case "imitate": + guard args.count >= 4 else { die("imitate needs ") } + imitate(inPath: args[2], outPath: args[3]) +case "hear-imitate": + guard args.count >= 4 else { die("hear-imitate needs ") } + let secs = Double(args[2]) ?? 4.0 + let outp = args[3] + let capp = outp.replacingOccurrences(of: ".wav", with: "") + ".heard.wav" + disclose("HEAR-IMITATE: open the ear, listen \(secs)s, grab the voice, speak it back.") + listen(seconds: secs, out: capp) // afferent: hear the voice + imitate(inPath: capp, outPath: outp) // extract signature + resynthesize + speak(outp) // efferent: speak back ALOUD in that voice +case "wav-info": + guard args.count >= 3, let i = wavInfo(args[2]) else { die("wav-info needs a readable wav") } + disclose("WAV \(args[2]): \(i.sampleRate)Hz \(i.channels)ch \(i.bits)bit \(i.frames) frames") + emit(["ok": true, "op": "wav-info", "sample_rate": i.sampleRate, "channels": i.channels, + "bits": i.bits, "frames": i.frames, + "seconds": Double(i.frames)/Double(max(i.sampleRate,1))]) +case "converse": + guard args.count >= 3 else { die("converse needs a manifest") } + let (segs, utter) = loadManifest(args[2]) + var authority = 0.5 + var bargeAt: Double? = nil + var bargeKind = "bargein" + var resume = false + var liveMic = false + var i = 3 + while i < args.count { + switch args[i] { + case "--authority": if i+1 < args.count { authority = Double(args[i+1]) ?? 0.5; i += 1 } + case "--barge-at": + if i+1 < args.count { + let parts = args[i+1].split(separator: ":") + bargeAt = Double(parts[0]) ?? nil + if parts.count > 1 { bargeKind = String(parts[1]) } + i += 1 + } + case "--resume": resume = true + case "--live-mic": liveMic = true + default: break + } + i += 1 + } + let resumePath = (ProcessInfo.processInfo.environment["PERIPH_HOME"] + ?? FileManager.default.currentDirectoryPath + "/peripheral") + "/.resume.json" + var startIndex = 0 + var runSegs = segs + if resume, let d = FileManager.default.contents(atPath: resumePath), + let o = try? JSONSerialization.jsonObject(with: d) as? [String: Any], + let rem = o["remaining"] as? [[String: Any]] { + runSegs = rem.map { Segment(file: $0["file"] as? String ?? "", + salience: ($0["salience"] as? NSNumber)?.doubleValue ?? 0.5, + text: $0["text"] as? String ?? "") } + startIndex = 0 + disclose("CONVERSE: resuming — \"as I was saying...\" (\(runSegs.count) segments left).") + emit(["op": "converse", "event": "resume", "remaining": runSegs.count]) + } + if runSegs.isEmpty { die("no segments to speak") } + disclose("CONVERSE: utterance = \"\(utter)\" (\(runSegs.count) segments).") + let convo = Conversation(resumePath: resumePath) + convo.run(runSegs, interrupterAuthority: authority, + injectBargeAt: bargeAt, injectKind: bargeKind, startIndex: startIndex, liveMic: liveMic) +default: + die("unknown command: \(args[1])") +}