Will Anderson
cbeb9c02eb
- Add scripts/: install_soul.py, install_all.py, launch_dharma.sh, stop_dharma.sh, wire_peers.py - Move all seeds into seeds/ directory (consolidated from root-level scattered files) - Update registry.json with engram_root_id, engram_api_key, engram_url for all 19 installed souls - Add src/soul.el, src/research.el; remove src/daemon.el - .gitignore: exclude imprints/, log/, sandboxes/ (runtime data) - Remove superseded scripts: deploy.py, install_imprints.py, reinstall_imprints.py, fix_collision.py - Remove old root-level seed files and shell scripts superseded by scripts/ versions Key: native engram binary uses _auth body field, per-soul keys ntn-<slug>-2026, DharmaPeer graph edges for peer wiring, sanitize_content() for JSON safety. 190/190 peer pairs wired successfully.
426 lines
14 KiB
Python
Executable File
426 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
Install a single soul's seed into their own dedicated Engram instance.
|
||
|
||
Uses the native Engram binary HTTP API:
|
||
POST /api/nodes {"content": "...", "node_type": "...", "salience": 0.8, "_auth": "<key>"}
|
||
POST /api/edges {"from_id": "...", "to_id": "...", "relation": "...", "weight": 0.8, "_auth": "<key>"}
|
||
GET /stats health check ({"node_count": N, "edge_count": M, "layer_count": K})
|
||
|
||
Auth: _auth field in the POST body.
|
||
Per-soul keys: ntn-<slug>-2026 (matching the running DHARMA infrastructure).
|
||
|
||
Usage:
|
||
python3 install_soul.py <slug>
|
||
python3 install_soul.py richard-feynman
|
||
python3 install_soul.py richard-feynman --force # reinstall even if data exists
|
||
|
||
Steps:
|
||
1. Checks if soul's Engram is already running (port in use) — installs into it.
|
||
If not running, starts the Engram binary, installs, then stops it.
|
||
2. Creates forge/imprints/<slug>/ data directory
|
||
3. Polls GET /stats until the instance is ready
|
||
4. Reads the soul's seed JSON
|
||
5. POSTs all nodes and edges using the soul's API key
|
||
6. Captures and returns the root node ID
|
||
7. If we started it, stops the instance (data persists)
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import requests
|
||
|
||
# ── Config ──────────────────────────────────────────────────────────────────
|
||
|
||
FORGE_DIR = Path("/Users/will/Development/neuron-technologies/forge")
|
||
ENGRAM_BIN = Path(
|
||
"/Users/will/Development/neuron-technologies/foundation/engram/dist/engram"
|
||
)
|
||
REGISTRY_FILE = FORGE_DIR / "registry.json"
|
||
|
||
STARTUP_TIMEOUT = 30
|
||
STARTUP_POLL_INTERVAL = 0.3
|
||
|
||
|
||
def load_registry() -> dict:
|
||
return json.loads(REGISTRY_FILE.read_text())
|
||
|
||
|
||
def save_registry(registry: dict) -> None:
|
||
REGISTRY_FILE.write_text(json.dumps(registry, indent=2))
|
||
|
||
|
||
def soul_api_key(soul: dict) -> str:
|
||
"""
|
||
Return the API key for a soul's Engram instance.
|
||
Per-soul keys: ntn-<slug>-2026 (matching the running DHARMA infrastructure).
|
||
If registry has engram_api_key set, that takes precedence.
|
||
"""
|
||
if soul.get("engram_api_key"):
|
||
return soul["engram_api_key"]
|
||
return f"ntn-{soul['slug']}-2026"
|
||
|
||
|
||
def find_soul(registry: dict, slug: str) -> dict | None:
|
||
for imp in registry["imprints"]:
|
||
if imp["slug"] == slug:
|
||
return imp
|
||
return None
|
||
|
||
|
||
def find_seed_file(registry_entry: dict) -> Path:
|
||
"""Resolve the seed file path from the registry entry."""
|
||
slug = registry_entry["slug"]
|
||
|
||
# Try registry-specified path first
|
||
seed_rel = registry_entry.get("seed_file", "")
|
||
if seed_rel:
|
||
candidate = FORGE_DIR / seed_rel
|
||
if candidate.exists():
|
||
return candidate
|
||
|
||
# Fallback: search known locations
|
||
candidates = [
|
||
FORGE_DIR / "seeds" / f"{slug}-seed.json",
|
||
FORGE_DIR / f"{slug}-seed.json",
|
||
# Legacy names used before seed file reorganization
|
||
FORGE_DIR / "seeds" / f"{slug.split('-')[0]}-seed.json", # e.g. alan from alan-turing
|
||
]
|
||
for c in candidates:
|
||
if c.exists():
|
||
return c
|
||
|
||
raise FileNotFoundError(
|
||
f"Cannot find seed file for {slug!r}. "
|
||
f"Tried: {[str(c) for c in candidates]}"
|
||
)
|
||
|
||
|
||
def is_port_in_use(port: int) -> bool:
|
||
"""Return True if a process is already listening on the port."""
|
||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||
return s.connect_ex(("127.0.0.1", port)) == 0
|
||
|
||
|
||
def start_engram(slug: str, port: int, db_path: Path, api_key: str) -> subprocess.Popen:
|
||
"""Start an Engram instance for the soul and return the process handle."""
|
||
db_path.mkdir(parents=True, exist_ok=True)
|
||
|
||
log_dir = FORGE_DIR / "log"
|
||
log_dir.mkdir(exist_ok=True)
|
||
log_file = log_dir / f"dharma-{slug}.log"
|
||
|
||
env = os.environ.copy()
|
||
# Native binary uses ENGRAM_DATA_DIR and ENGRAM_DB_PATH (both accepted)
|
||
env["ENGRAM_DATA_DIR"] = str(db_path)
|
||
env["ENGRAM_DB_PATH"] = str(db_path)
|
||
env["ENGRAM_BIND"] = f"0.0.0.0:{port}"
|
||
env["ENGRAM_API_KEY"] = api_key
|
||
|
||
with open(log_file, "a") as lf:
|
||
proc = subprocess.Popen(
|
||
[str(ENGRAM_BIN)],
|
||
env=env,
|
||
stdout=lf,
|
||
stderr=lf,
|
||
)
|
||
|
||
return proc
|
||
|
||
|
||
def wait_for_ready(port: int, timeout: float = STARTUP_TIMEOUT) -> bool:
|
||
"""Poll /stats until the instance responds or timeout expires."""
|
||
url = f"http://localhost:{port}/stats"
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
try:
|
||
resp = requests.get(url, timeout=2)
|
||
if resp.status_code == 200:
|
||
return True
|
||
except requests.exceptions.ConnectionError:
|
||
pass
|
||
time.sleep(STARTUP_POLL_INTERVAL)
|
||
return False
|
||
|
||
|
||
def sanitize_content(s: str) -> str:
|
||
"""
|
||
Sanitize text content for the native Engram binary's JSON parser.
|
||
|
||
The native binary's JSON response serializer does not escape double-quotes
|
||
in node content, which causes the response JSON to be malformed. Replace
|
||
double-quotes and other problematic characters with safe ASCII equivalents.
|
||
"""
|
||
replacements = [
|
||
('"', "'"), # double-quote → single (CRITICAL: prevents JSON response corruption)
|
||
("—", "--"), # em dash
|
||
("–", "-"), # en dash
|
||
("‘", "'"), ("’", "'"), # curly single quotes
|
||
("“", "'"), ("”", "'"), # curly double quotes
|
||
("…", "..."), # ellipsis
|
||
("é", "e"), ("è", "e"), ("ê", "e"), ("ë", "e"),
|
||
("à", "a"), ("â", "a"), ("á", "a"),
|
||
("ü", "u"), ("û", "u"), ("ú", "u"),
|
||
("ö", "o"), ("ô", "o"), ("ó", "o"),
|
||
("ä", "a"), ("ß", "ss"),
|
||
("î", "i"), ("í", "i"),
|
||
("É", "E"), ("Ê", "E"), ("È", "E"),
|
||
("Â", "A"), ("Î", "I"), ("Ô", "O"), ("Û", "U"),
|
||
("ñ", "n"), ("Ñ", "N"),
|
||
("ç", "c"), ("Ç", "C"),
|
||
]
|
||
for char, replacement in replacements:
|
||
s = s.replace(char, replacement)
|
||
# Replace any remaining non-ASCII with '?'
|
||
return s.encode("ascii", errors="replace").decode("ascii")
|
||
|
||
|
||
def post_node(port: int, content: str, node_type: str, salience: float, api_key: str) -> str:
|
||
"""POST a node via /api/nodes. Returns the node UUID."""
|
||
url = f"http://localhost:{port}/api/nodes"
|
||
safe_content = sanitize_content(content)
|
||
body = {
|
||
"content": safe_content,
|
||
"node_type": node_type,
|
||
"salience": salience,
|
||
"_auth": api_key,
|
||
}
|
||
resp = requests.post(url, json=body, timeout=30)
|
||
resp.raise_for_status()
|
||
result = resp.json()
|
||
if "id" not in result:
|
||
raise ValueError(f"Unexpected response from /api/nodes: {result}")
|
||
return result["id"]
|
||
|
||
|
||
def post_edge(port: int, from_id: str, to_id: str, relation: str, weight: float, api_key: str) -> None:
|
||
"""POST an edge via /api/edges."""
|
||
url = f"http://localhost:{port}/api/edges"
|
||
body = {
|
||
"from_id": from_id,
|
||
"to_id": to_id,
|
||
"relation": relation,
|
||
"weight": weight,
|
||
"_auth": api_key,
|
||
}
|
||
resp = requests.post(url, json=body, timeout=30)
|
||
resp.raise_for_status()
|
||
|
||
|
||
def install_seed(port: int, seed: dict, api_key: str) -> str:
|
||
"""
|
||
Install all nodes and edges from a seed dict.
|
||
Returns the root node UUID.
|
||
"""
|
||
subject = seed["subject"]
|
||
print(f" Installing nodes for {subject}...")
|
||
|
||
# Root identity node
|
||
root_id = post_node(
|
||
port,
|
||
content=f"IMPRINT: {subject} | dharma/1.0",
|
||
node_type="Identity",
|
||
salience=1.0,
|
||
api_key=api_key,
|
||
)
|
||
print(f" Root node: {root_id}")
|
||
|
||
# Values
|
||
for v in seed.get("values", []):
|
||
content = f"VALUE: {v['value']}"
|
||
if v.get("grounding"):
|
||
content += f" | grounding: {v['grounding']}"
|
||
nid = post_node(port, content, "Identity", float(v.get("weight", 0.8)), api_key)
|
||
post_edge(port, root_id, nid, "has_value", float(v.get("weight", 0.8)), api_key)
|
||
|
||
# Biography
|
||
for b in seed.get("biography", []):
|
||
content = f"BIOGRAPHY: {b['event']}"
|
||
nid = post_node(port, content, "Identity", float(b.get("weight", 0.7)), api_key)
|
||
post_edge(port, root_id, nid, "formed_by", float(b.get("weight", 0.7)), api_key)
|
||
|
||
# Relationships
|
||
for r in seed.get("relationships", []):
|
||
content = f"RELATIONSHIP: {r['name']}"
|
||
if r.get("role"):
|
||
content += f" ({r['role']})"
|
||
nid = post_node(port, content, "Identity", float(r.get("weight", 0.6)), api_key)
|
||
post_edge(port, root_id, nid, "relates_to", float(r.get("weight", 0.6)), api_key)
|
||
|
||
# Reasoning patterns
|
||
for pattern in seed.get("reasoning_patterns", []):
|
||
content = f"REASONING: {pattern}"
|
||
nid = post_node(port, content, "Identity", 0.7, api_key)
|
||
post_edge(port, root_id, nid, "reasons_with", 0.7, api_key)
|
||
|
||
# Voice profile — one consolidated node
|
||
voice = seed.get("voice_profile", {})
|
||
if voice:
|
||
voice_parts = []
|
||
for k, v in voice.items():
|
||
if v:
|
||
voice_parts.append(f"{k}: {v}")
|
||
if voice_parts:
|
||
voice_content = "VOICE: " + " | ".join(voice_parts)
|
||
nid = post_node(port, voice_content[:2000], "Identity", 0.9, api_key)
|
||
post_edge(port, root_id, nid, "speaks_as", 0.9, api_key)
|
||
|
||
# Stats after install
|
||
stats_resp = requests.get(f"http://localhost:{port}/stats", timeout=10)
|
||
stats = stats_resp.json()
|
||
print(
|
||
f" Engram stats after install: "
|
||
f"{stats.get('node_count', '?')} nodes, "
|
||
f"{stats.get('edge_count', '?')} edges"
|
||
)
|
||
|
||
# Persist to disk
|
||
save_url = f"http://localhost:{port}/api/save"
|
||
try:
|
||
save_resp = requests.post(save_url, json={"_auth": api_key}, timeout=10)
|
||
save_result = save_resp.json()
|
||
print(f" Saved to: {save_result.get('path', '?')}")
|
||
except Exception as exc:
|
||
print(f" WARN: save failed: {exc}")
|
||
|
||
return root_id
|
||
|
||
|
||
def install_soul(slug: str, force: bool = False) -> dict:
|
||
"""
|
||
Full install pipeline for one soul.
|
||
|
||
If the soul's Engram is already running (port in use), installs directly
|
||
into it. Otherwise starts a temporary instance, installs, and stops it.
|
||
|
||
Returns dict with: slug, root_id, engram_db_path, engram_port, engram_url, engram_api_key
|
||
"""
|
||
if not ENGRAM_BIN.exists():
|
||
raise FileNotFoundError(
|
||
f"Engram binary not found at {ENGRAM_BIN}."
|
||
)
|
||
|
||
registry = load_registry()
|
||
soul = find_soul(registry, slug)
|
||
if soul is None:
|
||
raise ValueError(f"Soul {slug!r} not found in registry.json")
|
||
|
||
port = soul["engram_port"]
|
||
db_path = FORGE_DIR / soul.get("engram_db_path", f"imprints/{slug}")
|
||
api_key = soul_api_key(soul)
|
||
|
||
# Skip if already installed (data dir non-empty AND registry has root_id),
|
||
# unless --force is given.
|
||
already_has_data = db_path.exists() and any(db_path.iterdir())
|
||
already_has_root = bool(soul.get("engram_root_id"))
|
||
if not force and already_has_data and already_has_root:
|
||
print(f"[{slug}] Already installed (root={soul['engram_root_id']}) — skipping.")
|
||
print(f"[{slug}] Use --force to reinstall.")
|
||
return {
|
||
"slug": slug,
|
||
"root_id": soul.get("engram_root_id"),
|
||
"skipped": True,
|
||
}
|
||
|
||
seed_file = find_seed_file(soul)
|
||
seed = json.loads(seed_file.read_text())
|
||
print(
|
||
f"[{slug}] Seed: {seed_file.name} "
|
||
f"({len(seed.get('values', []))} values, "
|
||
f"{len(seed.get('biography', []))} bio, "
|
||
f"{len(seed.get('reasoning_patterns', []))} patterns, "
|
||
f"{len(seed.get('relationships', []))} relationships)"
|
||
)
|
||
print(f"[{slug}] API key: {api_key}")
|
||
|
||
already_running = is_port_in_use(port)
|
||
proc = None
|
||
|
||
if already_running:
|
||
print(f"[{slug}] Engram already running on port {port} — installing into live instance.")
|
||
if not wait_for_ready(port, timeout=5):
|
||
raise RuntimeError(
|
||
f"Port {port} is in use but Engram is not responding at /stats"
|
||
)
|
||
else:
|
||
print(f"[{slug}] Starting Engram on port {port}...")
|
||
proc = start_engram(slug, port, db_path, api_key)
|
||
|
||
try:
|
||
if proc is not None:
|
||
if not wait_for_ready(port):
|
||
proc.terminate()
|
||
raise RuntimeError(
|
||
f"Engram for {slug} did not become ready within {STARTUP_TIMEOUT}s"
|
||
)
|
||
print(f"[{slug}] Engram ready on port {port}")
|
||
|
||
root_id = install_seed(port, seed, api_key)
|
||
print(f"[{slug}] Install complete. Root node: {root_id}")
|
||
|
||
finally:
|
||
if proc is not None:
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
print(f"[{slug}] Instance stopped.")
|
||
|
||
# Update registry entry
|
||
from datetime import date
|
||
for imp in registry["imprints"]:
|
||
if imp["slug"] == slug:
|
||
imp["engram_db_path"] = str(db_path.relative_to(FORGE_DIR))
|
||
imp["engram_port"] = port
|
||
imp["engram_url"] = f"http://localhost:{port}"
|
||
imp["engram_api_key"] = api_key
|
||
imp["engram_root_id"] = root_id
|
||
imp["installed"] = True
|
||
imp["installed_at"] = str(date.today())
|
||
break
|
||
|
||
save_registry(registry)
|
||
print(f"[{slug}] Registry updated.")
|
||
|
||
return {
|
||
"slug": slug,
|
||
"root_id": root_id,
|
||
"engram_db_path": str(db_path.relative_to(FORGE_DIR)),
|
||
"engram_port": port,
|
||
"engram_url": f"http://localhost:{port}",
|
||
"engram_api_key": api_key,
|
||
"skipped": False,
|
||
}
|
||
|
||
|
||
def main():
|
||
import argparse
|
||
|
||
parser = argparse.ArgumentParser(
|
||
description="Install a soul's seed into its Engram instance"
|
||
)
|
||
parser.add_argument("slug", help="Soul slug (e.g. richard-feynman)")
|
||
parser.add_argument(
|
||
"--force",
|
||
action="store_true",
|
||
help="Reinstall even if imprint directory already exists",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
result = install_soul(args.slug, force=args.force)
|
||
if result.get("skipped"):
|
||
print(f"\nSkipped (already installed). Root: {result.get('root_id', 'unknown')}")
|
||
else:
|
||
print(f"\nDone. Root node ID: {result['root_id']}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|