import json,sys,time,urllib.request,threading,queue
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
OUT=sys.argv[1]
URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text"
MAXB=2000  # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
tasks=[]
for n in d["nodes"]:
    c=n.get("content") or ""; t=n.get("node_type") or ""
    if len(c)<8: continue                     # eg_embed_eligible
    if t in ("InternalStateEvent","Tag"): continue
    b=c.encode('utf-8',errors='surrogateescape')[:MAXB]
    tasks.append((n.get("id") or "", b.decode('utf-8',errors='replace')))
del d
print("tasks",len(tasks),flush=True)
q=queue.Queue(); [q.put(t) for t in tasks]
lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0]
def work():
    while True:
        try: nid,txt=q.get_nowait()
        except queue.Empty: return
        v=None
        for attempt in range(3):
            try:
                body=json.dumps({"model":MODEL,"prompt":txt}).encode()
                r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"})
                with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"]
                break
            except Exception as e:
                if attempt==2:
                    with lock: fails[0]+=1
                time.sleep(0.5)
        with lock:
            if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n")
            done[0]+=1
            if done[0]%2000==0:
                el=time.time()-t0
                print("%d/%d  %.1f/s  eta %.1fmin  fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True)
                f.flush()
ths=[threading.Thread(target=work) for _ in range(8)]
[t.start() for t in ths]; [t.join() for t in ths]
f.close()
print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True)