Add Ollama provider, portable memory, cultivation digest, refugee importer, GLM-OCR spike
- P0: unified soul binary with engram_node_full fix, read-back-verify, search fix - P0: move API keys from plaintext plists to macOS Keychain - P0: fix MCP backend URL (port 8742 → 7770) - P1.6: memory-export/import scripts (AES-256-CBC, versioned .neuronmem format) - P1.7: nightly cultivation digest with sharpness metric (launchd at 23:55) - P2.10: Ollama provider in agentic loop (SOUL_LLM_PROVIDER=ollama) - P3.12: refugee importer for ChatGPT/Screenpipe/generic formats - P3.13: GLM-OCR spike — SHIP IT (mlx-vlm, 1.59GB, photo-to-memory.sh)
This commit is contained in:
Executable
+221
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env bash
|
||||
# cultivation-digest.sh — Neuron daily cultivation digest
|
||||
# Reads ~/.neuron/engram/snapshot.json and produces a sharpness report.
|
||||
# Writes to ~/.neuron/digests/YYYY-MM-DD.txt and appends to sharpness.json.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SNAPSHOT="$HOME/.neuron/engram/snapshot.json"
|
||||
DIGESTS_DIR="$HOME/.neuron/digests"
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
DIGEST_FILE="$DIGESTS_DIR/$DATE.txt"
|
||||
SHARPNESS_FILE="$DIGESTS_DIR/sharpness.json"
|
||||
|
||||
mkdir -p "$DIGESTS_DIR"
|
||||
|
||||
if [[ ! -f "$SNAPSHOT" ]]; then
|
||||
echo "ERROR: snapshot not found at $SNAPSHOT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cutoff: now minus 24 hours in milliseconds
|
||||
NOW_MS=$(( $(date +%s) * 1000 ))
|
||||
CUTOFF_MS=$(( NOW_MS - 86400000 ))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compute all metrics via a single jq pass (avoids re-reading 174 MB 10x)
|
||||
# Fields in item lines are tab-separated: type TAB importance TAB content
|
||||
# ---------------------------------------------------------------------------
|
||||
METRICS=$(jq -r --argjson cutoff "$CUTOFF_MS" '
|
||||
.nodes as $all |
|
||||
|
||||
# Real memory nodes — exclude InternalStateEvent and corrupted entries
|
||||
($all | map(select(
|
||||
.node_type != "InternalStateEvent" and
|
||||
(.node_type | test("^[A-Za-z]+$"))
|
||||
))) as $real |
|
||||
|
||||
# Created today
|
||||
($real | map(select(.created_at > $cutoff))) as $new |
|
||||
|
||||
# Activated today but not created today (reinforced)
|
||||
($real | map(select(
|
||||
(.last_activated // 0) > $cutoff and
|
||||
.created_at <= $cutoff
|
||||
))) as $reinforced |
|
||||
|
||||
# Stats for sharpness (across all real nodes)
|
||||
($real | length) as $real_count |
|
||||
($real | if length > 0 then (map(.importance) | add / length) else 0 end) as $avg_imp |
|
||||
($real | if length > 0 then (map(.confidence // 1) | add / length) else 0 end) as $avg_conf |
|
||||
|
||||
# activation_ratio: reinforced nodes today / total real nodes, capped 0-1
|
||||
(($reinforced | length) as $ra |
|
||||
if $real_count > 0 then ($ra / $real_count | if . > 1 then 1 else . end) else 0 end
|
||||
) as $act_ratio |
|
||||
|
||||
# Sharpness score 0-100
|
||||
((($avg_imp * 0.4) + ($avg_conf * 0.3) + ($act_ratio * 0.3)) * 100 | round) as $sharpness |
|
||||
|
||||
# Top new memories (by importance desc, cap 10)
|
||||
($new | sort_by(-.importance) | .[0:10]) as $top_new |
|
||||
|
||||
# Top reinforced (by last_activated desc, cap 10)
|
||||
($reinforced | sort_by(-.last_activated) | .[0:10]) as $top_reinforced |
|
||||
|
||||
# High-importance nodes (importance > 0.8), across all real nodes
|
||||
($real | map(select(.importance > 0.8)) | length) as $high_imp_count |
|
||||
|
||||
# Scalar metrics
|
||||
"TOTAL_REAL=\($real_count)",
|
||||
"NEW_COUNT=\($new | length)",
|
||||
"REINFORCED_COUNT=\($reinforced | length)",
|
||||
"TOTAL_NODES=\($all | length)",
|
||||
"AVG_IMP=\($avg_imp)",
|
||||
"AVG_CONF=\($avg_conf)",
|
||||
"ACT_RATIO=\($act_ratio)",
|
||||
"SHARPNESS=\($sharpness)",
|
||||
"HIGH_IMP=\($high_imp_count)",
|
||||
|
||||
# Item sections — fields separated by tab character (\t)
|
||||
"---NEW---",
|
||||
($top_new[] | [.node_type, (.importance | tostring), (.content[0:120] | gsub("\n";" "))] | join("\t")),
|
||||
"---REINFORCED---",
|
||||
($top_reinforced[] | [(.label[0:80] | gsub("\n";" ")), ("activated \(.activation_count)x total")] | join("\t"))
|
||||
' "$SNAPSHOT" 2>/dev/null)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse scalar metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
parse() { printf '%s' "$METRICS" | grep "^$1=" | head -1 | cut -d= -f2-; }
|
||||
|
||||
TOTAL_REAL=$(parse TOTAL_REAL)
|
||||
NEW_COUNT=$(parse NEW_COUNT)
|
||||
REINFORCED_COUNT=$(parse REINFORCED_COUNT)
|
||||
TOTAL_NODES=$(parse TOTAL_NODES)
|
||||
AVG_IMP=$(parse AVG_IMP)
|
||||
AVG_CONF=$(parse AVG_CONF)
|
||||
ACT_RATIO=$(parse ACT_RATIO)
|
||||
SHARPNESS=$(parse SHARPNESS)
|
||||
HIGH_IMP=$(parse HIGH_IMP)
|
||||
|
||||
# Format floats to 2dp (use awk, avoiding bc locale issues)
|
||||
fmt2() { awk "BEGIN{printf \"%.2f\", $1}"; }
|
||||
fmt4() { awk "BEGIN{printf \"%.4f\", $1}"; }
|
||||
AVG_IMP_FMT=$(fmt2 "$AVG_IMP")
|
||||
AVG_CONF_FMT=$(fmt2 "$AVG_CONF")
|
||||
ACT_RATIO_FMT=$(fmt4 "$ACT_RATIO")
|
||||
IMP_CONTRIB=$(fmt4 "$(awk "BEGIN{printf \"%.6f\", $AVG_IMP * 0.4}")")
|
||||
CONF_CONTRIB=$(fmt4 "$(awk "BEGIN{printf \"%.6f\", $AVG_CONF * 0.3}")")
|
||||
ACT_CONTRIB=$(fmt4 "$(awk "BEGIN{printf \"%.6f\", $ACT_RATIO * 0.3}")")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sharpness delta (compare to yesterday)
|
||||
# ---------------------------------------------------------------------------
|
||||
DELTA_STR=""
|
||||
if [[ -f "$SHARPNESS_FILE" ]]; then
|
||||
YESTERDAY=$(date -v-1d +%Y-%m-%d 2>/dev/null || date -d "yesterday" +%Y-%m-%d 2>/dev/null || echo "")
|
||||
if [[ -n "$YESTERDAY" ]]; then
|
||||
PREV_SHARPNESS=$(jq -r --arg d "$YESTERDAY" '.[] | select(.date == $d) | .sharpness' "$SHARPNESS_FILE" 2>/dev/null | tail -1)
|
||||
if [[ -n "$PREV_SHARPNESS" && "$PREV_SHARPNESS" != "null" ]]; then
|
||||
DELTA=$(( SHARPNESS - PREV_SHARPNESS ))
|
||||
if (( DELTA > 0 )); then
|
||||
DELTA_STR=" (up ${DELTA}% from yesterday)"
|
||||
elif (( DELTA < 0 )); then
|
||||
DELTA_STR=" (down ${DELTA#-}% from yesterday)"
|
||||
else
|
||||
DELTA_STR=" (no change from yesterday)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build new-memories section (tab-delimited: type TAB importance TAB content)
|
||||
# ---------------------------------------------------------------------------
|
||||
new_section() {
|
||||
local lines
|
||||
lines=$(printf '%s\n' "$METRICS" | awk '/^---NEW---/{found=1; next} /^---REINFORCED---/{exit} found{print}')
|
||||
if [[ -z "$lines" ]]; then
|
||||
echo " (none)"
|
||||
return
|
||||
fi
|
||||
while IFS=$'\t' read -r ntype importance content; do
|
||||
[[ -z "$ntype" ]] && continue
|
||||
imp_fmt=$(awk "BEGIN{printf \"%.1f\", $importance}")
|
||||
printf " [%-18s] (importance: %s) %s\n" "$ntype" "$imp_fmt" "$content"
|
||||
done <<< "$lines"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build reinforced section (tab-delimited: label TAB activation-info)
|
||||
# ---------------------------------------------------------------------------
|
||||
reinforced_section() {
|
||||
local lines
|
||||
lines=$(printf '%s\n' "$METRICS" | awk '/^---REINFORCED---/{found=1; next} found{print}')
|
||||
if [[ -z "$lines" ]]; then
|
||||
echo " (none today)"
|
||||
return
|
||||
fi
|
||||
while IFS=$'\t' read -r label acts; do
|
||||
[[ -z "$label" ]] && continue
|
||||
printf " \"%s\" — %s\n" "$label" "$acts"
|
||||
done <<< "$lines"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Render full digest
|
||||
# ---------------------------------------------------------------------------
|
||||
DIGEST=$(cat <<EOF
|
||||
=== Neuron Cultivation Digest — ${DATE} ===
|
||||
|
||||
SHARPNESS: ${SHARPNESS}%${DELTA_STR}
|
||||
|
||||
TODAY'S MEMORIES (${NEW_COUNT} new):
|
||||
$(new_section)
|
||||
|
||||
REINFORCED (${REINFORCED_COUNT} nodes re-activated today):
|
||||
$(reinforced_section)
|
||||
|
||||
MEMORY HEALTH:
|
||||
Total nodes (all): ${TOTAL_NODES}
|
||||
Real memory nodes: ${TOTAL_REAL}
|
||||
Avg importance: ${AVG_IMP_FMT}
|
||||
Avg confidence: ${AVG_CONF_FMT}
|
||||
High-importance nodes (>0.8): ${HIGH_IMP}
|
||||
Nodes created today: ${NEW_COUNT}
|
||||
Nodes re-activated today: ${REINFORCED_COUNT}
|
||||
|
||||
SHARPNESS FORMULA:
|
||||
Sharpness = (avg_importance x 0.4) + (avg_confidence x 0.3) + (activation_ratio x 0.3)
|
||||
avg_importance = ${AVG_IMP_FMT} -> ${AVG_IMP_FMT} x 0.4 = ${IMP_CONTRIB}
|
||||
avg_confidence = ${AVG_CONF_FMT} -> ${AVG_CONF_FMT} x 0.3 = ${CONF_CONTRIB}
|
||||
activation_ratio = ${ACT_RATIO_FMT} -> ratio x 0.3 = ${ACT_CONTRIB}
|
||||
Result: ${SHARPNESS}%
|
||||
|
||||
Generated: $(date)
|
||||
EOF
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Write digest file + print to stdout
|
||||
# ---------------------------------------------------------------------------
|
||||
printf '%s\n' "$DIGEST" | tee "$DIGEST_FILE"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Append to sharpness.json
|
||||
# ---------------------------------------------------------------------------
|
||||
NEW_ENTRY="{\"date\":\"${DATE}\",\"sharpness\":${SHARPNESS},\"node_count\":${TOTAL_NODES},\"real_node_count\":${TOTAL_REAL},\"nodes_added\":${NEW_COUNT},\"nodes_reinforced\":${REINFORCED_COUNT}}"
|
||||
|
||||
if [[ -f "$SHARPNESS_FILE" ]]; then
|
||||
UPDATED=$(jq --arg d "$DATE" --argjson entry "$NEW_ENTRY" '
|
||||
map(select(.date != $d)) + [$entry]
|
||||
' "$SHARPNESS_FILE" 2>/dev/null) || UPDATED="[$NEW_ENTRY]"
|
||||
printf '%s\n' "$UPDATED" > "$SHARPNESS_FILE"
|
||||
else
|
||||
printf '[%s]\n' "$NEW_ENTRY" > "$SHARPNESS_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Digest written to: $DIGEST_FILE"
|
||||
echo "Sharpness log: $SHARPNESS_FILE"
|
||||
Reference in New Issue
Block a user