will.anderson
dcc0bf550a
- P0: unified soul binary with engram_node_full fix, read-back-verify, search fix - P0: move API keys from plaintext plists to macOS Keychain - P0: fix MCP backend URL (port 8742 → 7770) - P1.6: memory-export/import scripts (AES-256-CBC, versioned .neuronmem format) - P1.7: nightly cultivation digest with sharpness metric (launchd at 23:55) - P2.10: Ollama provider in agentic loop (SOUL_LLM_PROVIDER=ollama) - P3.12: refugee importer for ChatGPT/Screenpipe/generic formats - P3.13: GLM-OCR spike — SHIP IT (mlx-vlm, 1.59GB, photo-to-memory.sh)
222 lines
8.2 KiB
Bash
Executable File
222 lines
8.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# cultivation-digest.sh — Neuron daily cultivation digest
|
|
# Reads ~/.neuron/engram/snapshot.json and produces a sharpness report.
|
|
# Writes to ~/.neuron/digests/YYYY-MM-DD.txt and appends to sharpness.json.
|
|
|
|
set -euo pipefail
|
|
|
|
SNAPSHOT="$HOME/.neuron/engram/snapshot.json"
|
|
DIGESTS_DIR="$HOME/.neuron/digests"
|
|
DATE=$(date +%Y-%m-%d)
|
|
DIGEST_FILE="$DIGESTS_DIR/$DATE.txt"
|
|
SHARPNESS_FILE="$DIGESTS_DIR/sharpness.json"
|
|
|
|
mkdir -p "$DIGESTS_DIR"
|
|
|
|
if [[ ! -f "$SNAPSHOT" ]]; then
|
|
echo "ERROR: snapshot not found at $SNAPSHOT" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Cutoff: now minus 24 hours in milliseconds
|
|
NOW_MS=$(( $(date +%s) * 1000 ))
|
|
CUTOFF_MS=$(( NOW_MS - 86400000 ))
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Compute all metrics via a single jq pass (avoids re-reading 174 MB 10x)
|
|
# Fields in item lines are tab-separated: type TAB importance TAB content
|
|
# ---------------------------------------------------------------------------
|
|
METRICS=$(jq -r --argjson cutoff "$CUTOFF_MS" '
|
|
.nodes as $all |
|
|
|
|
# Real memory nodes — exclude InternalStateEvent and corrupted entries
|
|
($all | map(select(
|
|
.node_type != "InternalStateEvent" and
|
|
(.node_type | test("^[A-Za-z]+$"))
|
|
))) as $real |
|
|
|
|
# Created today
|
|
($real | map(select(.created_at > $cutoff))) as $new |
|
|
|
|
# Activated today but not created today (reinforced)
|
|
($real | map(select(
|
|
(.last_activated // 0) > $cutoff and
|
|
.created_at <= $cutoff
|
|
))) as $reinforced |
|
|
|
|
# Stats for sharpness (across all real nodes)
|
|
($real | length) as $real_count |
|
|
($real | if length > 0 then (map(.importance) | add / length) else 0 end) as $avg_imp |
|
|
($real | if length > 0 then (map(.confidence // 1) | add / length) else 0 end) as $avg_conf |
|
|
|
|
# activation_ratio: reinforced nodes today / total real nodes, capped 0-1
|
|
(($reinforced | length) as $ra |
|
|
if $real_count > 0 then ($ra / $real_count | if . > 1 then 1 else . end) else 0 end
|
|
) as $act_ratio |
|
|
|
|
# Sharpness score 0-100
|
|
((($avg_imp * 0.4) + ($avg_conf * 0.3) + ($act_ratio * 0.3)) * 100 | round) as $sharpness |
|
|
|
|
# Top new memories (by importance desc, cap 10)
|
|
($new | sort_by(-.importance) | .[0:10]) as $top_new |
|
|
|
|
# Top reinforced (by last_activated desc, cap 10)
|
|
($reinforced | sort_by(-.last_activated) | .[0:10]) as $top_reinforced |
|
|
|
|
# High-importance nodes (importance > 0.8), across all real nodes
|
|
($real | map(select(.importance > 0.8)) | length) as $high_imp_count |
|
|
|
|
# Scalar metrics
|
|
"TOTAL_REAL=\($real_count)",
|
|
"NEW_COUNT=\($new | length)",
|
|
"REINFORCED_COUNT=\($reinforced | length)",
|
|
"TOTAL_NODES=\($all | length)",
|
|
"AVG_IMP=\($avg_imp)",
|
|
"AVG_CONF=\($avg_conf)",
|
|
"ACT_RATIO=\($act_ratio)",
|
|
"SHARPNESS=\($sharpness)",
|
|
"HIGH_IMP=\($high_imp_count)",
|
|
|
|
# Item sections — fields separated by tab character (\t)
|
|
"---NEW---",
|
|
($top_new[] | [.node_type, (.importance | tostring), (.content[0:120] | gsub("\n";" "))] | join("\t")),
|
|
"---REINFORCED---",
|
|
($top_reinforced[] | [(.label[0:80] | gsub("\n";" ")), ("activated \(.activation_count)x total")] | join("\t"))
|
|
' "$SNAPSHOT" 2>/dev/null)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Parse scalar metrics
|
|
# ---------------------------------------------------------------------------
|
|
parse() { printf '%s' "$METRICS" | grep "^$1=" | head -1 | cut -d= -f2-; }
|
|
|
|
TOTAL_REAL=$(parse TOTAL_REAL)
|
|
NEW_COUNT=$(parse NEW_COUNT)
|
|
REINFORCED_COUNT=$(parse REINFORCED_COUNT)
|
|
TOTAL_NODES=$(parse TOTAL_NODES)
|
|
AVG_IMP=$(parse AVG_IMP)
|
|
AVG_CONF=$(parse AVG_CONF)
|
|
ACT_RATIO=$(parse ACT_RATIO)
|
|
SHARPNESS=$(parse SHARPNESS)
|
|
HIGH_IMP=$(parse HIGH_IMP)
|
|
|
|
# Format floats to 2dp (use awk, avoiding bc locale issues)
|
|
fmt2() { awk "BEGIN{printf \"%.2f\", $1}"; }
|
|
fmt4() { awk "BEGIN{printf \"%.4f\", $1}"; }
|
|
AVG_IMP_FMT=$(fmt2 "$AVG_IMP")
|
|
AVG_CONF_FMT=$(fmt2 "$AVG_CONF")
|
|
ACT_RATIO_FMT=$(fmt4 "$ACT_RATIO")
|
|
IMP_CONTRIB=$(fmt4 "$(awk "BEGIN{printf \"%.6f\", $AVG_IMP * 0.4}")")
|
|
CONF_CONTRIB=$(fmt4 "$(awk "BEGIN{printf \"%.6f\", $AVG_CONF * 0.3}")")
|
|
ACT_CONTRIB=$(fmt4 "$(awk "BEGIN{printf \"%.6f\", $ACT_RATIO * 0.3}")")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sharpness delta (compare to yesterday)
|
|
# ---------------------------------------------------------------------------
|
|
DELTA_STR=""
|
|
if [[ -f "$SHARPNESS_FILE" ]]; then
|
|
YESTERDAY=$(date -v-1d +%Y-%m-%d 2>/dev/null || date -d "yesterday" +%Y-%m-%d 2>/dev/null || echo "")
|
|
if [[ -n "$YESTERDAY" ]]; then
|
|
PREV_SHARPNESS=$(jq -r --arg d "$YESTERDAY" '.[] | select(.date == $d) | .sharpness' "$SHARPNESS_FILE" 2>/dev/null | tail -1)
|
|
if [[ -n "$PREV_SHARPNESS" && "$PREV_SHARPNESS" != "null" ]]; then
|
|
DELTA=$(( SHARPNESS - PREV_SHARPNESS ))
|
|
if (( DELTA > 0 )); then
|
|
DELTA_STR=" (up ${DELTA}% from yesterday)"
|
|
elif (( DELTA < 0 )); then
|
|
DELTA_STR=" (down ${DELTA#-}% from yesterday)"
|
|
else
|
|
DELTA_STR=" (no change from yesterday)"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Build new-memories section (tab-delimited: type TAB importance TAB content)
|
|
# ---------------------------------------------------------------------------
|
|
new_section() {
|
|
local lines
|
|
lines=$(printf '%s\n' "$METRICS" | awk '/^---NEW---/{found=1; next} /^---REINFORCED---/{exit} found{print}')
|
|
if [[ -z "$lines" ]]; then
|
|
echo " (none)"
|
|
return
|
|
fi
|
|
while IFS=$'\t' read -r ntype importance content; do
|
|
[[ -z "$ntype" ]] && continue
|
|
imp_fmt=$(awk "BEGIN{printf \"%.1f\", $importance}")
|
|
printf " [%-18s] (importance: %s) %s\n" "$ntype" "$imp_fmt" "$content"
|
|
done <<< "$lines"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Build reinforced section (tab-delimited: label TAB activation-info)
|
|
# ---------------------------------------------------------------------------
|
|
reinforced_section() {
|
|
local lines
|
|
lines=$(printf '%s\n' "$METRICS" | awk '/^---REINFORCED---/{found=1; next} found{print}')
|
|
if [[ -z "$lines" ]]; then
|
|
echo " (none today)"
|
|
return
|
|
fi
|
|
while IFS=$'\t' read -r label acts; do
|
|
[[ -z "$label" ]] && continue
|
|
printf " \"%s\" — %s\n" "$label" "$acts"
|
|
done <<< "$lines"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Render full digest
|
|
# ---------------------------------------------------------------------------
|
|
DIGEST=$(cat <<EOF
|
|
=== Neuron Cultivation Digest — ${DATE} ===
|
|
|
|
SHARPNESS: ${SHARPNESS}%${DELTA_STR}
|
|
|
|
TODAY'S MEMORIES (${NEW_COUNT} new):
|
|
$(new_section)
|
|
|
|
REINFORCED (${REINFORCED_COUNT} nodes re-activated today):
|
|
$(reinforced_section)
|
|
|
|
MEMORY HEALTH:
|
|
Total nodes (all): ${TOTAL_NODES}
|
|
Real memory nodes: ${TOTAL_REAL}
|
|
Avg importance: ${AVG_IMP_FMT}
|
|
Avg confidence: ${AVG_CONF_FMT}
|
|
High-importance nodes (>0.8): ${HIGH_IMP}
|
|
Nodes created today: ${NEW_COUNT}
|
|
Nodes re-activated today: ${REINFORCED_COUNT}
|
|
|
|
SHARPNESS FORMULA:
|
|
Sharpness = (avg_importance x 0.4) + (avg_confidence x 0.3) + (activation_ratio x 0.3)
|
|
avg_importance = ${AVG_IMP_FMT} -> ${AVG_IMP_FMT} x 0.4 = ${IMP_CONTRIB}
|
|
avg_confidence = ${AVG_CONF_FMT} -> ${AVG_CONF_FMT} x 0.3 = ${CONF_CONTRIB}
|
|
activation_ratio = ${ACT_RATIO_FMT} -> ratio x 0.3 = ${ACT_CONTRIB}
|
|
Result: ${SHARPNESS}%
|
|
|
|
Generated: $(date)
|
|
EOF
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Write digest file + print to stdout
|
|
# ---------------------------------------------------------------------------
|
|
printf '%s\n' "$DIGEST" | tee "$DIGEST_FILE"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Append to sharpness.json
|
|
# ---------------------------------------------------------------------------
|
|
NEW_ENTRY="{\"date\":\"${DATE}\",\"sharpness\":${SHARPNESS},\"node_count\":${TOTAL_NODES},\"real_node_count\":${TOTAL_REAL},\"nodes_added\":${NEW_COUNT},\"nodes_reinforced\":${REINFORCED_COUNT}}"
|
|
|
|
if [[ -f "$SHARPNESS_FILE" ]]; then
|
|
UPDATED=$(jq --arg d "$DATE" --argjson entry "$NEW_ENTRY" '
|
|
map(select(.date != $d)) + [$entry]
|
|
' "$SHARPNESS_FILE" 2>/dev/null) || UPDATED="[$NEW_ENTRY]"
|
|
printf '%s\n' "$UPDATED" > "$SHARPNESS_FILE"
|
|
else
|
|
printf '[%s]\n' "$NEW_ENTRY" > "$SHARPNESS_FILE"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Digest written to: $DIGEST_FILE"
|
|
echo "Sharpness log: $SHARPNESS_FILE"
|