infrastructure/servers/legion/rosetta_experiment.py

#!/usr/bin/env python3
"""
Rosetta Stone compression experiment on real Kotlin source files.
"""

import re
from collections import Counter

SYMBOL_POOL = list("ΩΘΦΨΣΔΛΠΞΓαβγδεζηκλμνξπρστυφχψω§¶†‡•◆▲■★")
MIN_TERM_LEN = 6
MIN_TERM_FREQ = 2
HEADER_COST = 20   # chars per symbol entry in header
MIN_STRIP_LEN = 5
VOWELS = set("aeiouAEIOU")


def term_frequencies(text):
    return Counter(re.findall(r'[A-Za-z][A-Za-z0-9_]{' + str(MIN_TERM_LEN-1) + r',}', text))


def build_symbol_table(freq):
    ranked = sorted(
        [(t, f, (len(t)-1)*f - HEADER_COST) for t, f in freq.items()
         if f >= MIN_TERM_FREQ and (len(t)-1)*f - HEADER_COST > 0],
        key=lambda x: -x[2]
    )
    return {term: SYMBOL_POOL[i] for i, (term, _, _) in enumerate(ranked) if i < len(SYMBOL_POOL)}


def apply_symbols(text, table):
    for term in sorted(table, key=len, reverse=True):
        text = re.sub(r'\b' + re.escape(term) + r'\b', table[term], text)
    return text


def strip_vowels(text):
    def strip_word(m):
        w = m.group()
        if w.isupper(): return w          # ALL-CAPS preserved
        if '_' in w: return w             # snake_case preserved
        if any(c.isupper() for c in w[1:]): return w  # camelCase preserved
        return w[0] + ''.join(c for c in w[1:] if c not in VOWELS)
    return re.sub(r'\b[A-Za-z]{' + str(MIN_STRIP_LEN) + r',}\b', strip_word, text)


def build_header(table):
    if not table:
        return f"This is your Rosetta Stone.\nRule: vowels stripped from plain words≥{MIN_STRIP_LEN}chr (ALL-CAPS/camelCase/snake_case preserved).\n---\n"
    terms = "  ".join(f"{sym}={term}" for term, sym in table.items())
    return f"This is your Rosetta Stone.\nRule: vowels stripped from plain words≥{MIN_STRIP_LEN}chr (ALL-CAPS/camelCase/snake_case preserved).\nTerms: {terms}\n---\n"


def encode(text):
    freq = term_frequencies(text)
    table = build_symbol_table(freq)
    body = apply_symbols(text, table)
    body = strip_vowels(body)
    header = build_header(table)
    full = header + body
    return {
        'header': header,
        'body': body,
        'full': full,
        'symbol_table': table,
        'freq': freq,
        'orig_len': len(text),
        'enc_len': len(full),
        'ratio': len(full) / len(text),
        'reduction_pct': (1 - len(full)/len(text)) * 100
    }


def detailed_symbol_table(table, freq, text):
    """Return per-symbol stats: term, freq, gross savings, header cost, net savings."""
    rows = []
    for term, sym in table.items():
        f = freq[term]
        gross = (len(term) - 1) * f   # replacing len(term) chars with 1 char, times freq
        net = gross - HEADER_COST
        rows.append((sym, term, f, gross, net))
    rows.sort(key=lambda x: -x[4])
    return rows


def run_experiment(filepath, label):
    with open(filepath, 'r', encoding='utf-8') as fh:
        text = fh.read()

    result = encode(text)
    table = result['symbol_table']
    freq = result['freq']
    rows = detailed_symbol_table(table, freq, text)

    orig_tokens = result['orig_len'] / 4
    enc_tokens = result['enc_len'] / 4

    # Header vs body savings
    header_size = len(result['header'])
    body_size = len(result['body'])
    # Savings = orig_len - body_size (header is overhead)
    body_savings = result['orig_len'] - body_size
    net_savings = result['orig_len'] - result['enc_len']

    # Symbol savings vs vowel stripping savings
    # Apply only symbols (no vowel strip) to measure symbol contribution
    body_symbols_only = apply_symbols(text, table)
    symbol_savings = result['orig_len'] - len(body_symbols_only)

    # Apply only vowel strip (no symbols) to measure vowel contribution
    body_vowels_only = strip_vowels(text)
    vowel_savings = result['orig_len'] - len(body_vowels_only)

    print(f"\n{'='*72}")
    print(f"FILE: {label}")
    print(f"{'='*72}")
    print(f"\nOriginal:  {result['orig_len']:,} chars  (~{orig_tokens:,.0f} tokens)")
    print(f"Encoded:   {result['enc_len']:,} chars  (~{enc_tokens:,.0f} tokens)")
    print(f"Reduction: {result['reduction_pct']:.1f}%")
    print(f"\nHeader size: {header_size:,} chars (overhead)")
    print(f"Body savings vs orig: {body_savings:+,} chars")
    print(f"Net savings after header: {net_savings:+,} chars")
    print(f"\nSavings attribution (before header overhead):")
    print(f"  Symbol substitution alone:  {symbol_savings:,} chars saved")
    print(f"  Vowel stripping alone:      {vowel_savings:,} chars saved")
    combined_gross = symbol_savings + vowel_savings
    if combined_gross > 0:
        print(f"  Symbol share: {symbol_savings/combined_gross*100:.1f}%  |  Vowel share: {vowel_savings/combined_gross*100:.1f}%")

    print(f"\nSymbol table ({len(table)} symbols assigned, pool capacity {len(SYMBOL_POOL)}):")
    print(f"  {'SYM':<4} {'TERM':<45} {'FREQ':>5}  {'GROSS':>7}  {'NET':>7}")
    print(f"  {'-'*4} {'-'*45} {'-'*5}  {'-'*7}  {'-'*7}")
    for sym, term, f, gross, net in rows:
        print(f"  {sym:<4} {term:<45} {f:>5}  {gross:>7,}  {net:>7,}")

    print(f"\nTop 5 by net savings:")
    for sym, term, f, gross, net in rows[:5]:
        print(f"  {sym} = '{term}'  freq={f}  gross={gross}  net={net}")

    print(f"\nFirst 500 chars of encoded body:")
    print("-" * 60)
    print(result['body'][:500])
    print("-" * 60)

    return result


# ---- Run on both files ----

CULT_FILE = (
    "/Users/will/Development/neuron-technologies/neuron/neuron-core/"
    "src/main/kotlin/ai/neuron/core/synapse/services/managers/"
    "cultivation/CultivationManager.kt"
)
DHARMA_FILE = (
    "/Users/will/Development/neuron-technologies/neuron/neuron-core/"
    "src/main/kotlin/ai/neuron/core/synapse/services/managers/"
    "dharma/DharmaManager.kt"
)

r1 = run_experiment(CULT_FILE, "CultivationManager.kt")
r2 = run_experiment(DHARMA_FILE, "DharmaManager.kt")

# ---- Combined summary ----
total_orig = r1['orig_len'] + r2['orig_len']
total_enc = r1['enc_len'] + r2['enc_len']
total_reduction = (1 - total_enc / total_orig) * 100

print(f"\n{'='*72}")
print("COMBINED SUMMARY")
print(f"{'='*72}")
print(f"Total original:  {total_orig:,} chars  (~{total_orig/4:,.0f} tokens)")
print(f"Total encoded:   {total_enc:,} chars  (~{total_enc/4:,.0f} tokens)")
print(f"Combined reduction: {total_reduction:.1f}%")
print(f"Token budget saved: ~{(total_orig - total_enc)/4:,.0f} tokens")

# ---- Analysis ----
print("""
ANALYSIS
--------
Symbol table: terms must appear ≥2 times and have (len-1)*freq > 20 net chars saved.
In Kotlin source, the dominant repeating terms are fully-qualified class/method names,
type names, and domain vocabulary (prediction, evaluation, graph, properties, etc.).

The algorithm performs best on:
 - Long identifiers repeated many times (e.g. 'properties', 'evaluationId', 'graph')
 - Domain-specific vocabulary that recurs heavily across a file

Vowel stripping covers everything else: short repeated words, Kotlin keywords,
and identifiers that don't hit the symbol threshold.

Readability of the encoded form is coarse but navigable: the header provides a
lookup table and camelCase/ALL-CAPS/snake_case identifiers are left intact,
which preserves the structural skeleton of the code.
""")