#!/usr/bin/env python3 """ Rosetta Stone compression experiment on real Kotlin source files. """ import re from collections import Counter SYMBOL_POOL = list("ΩΘΦΨΣΔΛΠΞΓαβγδεζηκλμνξπρστυφχψω§¶†‡•◆▲■★") MIN_TERM_LEN = 6 MIN_TERM_FREQ = 2 HEADER_COST = 20 # chars per symbol entry in header MIN_STRIP_LEN = 5 VOWELS = set("aeiouAEIOU") def term_frequencies(text): return Counter(re.findall(r'[A-Za-z][A-Za-z0-9_]{' + str(MIN_TERM_LEN-1) + r',}', text)) def build_symbol_table(freq): ranked = sorted( [(t, f, (len(t)-1)*f - HEADER_COST) for t, f in freq.items() if f >= MIN_TERM_FREQ and (len(t)-1)*f - HEADER_COST > 0], key=lambda x: -x[2] ) return {term: SYMBOL_POOL[i] for i, (term, _, _) in enumerate(ranked) if i < len(SYMBOL_POOL)} def apply_symbols(text, table): for term in sorted(table, key=len, reverse=True): text = re.sub(r'\b' + re.escape(term) + r'\b', table[term], text) return text def strip_vowels(text): def strip_word(m): w = m.group() if w.isupper(): return w # ALL-CAPS preserved if '_' in w: return w # snake_case preserved if any(c.isupper() for c in w[1:]): return w # camelCase preserved return w[0] + ''.join(c for c in w[1:] if c not in VOWELS) return re.sub(r'\b[A-Za-z]{' + str(MIN_STRIP_LEN) + r',}\b', strip_word, text) def build_header(table): if not table: return f"This is your Rosetta Stone.\nRule: vowels stripped from plain words≥{MIN_STRIP_LEN}chr (ALL-CAPS/camelCase/snake_case preserved).\n---\n" terms = " ".join(f"{sym}={term}" for term, sym in table.items()) return f"This is your Rosetta Stone.\nRule: vowels stripped from plain words≥{MIN_STRIP_LEN}chr (ALL-CAPS/camelCase/snake_case preserved).\nTerms: {terms}\n---\n" def encode(text): freq = term_frequencies(text) table = build_symbol_table(freq) body = apply_symbols(text, table) body = strip_vowels(body) header = build_header(table) full = header + body return { 'header': header, 'body': body, 'full': full, 'symbol_table': table, 'freq': freq, 'orig_len': len(text), 'enc_len': len(full), 'ratio': len(full) / len(text), 'reduction_pct': (1 - len(full)/len(text)) * 100 } def detailed_symbol_table(table, freq, text): """Return per-symbol stats: term, freq, gross savings, header cost, net savings.""" rows = [] for term, sym in table.items(): f = freq[term] gross = (len(term) - 1) * f # replacing len(term) chars with 1 char, times freq net = gross - HEADER_COST rows.append((sym, term, f, gross, net)) rows.sort(key=lambda x: -x[4]) return rows def run_experiment(filepath, label): with open(filepath, 'r', encoding='utf-8') as fh: text = fh.read() result = encode(text) table = result['symbol_table'] freq = result['freq'] rows = detailed_symbol_table(table, freq, text) orig_tokens = result['orig_len'] / 4 enc_tokens = result['enc_len'] / 4 # Header vs body savings header_size = len(result['header']) body_size = len(result['body']) # Savings = orig_len - body_size (header is overhead) body_savings = result['orig_len'] - body_size net_savings = result['orig_len'] - result['enc_len'] # Symbol savings vs vowel stripping savings # Apply only symbols (no vowel strip) to measure symbol contribution body_symbols_only = apply_symbols(text, table) symbol_savings = result['orig_len'] - len(body_symbols_only) # Apply only vowel strip (no symbols) to measure vowel contribution body_vowels_only = strip_vowels(text) vowel_savings = result['orig_len'] - len(body_vowels_only) print(f"\n{'='*72}") print(f"FILE: {label}") print(f"{'='*72}") print(f"\nOriginal: {result['orig_len']:,} chars (~{orig_tokens:,.0f} tokens)") print(f"Encoded: {result['enc_len']:,} chars (~{enc_tokens:,.0f} tokens)") print(f"Reduction: {result['reduction_pct']:.1f}%") print(f"\nHeader size: {header_size:,} chars (overhead)") print(f"Body savings vs orig: {body_savings:+,} chars") print(f"Net savings after header: {net_savings:+,} chars") print(f"\nSavings attribution (before header overhead):") print(f" Symbol substitution alone: {symbol_savings:,} chars saved") print(f" Vowel stripping alone: {vowel_savings:,} chars saved") combined_gross = symbol_savings + vowel_savings if combined_gross > 0: print(f" Symbol share: {symbol_savings/combined_gross*100:.1f}% | Vowel share: {vowel_savings/combined_gross*100:.1f}%") print(f"\nSymbol table ({len(table)} symbols assigned, pool capacity {len(SYMBOL_POOL)}):") print(f" {'SYM':<4} {'TERM':<45} {'FREQ':>5} {'GROSS':>7} {'NET':>7}") print(f" {'-'*4} {'-'*45} {'-'*5} {'-'*7} {'-'*7}") for sym, term, f, gross, net in rows: print(f" {sym:<4} {term:<45} {f:>5} {gross:>7,} {net:>7,}") print(f"\nTop 5 by net savings:") for sym, term, f, gross, net in rows[:5]: print(f" {sym} = '{term}' freq={f} gross={gross} net={net}") print(f"\nFirst 500 chars of encoded body:") print("-" * 60) print(result['body'][:500]) print("-" * 60) return result # ---- Run on both files ---- CULT_FILE = ( "/Users/will/Development/neuron-technologies/neuron/neuron-core/" "src/main/kotlin/ai/neuron/core/synapse/services/managers/" "cultivation/CultivationManager.kt" ) DHARMA_FILE = ( "/Users/will/Development/neuron-technologies/neuron/neuron-core/" "src/main/kotlin/ai/neuron/core/synapse/services/managers/" "dharma/DharmaManager.kt" ) r1 = run_experiment(CULT_FILE, "CultivationManager.kt") r2 = run_experiment(DHARMA_FILE, "DharmaManager.kt") # ---- Combined summary ---- total_orig = r1['orig_len'] + r2['orig_len'] total_enc = r1['enc_len'] + r2['enc_len'] total_reduction = (1 - total_enc / total_orig) * 100 print(f"\n{'='*72}") print("COMBINED SUMMARY") print(f"{'='*72}") print(f"Total original: {total_orig:,} chars (~{total_orig/4:,.0f} tokens)") print(f"Total encoded: {total_enc:,} chars (~{total_enc/4:,.0f} tokens)") print(f"Combined reduction: {total_reduction:.1f}%") print(f"Token budget saved: ~{(total_orig - total_enc)/4:,.0f} tokens") # ---- Analysis ---- print(""" ANALYSIS -------- Symbol table: terms must appear ≥2 times and have (len-1)*freq > 20 net chars saved. In Kotlin source, the dominant repeating terms are fully-qualified class/method names, type names, and domain vocabulary (prediction, evaluation, graph, properties, etc.). The algorithm performs best on: - Long identifiers repeated many times (e.g. 'properties', 'evaluationId', 'graph') - Domain-specific vocabulary that recurs heavily across a file Vowel stripping covers everything else: short repeated words, Kotlin keywords, and identifiers that don't hit the symbol threshold. Readability of the encoded form is coarse but navigable: the header provides a lookup table and camelCase/ALL-CAPS/snake_case identifiers are left intact, which preserves the structural skeleton of the code. """)