"""T12. Placement on the cross-linguistic entropy law (after Bentz, Alikaniotis, Cysouw & Ferrer-i-Cancho
2017, who report entropy rate ~ -1.12 + 0.78 * unigram entropy, r ~ 0.95, across 1,115 languages).
We refit the relation on our own 103 Bible samples (same length as the manuscript's running text) with
two entropy-rate estimators: conditional word entropy h2(words) and a compression estimate (zlib on
word indices, bits per word).  Report the manuscript's residual in standard deviations of the fit.
Prediction under 'encoded language at the word level': residual within +-2 sd.  Writes out/entropy_law.csv.
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, csv, math, zlib, sys
import numpy as np
from harness import structured, flatten, languages, SURROGATES, log

N = 34217


def H1(words):
    c = collections.Counter(words); n = len(words)
    return -sum(v / n * math.log2(v / n) for v in c.values())


def h2(words):
    c1 = collections.Counter(words); c2 = collections.Counter(zip(words, words[1:])); n = len(words) - 1
    return -sum(v / n * math.log2(v / c1[a]) for (a, b), v in c2.items())


def comp_rate(words, seed=0):
    """bits per word of the zlib-compressed word-index stream.  Word ids are assigned at random
    (seeded), not by first appearance: first-appearance ids grow along the text and zlib exploits
    that growth, which would credit vocabulary growth to word order."""
    idx = {}; a = np.array([idx.setdefault(w, len(idx)) for w in words], dtype=np.uint32)
    perm = np.random.default_rng(seed).permutation(len(idx)); a = perm[a].astype(np.uint32)
    return 8 * len(zlib.compress(a.tobytes(), 9)) / len(words)


rows = []
pages = structured(); voy = flatten(pages)[:N]
rows.append(dict(text="Voynich", H1=H1(voy), h2=h2(voy), comp=comp_rate(voy)))
sh = flatten(SURROGATES["words-global"](pages, seed=1))[:N]
rows.append(dict(text="Voynich, words shuffled", H1=H1(sh), h2=h2(sh), comp=comp_rate(sh)))
NO_SPACES = {"Japanese"}  # sample written without word spaces: its "words" are phrases (mean 13 characters)
for name, ws in languages().items():
    if name.endswith(".novowel") or name in NO_SPACES: continue
    ws = ws[:N]
    if len(ws) < N * 0.9: continue
    rows.append(dict(text=name, H1=H1(ws), h2=h2(ws), comp=comp_rate(ws)))
langs = [r for r in rows if not r["text"].startswith("Voynich")]
out = []
for est in ("h2", "comp"):
    x = np.array([r["H1"] for r in langs]); y = np.array([r[est] for r in langs])
    b, a = np.polyfit(x, y, 1); res = y - (a + b * x); sd = res.std(); rr = np.corrcoef(x, y)[0, 1]
    for r in rows[:2]:
        z = (r[est] - (a + b * r["H1"])) / sd
        out.append(dict(estimator=est, text=r["text"], H1=round(r["H1"], 3), rate=round(r[est], 3), predicted=round(a + b * r["H1"], 3), residual_sd=round(z, 2), fit=f"{a:.3f} + {b:.3f} H1, r={rr:.3f}, n={len(langs)}"))
        print(out[-1])
with open("out/entropy_law.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=["text", "H1", "h2", "comp"]); w.writeheader()
    for r in rows: w.writerow({k: (round(v, 3) if isinstance(v, float) else v) for k, v in r.items()})
with open("out/entropy_law_fit.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=out[0].keys()); w.writeheader(); w.writerows(out)
v = [o for o in out if o["text"] == "Voynich"]
log("H-LANG-WORDLEVEL", "T12-entropy-law", "residual from the H1 vs entropy-rate law (word level), sd units", f"h2: {v[0]['residual_sd']}; compression: {v[1]['residual_sd']}", "103 languages, same length", "within 2 sd" if all(abs(x["residual_sd"]) < 2 for x in v) else "outside 2 sd", "out/entropy_law_fit.csv")


# ---- gain from word order: compressed size of the shuffled text minus the text, bits per word
import random
res = []
sh = voy[:]; random.Random(0).shuffle(sh); res.append(("Voynich", comp_rate(sh) - comp_rate(voy)))
for name, ws in languages().items():
    if name.endswith(".novowel") or name in NO_SPACES: continue
    ws = ws[:N]
    if len(ws) < N * 0.9: continue
    s2 = ws[:]; random.Random(0).shuffle(s2); res.append((name, comp_rate(s2) - comp_rate(ws)))
g = sorted(v for n, v in res[1:])
print("order gain: Voynich %.3f; languages min %.3f median %.3f max %.3f; n below Voynich %d of %d" % (res[0][1], g[0], g[len(g) // 2], g[-1], sum(v < res[0][1] for v in g), len(g)))
with open("out/order_gain.csv", "w", newline="") as f:
    w = csv.writer(f); w.writerow(["text", "order_gain_bits_per_word"]); w.writerows([(n, round(v, 3)) for n, v in res])
log("H-LANG-WORDLEVEL", "T12-order-gain", "bits per word saved by word order (random word ids)", round(res[0][1], 3), f"{len(g)} languages with word spaces: {g[0]:.2f}-{g[-1]:.2f}, median {g[len(g)//2]:.2f}", "outside the language range (below all)" if res[0][1] < g[0] else "inside", "Japanese excluded: sample has no word spaces; out/order_gain.csv")