"""Session S-007: tests of the layout layer, registered in out/prereg_S007.yaml before this ran.
Reads the CONFIRMATION half only; the rules come from half E (layout_layer.py explore).
Three versions of half C are compared: raw, normalised (line-edge spellings rewritten in their inside
form) and interior (first and last word of every line dropped).  Writes out/layout_tests.csv and
out/layout_signatures.csv.

T32 rule replication     each half-E rule: share at its position vs inside on half C (within-page shuffle z)
T33 layout removed       information of line position about first and last glyph, raw vs normalised vs
                         interior; Latin laid in the same lines as control
T34 signatures           glyph conditional entropy, word-order gain, form-function coupling, adjacent
                         repeats, repeated 3- and 4-word phrases, adjacent-page vocabulary z; for each
                         version, against reference languages cut to the same number of words
T35 crowding             H-G-COPIED: is the line-final -m form commoner in fuller lines?  Fullness is the
                         glyph count of the line before its last word, over the page's median; -m share
                         in the fullest vs emptiest third of lines (within-page shuffle z)
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, itertools, math, random, zlib
import numpy as np, pandas as pd
HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE); sys.path.insert(0, os.path.join(HERE, "..")); sys.path.insert(0, os.path.join(HERE, "..", "framework"))
from scipy.stats import spearmanr
from corpus import tokenize_glyphs
from texts import voynich, latin, page_words
import layout_layer as LL
from harness import languages, entropies

rows = []; rng = random.Random(0)
def rec(test, hyp, statistic, value, baseline, decision, note=""):
    rows.append(dict(test=test, hypothesis=hyp, statistic=statistic, value=value, baseline=baseline, decision=decision, note=note)); print(rows[-1], flush=True)


V = voynich(); R = LL.rules(); VC = LL.half(V, "C")
LA = latin(V); LC = LL.half(LA, "C")
VERS = {"raw": VC, "normalised": LL.normalise_pages(VC, R, ref=V)[0], "interior": LL.interior(VC)}
LVERS = {"raw": LC, "normalised": LC, "interior": LL.interior(LC)}  # Latin has no kept rules to apply (layout_control.csv)
share = LL.normalise_pages(VC, R, ref=V)[1]


# ---- T32 rule replication on half C
def rule_share(pages, r, n_perm=300):
    occ = LL.positioned(pages); a = tuple(tokenize_glyphs(r["edge_form"])); b = () if r["inside_form"] == "(none)" else tuple(tokenize_glyphs(r["inside_form"]))
    U = {w: tuple(tokenize_glyphs(w)) for _, _, w in occ}; voc = set(U.values())
    def stem(t, form):
        if r["side"] == "end": return t[:-len(form)] if form and len(t) > len(form) and t[-len(form):] == form else None
        return t[len(form):] if len(t) > len(form) and t[:len(form)] == form else (t if not form else None)
    sa = {stem(t, a) for t in voc} - {None}; sb = {stem(t, b) for t in voc} - {None}; stems = sa & sb
    sel = []
    for pg, pos, w in occ:
        if pos not in (r["position"], "inside"): continue
        t = U[w]; s1 = stem(t, a)
        if s1 in stems: sel.append((pg, pos, True)); continue
        s2 = stem(t, b) if b else t
        if s2 in stems: sel.append((pg, pos, False))
    atP = [x for _, p, x in sel if p == r["position"]]; ins = [x for _, p, x in sel if p == "inside"]
    if len(atP) < 5 or not ins: return len(stems), len(atP), None, None, None
    obs = np.mean(atP) - np.mean(ins); by = collections.defaultdict(list)
    for k, (pg, _, _) in enumerate(sel): by[pg].append(k)
    null = []
    for _ in range(n_perm):
        lab = [p for _, p, _ in sel]
        for ks in by.values():
            v = [lab[k] for k in ks]; rng.shuffle(v)
            for k, x in zip(ks, v): lab[k] = x
        e = [sel[k][2] for k in range(len(sel)) if lab[k] == r["position"]]; i_ = [sel[k][2] for k in range(len(sel)) if lab[k] == "inside"]
        null.append((np.mean(e) if e else 0) - (np.mean(i_) if i_ else 0))
    return len(stems), len(atP), float(np.mean(atP)), float(np.mean(ins)), float((obs - np.mean(null)) / (np.std(null) + 1e-12))


rep = []
for r in R.to_dict("records"):
    n, nP, sP, sI, z = rule_share(VC, r)
    ok = z is not None and sI > 0 and sP / sI >= 2 and z >= 3
    rep.append(dict(rule=f"{r['position']}: {r['edge_form']} for {r['inside_form']}", stems_C=n, tokens_C=nP, share_at_P_C=sP, share_inside_C=sI, z_C=z, replicated=ok,
                    share_at_P_E=r["share_at_P"], share_inside_E=r["share_inside"], z_E=r["z"]))
pd.DataFrame(rep).to_csv(os.path.join(HERE, "out", "layout_replication.csv"), index=False)
k = sum(x["replicated"] for x in rep)
rec("T32-rule-replication", "H-G-LINEEDGE", "half-E line-edge rules that replicate on half C (ratio >= 2, z >= 3)", f"{k} of {len(rep)}",
    "within-page shuffle per rule", "supported" if k >= 0.8 * len(rep) else "partly supported" if k >= 0.5 * len(rep) else "not supported", "out/layout_replication.csv")

# ---- T33 layout removed
for name, P in VERS.items():
    f, l = LL.layout_mi(P); lf, ll = LL.layout_mi(LVERS[name], glyphs=False)
    rec("T33-layout-mi", "H-G-LINEEDGE", f"MI of line position about first / last glyph, half C, {name}", f"{f} / {l}", f"Latin same lines {lf} / {ll}", name,
        f"{100 * share:.1f} % of half-C tokens rewritten" if name == "normalised" else "")
raw_mi = LL.layout_mi(VC); norm_mi = LL.layout_mi(VERS["normalised"])
removed = 1 - (norm_mi[0] + norm_mi[1]) / (raw_mi[0] + raw_mi[1])
rec("T33-layout-removed", "H-G-LINEEDGE", "share of the line-position information removed by the normaliser (half C)", f"{100 * removed:.0f} %",
    "registered bar: at least half", "supported" if removed >= 0.5 else "not supported")


# ---- T34 signatures
def flat(P): return [w for p in P for l in p["lines"] for w in l]
def h2_units(words):
    s = [u for w in words for u in tokenize_glyphs(w) + [" "]]; return entropies(s)[2]
def h2_chars(words):
    s = [c for w in words for c in list(w) + [" "]]; return entropies(s)[2]
def comp_rate(words, seed=0):
    idx = {}; a = np.array([idx.setdefault(w, len(idx)) for w in words], dtype=np.uint32)
    perm = np.random.default_rng(seed).permutation(len(idx)); a = perm[a].astype(np.uint32)
    return 8 * len(zlib.compress(a.tobytes(), 9)) / len(words)
def order_gain(words):
    s = words[:]; random.Random(0).shuffle(s); return comp_rate(s) - comp_rate(words)
def edit(a, b):
    if a == b: return 0
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i]
        for j, cb in enumerate(b, 1): cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
        prev = cur
    return prev[-1]
def form_function(words, K=300):
    cnt = collections.Counter(words); top = [w for w, c in cnt.most_common(K)]; ti = {w: i for i, w in enumerate(top)}; K = len(top)
    ctx = np.zeros((K, 2 * K))
    for i in range(1, len(words) - 1):
        w = words[i]
        if w in ti:
            if words[i - 1] in ti: ctx[ti[w], ti[words[i - 1]]] += 1
            if words[i + 1] in ti: ctx[ti[w], K + ti[words[i + 1]]] += 1
    ctx = ctx / (np.linalg.norm(ctx, axis=1, keepdims=True) + 1e-9); D = ctx @ ctx.T
    form = np.zeros((K, K))
    for i, j in itertools.combinations(range(K), 2): form[i, j] = form[j, i] = 1 - edit(top[i], top[j]) / max(len(top[i]), len(top[j]))
    iu = np.triu_indices(K, 1); return float(spearmanr(form[iu], D[iu]).correlation)
def adjacent(P):
    same = near = n = 0
    for p in P:
        for l in p["lines"]:
            for a, b in zip(l, l[1:]): n += 1; same += a == b; near += edit(a, b) <= 1
    return same / max(n, 1), near / max(n, 1)
def repeats(P, n):
    where = collections.defaultdict(set)
    for p in P:
        for li, l in enumerate(p["lines"]):
            for i in range(len(l) - n + 1): where[tuple(l[i:i + n])].add((p["page"], li))
    return sum(1 for g, w in where.items() if len(w) > 1)
def adjacent_pages_z(P, n_top=400, n_perm=300):
    P = [p for p in P if len(flat([p])) >= 60]; cnt = collections.Counter(flat(P)); top = [w for w, c in cnt.most_common(n_top)]; ti = {w: i for i, w in enumerate(top)}
    M = np.zeros((len(P), len(top)))
    for i, p in enumerate(P):
        for w in flat([p]):
            if w in ti: M[i, ti[w]] += 1
    M = M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9); S = M @ M.T
    order = [p["page"] for p in V]; pos = {pg: i for i, pg in enumerate(order)}
    idx = [(i, j) for i in range(len(P)) for j in range(i + 1, len(P)) if abs(pos[P[i]["page"]] - pos[P[j]["page"]]) == 1]
    obs = np.mean([S[i, j] for i, j in idx]); bysec = collections.defaultdict(list)
    for k, p in enumerate(P): bysec[p["section"]].append(k)
    null = []
    for _ in range(n_perm):
        v = []
        for i, j in idx:
            a = rng.choice(bysec[P[i]["section"]]); b = rng.choice(bysec[P[j]["section"]])
            while b == a and len(bysec[P[j]["section"]]) > 1: b = rng.choice(bysec[P[j]["section"]])
            v.append(S[a, b])
        null.append(np.mean(v))
    return float((obs - np.mean(null)) / (np.std(null) + 1e-12))


L = {k: v for k, v in languages().items() if not k.endswith(".novowel") and k != "Japanese"}
FF_LANGS = ["Latin", "Italian", "German", "Czech", "Hungarian", "Hebrew", "Nahuatl-NT", "Swahili-NT", "Quichua-NT", "Malagasy", "Turkish", "Finnish"]
sig = []
for name, P in VERS.items():
    W = flat(P); N = len(W); LW = {k: v[:N] for k, v in L.items() if len(v) >= N}
    lang_h2 = sorted(h2_chars(v) for v in LW.values()); lang_gain = sorted(order_gain(v) for v in LW.values())
    lang_ff = sorted(form_function(LW[k]) for k in FF_LANGS if k in LW)
    lang_adj = sorted(adjacent([dict(page="x", lines=[v])])[1] for v in LW.values() if np.mean([len(w) for w in v]) >= 3)  # languages written in words, not syllables
    la = flat(LVERS[name])
    s = dict(version=name, words=N, h2_eva=h2_chars(W), h2_units=h2_units(W), h2_lang_min=lang_h2[0], h2_lang_median=lang_h2[len(lang_h2) // 2],
             order_gain=order_gain(W), gain_lang_min=lang_gain[0], gain_lang_median=lang_gain[len(lang_gain) // 2],
             form_function=form_function(W), ff_lang_max=lang_ff[-1], ff_latin_lines=form_function(la),
             adj_same=adjacent(P)[0], adj_edit1=adjacent(P)[1], adj_edit1_lang_median=lang_adj[len(lang_adj) // 2], adj_edit1_lang_max=lang_adj[-1], adj_edit1_latin_lines=adjacent(LVERS[name])[1],
             rep3=repeats(P, 3), rep4=repeats(P, 4), rep3_latin_lines=repeats(LVERS[name], 3), rep4_latin_lines=repeats(LVERS[name], 4),
             adjacent_pages_z=adjacent_pages_z(P))
    sig.append(s); print({k: (round(v, 3) if isinstance(v, float) else v) for k, v in s.items()}, flush=True)
S = pd.DataFrame(sig).round(4); S.to_csv(os.path.join(HERE, "out", "layout_signatures.csv"), index=False)

raw, nrm, itr = sig
def outside(s): return dict(entropy=s["h2_eva"] < s["h2_lang_min"], order=s["order_gain"] < s["gain_lang_min"], ff=s["form_function"] > s["ff_lang_max"], phrases=s["rep4"] <= 2)
for s in sig:
    o = outside(s)
    rec("T34-signatures", "H-G-LAYOUT-ROBUST", f"half C, {s['version']}: entropy / order gain / form-function / 4-word repeats outside the language range",
        f"h2 {s['h2_eva']:.2f}, gain {s['order_gain']:.2f}, coupling {s['form_function']:.3f}, 4-gram repeats {s['rep4']}",
        f"languages: h2 min {s['h2_lang_min']:.2f}, gain min {s['gain_lang_min']:.2f}, coupling max {s['ff_lang_max']:.3f}; Latin in lines {s['rep4_latin_lines']} 4-gram repeats",
        "all four outside" if all(o.values()) else "inside on: " + ", ".join(k for k, v in o.items() if not v))
robust = all(all(outside(s).values()) for s in (nrm, itr))
rec("T34-robust", "H-G-LAYOUT-ROBUST", "do the four signatures survive both treatments?", "yes" if robust else "no",
    "registered: supported only if all four stay outside the language range under both", "supported" if robust else "not supported")
mask = [(s["version"], s["form_function"] <= raw["form_function"] * 2 / 3, s["order_gain"] >= raw["order_gain"] + 0.10) for s in (nrm, itr)]
both = all(a or b for _, a, b in mask); one = any(a or b for _, a, b in mask)
rec("T34-mask", "H-G-LAYOUT-MASK", "coupling falls by a third, or order gain rises by 0.10 bits, under each treatment",
    "; ".join(f"{v}: coupling {'yes' if a else 'no'}, gain {'yes' if b else 'no'}" for v, a, b in mask),
    f"raw coupling {raw['form_function']:.3f}, raw gain {raw['order_gain']:.2f}", "supported" if both else "partly supported" if one else "not supported")


# ---- T35 crowding (H-G-COPIED)
words = collections.Counter(w for p in V for w in page_words(p)); stems = set()
for w in words:
    if w.endswith("m") and len(w) > 1 and any(w[:-1] + e in words for e in ("in", "iin", "n", "r", "l")): stems.add(w[:-1])
occ = []
for p in VC:
    lens = [sum(len(tokenize_glyphs(w)) for w in l[:-1]) for l in p["lines"] if len(l) > 2]
    if len(lens) < 3: continue
    med = float(np.median(lens))
    for l in p["lines"]:
        if len(l) < 3: continue
        w = l[-1]; s = w[:-1] if w.endswith("m") else next((w[:-len(e)] for e in ("iin", "in", "n", "r", "l") if w.endswith(e) and w[:-len(e)] in stems), None)
        if s not in stems: continue
        occ.append((p["page"], sum(len(tokenize_glyphs(x)) for x in l[:-1]) / med, w.endswith("m")))
full = np.array([f for _, f, _ in occ]); m = np.array([x for _, _, x in occ]); lo, hi = np.quantile(full, [1 / 3, 2 / 3])
def diff(fv): return m[fv >= hi].mean() - m[fv <= lo].mean()
obs = diff(full); by = collections.defaultdict(list)
for k, (pg, _, _) in enumerate(occ): by[pg].append(k)
null = []
for _ in range(1000):
    fv = full.copy()
    for ks in by.values():
        v = list(fv[ks]); rng.shuffle(v); fv[ks] = v
    null.append(diff(fv))
z = (obs - np.mean(null)) / (np.std(null) + 1e-12)
rec("T35-crowding", "H-G-COPIED", "-m share among line-final alternants: fullest third of lines minus emptiest third (half C)",
    f"{m[full >= hi].mean():.3f} vs {m[full <= lo].mean():.3f} (z {z:.1f}, {len(occ)} line ends)", "within-page shuffle of fullness",
    "supported" if z > 3 else "not supported (no crowding effect)" if abs(z) < 2 else "not supported at the registered bar (trend in the predicted direction)" if z > 0 else "contrary")

pd.DataFrame(rows).to_csv(os.path.join(HERE, "out", "layout_tests.csv"), index=False)