"""T21c. Is the choice between spelling variants tied to the layout of the line?

Under H-CIPHER-MIXED the written variant of a plaintext word is a homophone choice made while
enciphering; the line break falls later, where the page runs out.  So among words that differ only
in one alternating glyph unit (a minimal-pair frame, e.g. chol / chor), which variant is written
should not depend on where in its line the word stands.  The test conditions on the frame: within
each frame, variant x line position (first word / inside / last word), conditional mutual
information summed over frames, against 500 shuffles of the variants within page and frame.
Paragraph-first and paragraph-last lines are excluded (their beginnings and ends are special for
other reasons).  Reported per alternation class, per scribe and per Currier language; Latin laid out
in the same lines and the mixed-key simulators are the controls.
Writes out/line_position.csv.
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, csv, math, random, sys
import numpy as np
from corpus import tokenize_glyphs
from harness import corpus, structured, split, flatten, languages, log
import ivtff, battery as B, generators as G, mixedkey as MK
from t_collapse2 import alternations

parse_g = lambda w: tuple(tokenize_glyphs(w)); parse_l = lambda w: tuple(w)


def pages_inner():
    """manuscript paragraph text without paragraph-first and -last lines"""
    C = corpus(); out = []
    for p in C.pages:
        v = C.pagevars[p]; lines = []
        for l in C.loci_where(generic="P", pages={p}):
            if l.par_start or l.par_end: continue
            ws = [w for w in ivtff.words(l.text) if "?" not in w]
            if len(ws) >= 3: lines.append(ws)
        if lines: out.append(dict(page=p, hand=v.get("H"), lang=v.get("L"), lines=lines))
    return out


def frames(words, parse, cls):
    """minimal-pair frames of alternation class (a, b): map word -> (frame id, variant)"""
    a, b = cls; types = {w: parse(w) for w in set(words)}; byrest = collections.defaultdict(dict)
    for w, t in types.items():
        for i, u in enumerate(t):
            if u in (a, b): byrest[(len(t), i, t[:i] + t[i + 1:])][u] = w
    out = {}
    for k, d in byrest.items():
        if len(d) == 2:
            for u, w in d.items(): out.setdefault(w, (k, u))
    return out


def cmi_test(pages, fmap, n_shuf=500, seed=0, group=lambda p: "all"):
    rng = random.Random(seed); res = {}
    occ = collections.defaultdict(list)
    for p in pages:
        g = group(p)
        for ln in p["lines"]:
            for i, w in enumerate(ln):
                if w in fmap:
                    fr, var = fmap[w]; lp = 0 if i == 0 else (2 if i == len(ln) - 1 else 1)
                    occ[g].append((p["page"], fr, var, lp))
    def cmi(rows):
        by = collections.defaultdict(list)
        for pg, fr, var, lp in rows: by[fr].append((var, lp))
        N = len(rows); tot = 0.0
        for fr, pr in by.items():
            n = len(pr); c = collections.Counter(pr); cx = collections.Counter(x for x, y in pr); cy = collections.Counter(y for x, y in pr)
            tot += n / N * sum(v / n * math.log2(v * n / (cx[x] * cy[y])) for (x, y), v in c.items())
        return tot
    for g, rows in occ.items():
        if len(rows) < 100: continue
        obs = cmi(rows); cells = collections.defaultdict(list)
        for k, r in enumerate(rows): cells[(r[0], r[1])].append(k)
        nul = []
        for _ in range(n_shuf):
            var = [r[2] for r in rows]
            for ks in cells.values():
                if len(ks) > 1:
                    vs = [var[k] for k in ks]; rng.shuffle(vs)
                    for k, v in zip(ks, vs): var[k] = v
            nul.append(cmi([(r[0], r[1], v, r[3]) for r, v in zip(rows, var)]))
        # line-final share of each variant, to show the direction
        res[g] = dict(n=len(rows), cmi=round(obs, 5), cmi_null=round(float(np.mean(nul)), 5), z=round((obs - np.mean(nul)) / (np.std(nul) + 1e-12), 1),
                      p=round((1 + sum(x >= obs for x in nul)) / (1 + n_shuf), 4))
    return res


def run(name, pages, parse, top=10, by=None):
    words = B.flat(pages); alts = alternations(words, parse, positional=False)
    rows = []
    for (pos, a, b), s, n in alts[:top]:
        fmap = frames(words, parse, (a, b))
        r = cmi_test(pages, fmap)
        if "all" in r: rows.append(dict(text=name, alternation=f"{a}/{b}", group="all", **r["all"]))
        if by:
            for g, d in sorted(cmi_test(pages, fmap, group=by).items(), key=lambda x: str(x[0])): rows.append(dict(text=name, alternation=f"{a}/{b}", group=g, **d))
    for r in rows: print(r, flush=True)
    return rows


if __name__ == "__main__":
    pin = pages_inner(); full = structured(); tr, te = split(full); lay = G.layout_of(full); trw = flatten(tr)
    out = []
    out += run("Voynich", pin, parse_g, by=lambda p: f"scribe {p['hand']}")
    for r in run("Voynich", pin, parse_g, by=lambda p: f"language {p['lang']}"):
        if r["group"].startswith("language"): out.append(r)
    lat = G.fill(lay, languages()["Latin"])
    out += run("Latin (same lines)", lat, parse_l)
    for sp in ("word", "split", "stream"):
        sim = MK.gen_tranchedino_sectioned(trw, lay, seed=7, spacing=sp, n_syl=60, n_tri=0, n_nom=80, k_v=3, k_c=2, p_syl=0.7)
        out += run(f"mixed key, {sp} spacing", sim, parse_g)
    keys = list(out[0].keys())
    with open("out/line_position.csv", "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=keys); w.writeheader(); w.writerows(out)
    def summ(text):
        z = [r["z"] for r in out if r["text"] == text and r["group"] == "all"]
        return f"median z {np.median(z):.1f} over {len(z)} classes ({sum(x > 3 for x in z)} above 3)" if z else "no classes"
    sc = [r for r in out if r["text"] == "Voynich" and r["group"].startswith("scribe")]
    log("H-CIPHER-MIXED", "T21c-line-position (within minimal-pair frames)", "conditional MI of variant and line position within frames vs within-page shuffles; paragraph-first/last lines excluded",
        summ("Voynich"), f"Latin {summ('Latin (same lines)')}; simulators word {summ('mixed key, word spacing')}, split {summ('mixed key, split spacing')}, stream {summ('mixed key, stream spacing')}; scribe rows with z>3: {sum(r['z'] > 3 for r in sc)}/{len(sc)}",
        "see note", "out/line_position.csv")