"""Deductive checks of hypotheses generated in session S-006 from the inductive battery and the
readings.  Every test reads only the CONFIRMATION half of the bifolium split (out/split.json), which
the inductive battery's candidate generation did not use (it read half E).  Writes out/generated_tests.csv.

T24 line-final -m:     among stems seen with both an -m ending and an -in/-n ending (dam/dain, okam/okain),
                       the -m form's share at line end vs inside the line (within-page shuffle null).
T25 A-B continuum:     correspondence analysis fitted on half E, half-C pages projected; is the axis bimodal
                       (two languages) or a continuum (a drifting system)? bimodality coefficient, middle-third share.
T26 shared labels:     share of pharmaceutical-label types that occur in herbal running text, vs zodiac/bather
                       label types (frequency-matched resampling).
T27 entry-initial words: herbal pages: first word of the page's text occurs on no other page more often than
                       words elsewhere in the first line (plant-name position).
T28 category prefixes: does the first glyph of a word carry the section more than a Latin first letter does,
                       and is ol- elevated in the astronomical and balneological sections (v0.1 M-03)?
T29 short tokens:      words of one or two glyphs (quantities, particles) more frequent in recipes than herbal.
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, json, math, os, random, sys
import numpy as np, pandas as pd
HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE); sys.path.insert(0, os.path.join(HERE, ".."))
from texts import voynich, latin, page_words
from corpus import tokenize_glyphs

H = json.load(open(os.path.join(HERE, "out", "split.json")))["half"]
V = voynich(); VC = [p for p in V if H[p["page"]] == "C"]; VE = [p for p in V if H[p["page"]] == "E"]
LA = latin(V); LC = [p for p in LA if H[p["page"]] == "C"]
rows = []; rng = random.Random(0)


def rec(test, hyp, statistic, value, baseline, decision, note=""):
    rows.append(dict(test=test, hypothesis=hyp, statistic=statistic, value=value, baseline=baseline, decision=decision, note=note)); print(rows[-1], flush=True)


# ---- T24
NASAL = [("m", "in"), ("m", "iin"), ("m", "n")]
def t24(pages):
    words = collections.Counter(w for p in pages for w in page_words(p)); stems = set()
    for w in words:
        if w.endswith("m") and len(w) > 1:
            s = w[:-1]
            if any(s + e in words for e in ("in", "iin", "n")): stems.add(s)
    occ = []
    for p in pages:
        for l in p["lines"]:
            for i, w in enumerate(l):
                for s in stems:
                    if w == s + "m" or w in (s + "in", s + "iin", s + "n"):
                        occ.append((p["page"], w.endswith("m"), i == len(l) - 1 and len(l) > 1))
    if not occ: return None
    m_end = np.mean([m for pg, m, e in occ if e]); m_in = np.mean([m for pg, m, e in occ if not e])
    obs = m_end - m_in; null = []
    by = collections.defaultdict(list)
    for k, o in enumerate(occ): by[o[0]].append(k)
    for _ in range(1000):
        mm = [o[1] for o in occ]
        for ks in by.values():
            v = [mm[k] for k in ks]; rng.shuffle(v)
            for k, x in zip(ks, v): mm[k] = x
        e = [mm[k] for k, o in enumerate(occ) if o[2]]; ins = [mm[k] for k, o in enumerate(occ) if not o[2]]
        null.append(np.mean(e) - np.mean(ins))
    z = (obs - np.mean(null)) / (np.std(null) + 1e-12)
    return len(stems), len(occ), m_end, m_in, z
r = t24(VC)
rec("T24-line-final-m", "H-G-LINEFINAL-M", "-m share at line end vs inside, stems with both -m and -in/-n (half C)",
    f"{r[2]:.3f} vs {r[3]:.3f} (z {r[4]:.1f})", f"{r[0]} stems, {r[1]} tokens; within-page shuffle", "supported" if r[4] > 3 else "not supported")


# ---- T25
def ca_fit(pages, n_top=400):
    cnt = collections.Counter(w for p in pages for w in page_words(p)); top = [w for w, c in cnt.most_common(n_top)]; ti = {w: i for i, w in enumerate(top)}
    def mat(ps):
        X = np.zeros((len(ps), len(top)))
        for k, p in enumerate(ps):
            for w in page_words(p):
                if w in ti: X[k, ti[w]] += 1
        return X
    X = mat(pages); P = X / X.sum(); r = P.sum(1, keepdims=True); c = P.sum(0, keepdims=True)
    S = (P - r @ c) / np.sqrt(r @ c + 1e-12); U, s, Vt = np.linalg.svd(S, full_matrices=False)
    G = Vt[0] / np.sqrt(c[0] + 1e-12)  # column standard coordinates, axis 1
    def project(ps):
        Y = mat(ps); prof = Y / (Y.sum(1, keepdims=True) + 1e-12); return prof @ G
    return project
proj = ca_fit(VE); xc = proj(VC); isB = np.array([p["lang"] == "B" for p in VC])
from scipy.stats import skew, kurtosis
n = len(xc); bc = (skew(xc) ** 2 + 1) / (kurtosis(xc) + 3 * (n - 1) ** 2 / ((n - 2) * (n - 3)))
lo, hi = np.percentile(xc, [5, 95]); mid = np.mean((xc > lo + (hi - lo) / 3) & (xc < hi - (hi - lo) / 3))
sepA, sepB = xc[~isB & np.array([p["lang"] == "A" for p in VC])], xc[isB]
overlap = np.mean([(b < max(sepA)) for b in sepB]) if len(sepA) and len(sepB) else None
rec("T25-ab-continuum", "H-G-AB-DRIFT", "bimodality coefficient of half-C pages on the E-fitted A-B axis; share in middle third; share of B pages inside the A range",
    f"BC {bc:.3f}; middle {mid:.2f}; B inside A range {overlap:.2f}", "BC > 0.555 suggests two modes", "bimodal: two systems" if bc > 0.555 else "continuum")


# ---- T26
import pandas as pd
TOK = pd.read_parquet(os.path.join(HERE, "out", "tokens.parquet"))
lab = TOK[TOK.kind == "label"]; herbal_C = set(w for p in VC if p["section"] == "H" for w in page_words(p))
def share(types): return np.mean([t in herbal_C for t in types]) if types else float("nan")
pl = sorted(set(lab[lab.section == "P"].word)); other = sorted(set(lab[lab.section.isin(["Z", "B", "C", "A"])].word))
freq = collections.Counter(w for p in V for w in page_words(p))
obs = share(pl); null = [share(rng.sample(other, min(len(pl), len(other)))) for _ in range(1000)]
z = (obs - np.mean(null)) / (np.std(null) + 1e-12)
rec("T26-shared-labels", "H-G-LABEL-NOMENCLATURE", "share of pharmaceutical label types found in half-C herbal running text vs other labels",
    f"{obs:.3f} ({len(pl)} types)", f"other labels {np.mean(null):.3f} (z {z:.1f})", "supported" if z > 2 else "not supported")


# ---- T27
pageset = collections.defaultdict(set)
for p in V:
    for w in set(page_words(p)): pageset[w].add(p["page"])
first, rest = [], []
for p in VC:
    if p["section"] != "H" or not p["lines"]: continue
    l = p["lines"][0]; first.append(len(pageset[l[0]]) == 1); rest += [len(pageset[w]) == 1 for w in l[1:]]
lfirst, lrest = [], []
lpageset = collections.defaultdict(set)
for p in LA:
    for w in set(page_words(p)): lpageset[w].add(p["page"])
for p in LC:
    if p["section"] != "H" or not p["lines"]: continue
    l = p["lines"][0]; lfirst.append(len(lpageset[l[0]]) == 1); lrest += [len(lpageset[w]) == 1 for w in l[1:]]
rec("T27-entry-initial-unique", "H-G-HERBAL-NAMES", "herbal pages (half C): first word unique to its page vs rest of first line",
    f"{np.mean(first):.2f} vs {np.mean(rest):.2f} (n {len(first)})", f"Latin same layout {np.mean(lfirst):.2f} vs {np.mean(lrest):.2f}",
    "supported" if np.mean(first) > np.mean(rest) + 0.15 else "not supported")


# ---- T28
def mi_first_section(pages, first):
    tab = collections.Counter((p["section"], first(w)) for p in pages for w in page_words(p)); N = sum(tab.values())
    ps = collections.Counter(); pf = collections.Counter()
    for (s, f), v in tab.items(): ps[s] += v; pf[f] += v
    return sum(v / N * math.log2(v * N / (ps[s] * pf[f])) for (s, f), v in tab.items())
mv = mi_first_section(VC, lambda w: tokenize_glyphs(w)[0]); ml = mi_first_section(LC, lambda w: w[0])
# whole-word information about section, for scale
mvw = mi_first_section(VC, lambda w: w); mlw = mi_first_section(LC, lambda w: w)
ol = {s: np.mean([w.startswith("ol") for p in VC if p["section"] == s for w in page_words(p)]) for s in sorted({p["section"] for p in VC})}
rec("T28-category-prefix", "H-V01-CONLANG", "MI(first glyph; section) as a share of MI(word; section), half C",
    f"{mv / mvw:.3f}", f"Latin first letter {ml / mlw:.3f}", "prefix carries more topic than in Latin" if mv / mvw > 1.5 * ml / mlw else "no excess")
rec("T28b-ol-prefix-by-section", "H-V01-CONLANG", "share of ol-initial words by section (v0.1 M-03 predicted astronomical and balneological highest), half C",
    "; ".join(f"{s} {v:.3f}" for s, v in ol.items()), "", "see values")


# ---- T29
short = {s: np.mean([len(tokenize_glyphs(w)) <= 2 for p in VC if p["section"] == s for w in page_words(p)]) for s in sorted({p["section"] for p in VC})}
rec("T29-short-tokens", "H-G-INFIRMARY", "share of 1-2 glyph words by section, half C (recipe reading predicts recipes > herbal)",
    "; ".join(f"{s} {v:.3f}" for s, v in short.items()), "", "supported" if short.get("S", 0) > short.get("H", 1) else "not supported")

pd.DataFrame(rows).to_csv(os.path.join(HERE, "out", "generated_tests.csv"), index=False)