"""T4. Sequence-analysis tests borrowed from bioinformatics / complex systems, each with
surrogates at several levels and the 103-language reference distribution where cheap.

T4a Mutual-information decay: I(X_t; X_{t+d}) over glyph units for d = 1..60, for the
    manuscript, its surrogates (chars-in-words, words-in-lines, words-in-pages, lines-global)
    and five reference languages. Long-range MI above the words-global surrogate = structure
    beyond the word; the level at which it vanishes locates it.
T4b Lempel-Ziv / gzip compressibility per section and vs languages.
T4c BPE merge curve: compression gain of the first 200 merges (a signature of unit size);
    manuscript vs languages.
T4d Repeated passages: longest exact word n-gram repeats across lines/pages (local alignment
    by shared 3-gram seeds), with counts vs the words-global surrogate.
Outputs: out/mi_decay.csv, out/compress.csv, out/bpe_curve.csv, out/repeats_passages.csv
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import math, csv, zlib, collections, random
import numpy as np
from harness import *

pages = structured()
V = flatten(pages)
langs = languages()
REF = ["Latin", "Italian", "German", "Czech", "Hungarian", "Hebrew", "Nahuatl-NT", "Malagasy"]


def mi(seq, d, alphabet):
    idx = {a: i for i, a in enumerate(alphabet)}; s = np.array([idx.get(x, -1) for x in seq]); s = s[s >= 0]
    a, b = s[:-d], s[d:]
    L = len(alphabet); j = np.zeros((L, L)); np.add.at(j, (a, b), 1); j /= j.sum()
    pa, pb = j.sum(1, keepdims=True), j.sum(0, keepdims=True)
    nz = j > 0
    return float((j[nz] * np.log2(j[nz] / (pa @ pb)[nz])).sum())


def units_stream(pgs):
    return [u for p in pgs for ln in p["lines"] for w in ln for u in tokenize_glyphs(w) + [" "]]


# ---- T4a
ds = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 25, 30, 40, 50, 60]
rows = []
base = units_stream(pages); alpha = [a for a, c in collections.Counter(base).most_common() if c >= 50]
rows.append(dict(series="Voynich", **{f"d{d}": mi(base, d, alpha) for d in ds}))
for name, fn in SURROGATES.items():
    if name == "pages": continue
    s = units_stream(fn(pages, seed=1)); rows.append(dict(series=f"surrogate:{name}", **{f"d{d}": mi(s, d, alpha) for d in ds}))
for lname in REF:
    ws = langs[lname][:len(V)]; s = [c for w in ws for c in list(w) + [" "]]
    al = [a for a, c in collections.Counter(s).most_common() if c >= 50]
    rows.append(dict(series=lname, **{f"d{d}": mi(s, d, al) for d in ds}))
with open("out/mi_decay.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)
for r in rows: print(f"{r['series']:28s}", " ".join(f"{r[f'd{d}']:.3f}" for d in [1, 2, 5, 10, 20, 50]))

# ---- T4b compressibility
def ratio(words):
    b = " ".join(words).encode(); return len(zlib.compress(b, 9)) / len(b)
crows = [dict(text="Voynich (all)", ratio=ratio(V))]
for I, nm in [("H", "Herbal"), ("B", "Bathing"), ("S", "Recipes"), ("P", "Pharma")]:
    crows.append(dict(text=f"Voynich {nm}", ratio=ratio(flatten(structured(section={I})))))
crows.append(dict(text="Voynich words-global shuffled", ratio=ratio(flatten(shuffle_words_global(pages)))))
crows.append(dict(text="Voynich chars-in-words shuffled", ratio=ratio(flatten(shuffle_chars_in_words(pages)))))
vals = []
for lname, ws in langs.items():
    if lname.endswith(".novowel"): continue
    r = ratio(ws[:len(V)]); vals.append(r); crows.append(dict(text=lname, ratio=r))
with open("out/compress.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=["text", "ratio"]); w.writeheader(); w.writerows(crows)
print("compress: Voynich", round(crows[0]["ratio"], 3), "langs", round(min(vals), 3), round(float(np.median(vals)), 3), round(max(vals), 3))

# ---- T4c BPE merge curve
def bpe_curve(words, n_merges=150):
    seqs = [list(w) for w in words]
    total0 = sum(len(s) for s in seqs); gains = []
    for k in range(n_merges):
        pc = collections.Counter()
        for s in seqs:
            for a, b in zip(s, s[1:]): pc[(a, b)] += 1
        if not pc: break
        (a, b), c = pc.most_common(1)[0]
        ab = a + b
        new = []
        for s in seqs:
            out = []; i = 0
            while i < len(s):
                if i < len(s) - 1 and s[i] == a and s[i + 1] == b: out.append(ab); i += 2
                else: out.append(s[i]); i += 1
            new.append(out)
        seqs = new
        gains.append(c / total0)
    return gains
brows = [dict(series="Voynich", **{f"m{i+1}": g for i, g in enumerate(bpe_curve(V))})]
for lname in REF:
    brows.append(dict(series=lname, **{f"m{i+1}": g for i, g in enumerate(bpe_curve(langs[lname][:len(V)]))}))
brows.append(dict(series="Voynich chars-in-words shuffled", **{f"m{i+1}": g for i, g in enumerate(bpe_curve(flatten(shuffle_chars_in_words(pages))))}))
keys = ["series"] + [f"m{i+1}" for i in range(150)]
with open("out/bpe_curve.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore"); w.writeheader(); w.writerows(brows)
for r in brows: print(f"{r['series']:28s}", " ".join(f"{r.get(f'm{i}', 0):.3f}" for i in [1, 2, 3, 5, 10, 20, 50, 100]))

# ---- T4d repeated passages (exact word n-grams, n>=3, across different lines)
def repeats(pgs, nmin=3):
    lines = [(p["page"], li, ln) for p in pgs for li, ln in enumerate(p["lines"])]
    seen = {}
    found = collections.Counter(); where = collections.defaultdict(set)
    for page, li, ln in lines:
        for n in range(nmin, min(9, len(ln)) + 1):
            for i in range(len(ln) - n + 1):
                g = tuple(ln[i:i + n]); where[g].add((page, li))
    rep = {g: w for g, w in where.items() if len(w) > 1}
    # keep maximal: drop g if a longer repeated gram contains it with the same locations count
    out = []
    for g, w in rep.items():
        out.append((len(g), len(w), len({p for p, _ in w}), " ".join(g), sorted({p for p, _ in w})[:6]))
    out.sort(key=lambda r: (-r[0], -r[1]))
    return out
real = repeats(pages); sur = repeats(shuffle_words_global(pages, seed=2))
with open("out/repeats_passages.csv", "w", newline="") as f:
    w = csv.writer(f); w.writerow(["n_words", "occurrences", "pages", "passage", "example_pages"])
    for r in real[:400]: w.writerow(r)
summ = {}
for n in range(3, 9):
    summ[n] = (sum(1 for r in real if r[0] == n), sum(1 for r in sur if r[0] == n))
print("repeated n-grams (real vs shuffled):", summ)
print("longest:", real[:8])
log("H-STRUCT-REPEAT", "T4d", "repeated 4-grams real/shuffled", summ[4][0], summ[4][1], "see note", "exact repeated word 4-grams across lines")