"""T11. Periodicity / transposition battery (after the Zodiac-340 diagnostics) and isomorphs.

Streams: glyph units of the running text with word spaces (U1s) and without (U1).
(a) periodic coincidence kappa(p) = P(S[i] == S[i+p]) for p = 1..80, vs the same on surrogates
    (chars shuffled within words; words shuffled within lines; lines shuffled globally).
(b) repeated-bigram count when the stream is read at period p (columns of width p), p = 2..60,
    vs the within-word shuffle: a columnar transposition shows a peak at its period.
(c) line-position kappa: share of positions where consecutive lines of a paragraph carry the same
    unit, vs random line pairs of the same lengths (detects line-keyed alphabets / copy-from-above).
(d) isomorphs: windows of 6-9 units with the same repetition pattern but no unit in common,
    counted at matched line positions vs anywhere, vs the within-line word shuffle.
Hypothesis H-CIPHER-TRANSPOSITION predicts a period in (a)/(b) absent from every surrogate;
H-CIPHER-POLYALPHABETIC predicts a lift in (c) or (d) absent from surrogates.
Writes out/periodicity.csv, out/isomorphs.csv; logs to results.csv.
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, csv, random, sys
import numpy as np
pass
from corpus import tokenize_glyphs
from harness import structured, SURROGATES, log

pages = structured()


def streams(pgs, spaces):
    S = []; lines = []
    for p in pgs:
        for ln in p["lines"]:
            L = []
            for w in ln:
                L += tokenize_glyphs(w)
                if spaces: L.append(" ")
            lines.append(L); S += L
    return S, lines


def encode(S):
    idx = {}; return np.array([idx.setdefault(u, len(idx)) for u in S], dtype=np.int32)


def kappa(a, pmax=80):
    return [float((a[:-p] == a[p:]).mean()) for p in range(1, pmax + 1)]


def period_bigrams(a, pmax=60):
    out = []
    for p in range(2, pmax + 1):
        cols = [a[k::p] for k in range(p)]
        t = np.concatenate(cols)
        pairs = t[:-1].astype(np.int64) * 100000 + t[1:]
        _, c = np.unique(pairs, return_counts=True)
        out.append(int((c[c > 1] - 1).sum()))
    return out


def line_kappa(lines, rng):
    same = tot = 0; rs = rt = 0
    for a, b in zip(lines, lines[1:]):
        n = min(len(a), len(b))
        if n < 5: continue
        same += sum(x == y for x, y in zip(a, b)); tot += n
    L = [l for l in lines if len(l) >= 5]
    for _ in range(len(lines)):
        a, b = rng.sample(L, 2); n = min(len(a), len(b))
        rs += sum(x == y for x, y in zip(a, b)); rt += n
    return same / tot, rs / rt


def isomorphs(lines, wmin=6, wmax=9):
    """count pairs of windows with identical repetition pattern and disjoint unit sets; split by
    whether the two windows start at the same within-line position."""
    sig = collections.defaultdict(list)
    for li, L in enumerate(lines):
        for w in range(wmin, wmax + 1):
            for i in range(len(L) - w + 1):
                win = L[i:i + w]
                if len(set(win)) == w: continue  # no repeats: pattern carries no information
                first = {}; pat = tuple(first.setdefault(u, len(first)) for u in win)
                sig[(w, pat)].append((li, i, frozenset(win)))
    same_pos = any_pos = 0
    for k, occ in sig.items():
        if len(occ) < 2 or len(occ) > 400: continue
        for x in range(len(occ)):
            for y in range(x + 1, len(occ)):
                if occ[x][2].isdisjoint(occ[y][2]):
                    any_pos += 1
                    if occ[x][1] == occ[y][1]: same_pos += 1
    return same_pos, any_pos


rng = random.Random(0)
variants = {"text": pages}
for name in ("chars-in-words", "words-in-lines", "lines-global"):
    variants[name] = SURROGATES[name](pages, seed=1)

rows = []; iso_rows = []
for vname, pg in variants.items():
    for spaces in (True, False):
        S, lines = streams(pg, spaces); a = encode(S)
        k = kappa(a); pb = period_bigrams(a)
        lk, lk_rand = line_kappa(lines, random.Random(2))
        rows.append(dict(variant=vname, spaces=spaces, line_kappa=round(lk, 4), line_kappa_random=round(lk_rand, 4),
                         kappa_max_p=int(np.argmax(k) + 1), kappa_max=round(max(k), 4), kappa_mean=round(float(np.mean(k)), 4),
                         **{f"kappa_{p}": round(k[p - 1], 4) for p in (1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 60, 80)},
                         pb_argmax=int(np.argmax(pb) + 2), pb_max=max(pb), pb_median=int(np.median(pb))))
        print(vname, spaces, "line_kappa %.4f (random %.4f)  kappa peak p=%d %.4f  period-bigram peak p=%d %d (median %d)" % (lk, lk_rand, rows[-1]["kappa_max_p"], max(k), rows[-1]["pb_argmax"], max(pb), rows[-1]["pb_median"]), flush=True)
    S, lines = streams(pg, False)
    sp, ap = isomorphs(lines)
    iso_rows.append(dict(variant=vname, same_position=sp, any_position=ap, share_same=round(sp / ap, 4) if ap else 0))
    print(vname, "isomorphs disjoint: same line position", sp, "any", ap, flush=True)

with open("out/periodicity.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)
with open("out/isomorphs.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=iso_rows[0].keys()); w.writeheader(); w.writerows(iso_rows)
t = [r for r in rows if r["variant"] == "text" and r["spaces"]][0]; s = [r for r in rows if r["variant"] == "chars-in-words" and r["spaces"]][0]
log("H-CIPHER-TRANSPOSITION", "T11-periodicity", "period-bigram peak (period, count) in text vs within-word shuffle", f"p={t['pb_argmax']} {t['pb_max']} (median {t['pb_median']})", f"shuffle p={s['pb_argmax']} {s['pb_max']} (median {s['pb_median']})", "see note", "out/periodicity.csv")
log("H-CIPHER-POLYALPHABETIC", "T11-line-kappa", "coincidence between consecutive lines at same position", t["line_kappa"], f"random line pairs {t['line_kappa_random']}; words-in-lines shuffle {[r for r in rows if r['variant']=='words-in-lines' and r['spaces']][0]['line_kappa']}", "see note", "out/periodicity.csv")
log("H-CIPHER-POLYALPHABETIC", "T11-isomorphs", "glyph-disjoint isomorph pairs at same line position / anywhere", f"{iso_rows[0]['same_position']}/{iso_rows[0]['any_position']}", f"words-in-lines shuffle {iso_rows[2]['same_position']}/{iso_rows[2]['any_position']}", "see note", "out/isomorphs.csv")