"""T5. Page-to-page text similarity against the physical structure.
Similarity = cosine over the 400 commonest words (TF) and NCD (zlib). Tests whether pages
that share a leaf (recto/verso), a bifolio (sheet partners), or bound adjacency are more
similar than random pairs drawn from the same section and language (permutation z-scores).
Then: for the bathing quire (one scribe, one language) and the herbal-B pages, rank each
page's nearest neighbours and report which physical pairings the text supports.
Outputs: out/page_similarity.csv, out/page_pairs_tests.csv, out/page_neighbours.csv
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import csv, zlib, random, collections, itertools, re
import numpy as np
from harness import *

pages = [p for p in structured() if sum(len(l) for l in p["lines"]) >= 60]
C = corpus()
texts = {p["page"]: flatten([p]) for p in pages}
names = [p["page"] for p in pages]
vocab = [w for w, c in collections.Counter(w for t in texts.values() for w in t).most_common(400)]
vi = {w: i for i, w in enumerate(vocab)}
M = np.zeros((len(names), len(vocab)))
for i, n in enumerate(names):
    for w in texts[n]:
        if w in vi: M[i, vi[w]] += 1
M = M / np.maximum(M.sum(1, keepdims=True), 1)
Mn = M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
COS = Mn @ Mn.T


def ncd(a, b):
    za, zb = len(zlib.compress(a, 9)), len(zlib.compress(b, 9)); zab = len(zlib.compress(a + b"\n" + b, 9))
    return (zab - min(za, zb)) / max(za, zb)


blobs = {n: " ".join(texts[n]).encode() for n in names}
NCD = np.zeros((len(names), len(names)))
for i, a in enumerate(names):
    for j, b in enumerate(names):
        if j > i: NCD[i, j] = NCD[j, i] = ncd(blobs[a], blobs[b])
with open("out/page_similarity.csv", "w", newline="") as f:
    w = csv.writer(f); w.writerow(["page_a", "page_b", "cosine", "ncd"])
    for i, j in itertools.combinations(range(len(names)), 2): w.writerow([names[i], names[j], round(COS[i, j], 4), round(NCD[i, j], 4)])

# physical relations
pv = C.pagevars
def folio(p): return int(re.match(r"f(\d+)", p).group(1))
def leafmates(a, b): return folio(a) == folio(b)
def bifolio_mates(a, b):
    va, vb = pv[a], pv[b]
    return va.get("Q") == vb.get("Q") and va.get("B") == vb.get("B") and folio(a) != folio(b)
order = {n: i for i, n in enumerate(C.pages)}
def adjacent(a, b): return abs(order[a] - order[b]) == 1
grp = {n: (pv[n].get("I"), pv[n].get("L")) for n in names}

rows = []
rng = random.Random(0)
for relname, rel in [("same leaf (r/v)", leafmates), ("same bifolio", bifolio_mates), ("bound-adjacent", adjacent)]:
    for measure, S, sign in [("cosine", COS, 1), ("ncd", NCD, -1)]:
        pairs = [(i, j) for i, j in itertools.combinations(range(len(names)), 2) if rel(names[i], names[j]) and grp[names[i]] == grp[names[j]]]
        if not pairs: continue
        obs = np.mean([S[i, j] for i, j in pairs])
        # null: for each pair, replace j by a random page of the same (section, language) group
        nulls = []
        for k in range(300):
            vals = []
            for i, j in pairs:
                cands = [x for x in range(len(names)) if x != i and grp[names[x]] == grp[names[i]]]
                vals.append(S[i, rng.choice(cands)])
            nulls.append(np.mean(vals))
        z = sign * (obs - np.mean(nulls)) / (np.std(nulls) + 1e-9)
        p = np.mean([(sign * (nl - obs)) >= 0 for nl in nulls])
        rows.append(dict(relation=relname, measure=measure, n_pairs=len(pairs), observed=round(float(obs), 4), null_mean=round(float(np.mean(nulls)), 4), z=round(float(z), 2), p_perm=round(float(p), 3)))
        print(rows[-1])
with open("out/page_pairs_tests.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)

# nearest neighbours for bathing quire and herbal-B pages
out = []
for n in names:
    i = names[i_ := names.index(n)]
    if grp[n] in [("B", "B"), ("H", "B")]:
        cands = [(COS[i_, j], names[j]) for j in range(len(names)) if j != i_ and grp[names[j]] == grp[n]]
        cands.sort(reverse=True)
        out.append(dict(page=n, group=grp[n][0], bifolio=pv[n].get("B"), nn1=cands[0][1], s1=round(cands[0][0], 3), nn2=cands[1][1], s2=round(cands[1][0], 3), nn3=cands[2][1], s3=round(cands[2][0], 3),
                        leafmate_rank=next((k + 1 for k, (s, m) in enumerate(cands) if leafmates(m, n)), None),
                        bifolio_mate_rank=next((k + 1 for k, (s, m) in enumerate(cands) if bifolio_mates(m, n)), None)))
with open("out/page_neighbours.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=out[0].keys()); w.writeheader(); w.writerows(out)
for r in out: print(r)