"""A6. Labels vs running text; A7. repetition and near-repetition; A8. agreement
between two independent transliterations (ZL vs Takahashi).
Outputs: out/tables/labels.csv, out/tables/repeats.csv, out/tables/agreement.csv,
         out/figs/repeats.pdf
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, csv, os, random
import matplotlib; matplotlib.use("Agg")
import matplotlib.pyplot as plt
from corpus import Corpus, IT, section_name
import refcorpora, ivtff

C = Corpus()
run = C.running_text(); run_types = collections.Counter(run)

# ---------- A6 labels
rows = []
for lt, desc in [("Lz", "zodiac figure labels"), ("Ln", "nymph labels (biological)"), ("Lf", "plant-part labels (pharmaceutical)"),
                 ("Lc", "container labels (pharmaceutical)"), ("Ls", "star labels"), ("Lt", "tube/tub labels"), ("L0", "other labels"), ("La", "astronomical labels"), ("Lp", "herb labels")]:
    ws = C.labels(ltypes={lt})
    if not ws: continue
    ty = collections.Counter(ws)
    in_text = sum(1 for w in ty if w in run_types) / len(ty)
    in_text_tok = sum(c for w, c in ty.items() if w in run_types) / len(ws)
    rows.append(dict(type=lt, description=desc, tokens=len(ws), types=len(ty), repeated_types=sum(1 for c in ty.values() if c > 1),
                     q_initial=sum(w.startswith("q") for w in ws) / len(ws), o_initial=sum(w.startswith("o") for w in ws) / len(ws),
                     y_final=sum(w.endswith("y") for w in ws) / len(ws), mean_len=sum(map(len, ws)) / len(ws),
                     types_also_in_running_text=in_text, tokens_also_in_running_text=in_text_tok,
                     top=" ".join(f"{w}:{c}" for w, c in ty.most_common(6))))
allrun = dict(type="P", description="running text (reference)", tokens=len(run), types=len(run_types), repeated_types=sum(1 for c in run_types.values() if c > 1),
              q_initial=sum(w.startswith("q") for w in run) / len(run), o_initial=sum(w.startswith("o") for w in run) / len(run),
              y_final=sum(w.endswith("y") for w in run) / len(run), mean_len=sum(map(len, run)) / len(run), types_also_in_running_text=1, tokens_also_in_running_text=1, top="")
rows.append(allrun)
with open("out/tables/labels.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)
for r in rows: print({k: (round(v, 3) if isinstance(v, float) else v) for k, v in r.items() if k != "top"}, r["top"])

# zodiac labels: how many recur across different signs?
zl = collections.defaultdict(set)
for l in C.loci_where(ltypes={"Lz"}):
    for w in ivtff.words(l.text):
        if "?" not in w: zl[w].add(l.page)
multi = {w: p for w, p in zl.items() if len(p) > 1}
print("zodiac label types:", len(zl), "appearing on >1 zodiac page:", len(multi), sorted(multi.items(), key=lambda kv: -len(kv[1]))[:10])
# month names (extraneous) pages
print("pages flagged with month names ($X contains M):", [p for p, v in C.pagevars.items() if "M" in v.get("X", "")])

# ---------- A7 repetition: exact adjacent repeats and edit-distance-1 neighbours
def lev1(a, b):
    if a == b: return False
    if abs(len(a) - len(b)) > 1: return False
    if len(a) == len(b):
        return sum(x != y for x, y in zip(a, b)) == 1
    s, t = (a, b) if len(a) < len(b) else (b, a)
    i = 0
    while i < len(s) and s[i] == t[i]: i += 1
    return s[i:] == t[i + 1:]


def rep_stats(ws):
    n = len(ws) - 1
    exact = sum(ws[i] == ws[i + 1] for i in range(n)) / n
    near = sum(lev1(ws[i], ws[i + 1]) for i in range(n)) / n
    # within-line repeats handled on word stream; also a window-of-5 near-repeat rate
    win = sum(any(ws[i] == ws[j] or lev1(ws[i], ws[j]) for j in range(i + 1, min(i + 6, len(ws)))) for i in range(len(ws))) / len(ws)
    return exact, near, win


refs = refcorpora.all_refs(len(run))
rep_rows = []
for name, ws in [("Voynich (running text)", run), ("Voynich A", C.running_text(L={"A"})), ("Voynich B", C.running_text(L={"B"}))] + list(refs.items()):
    e, nr, win = rep_stats(ws)
    rep_rows.append(dict(corpus=name, adjacent_exact=e, adjacent_edit1=nr, window5_same_or_edit1=win))
# shuffled Voynich baseline
random.seed(1); sh = run[:]; random.shuffle(sh)
e, nr, win = rep_stats(sh); rep_rows.append(dict(corpus="Voynich, words shuffled", adjacent_exact=e, adjacent_edit1=nr, window5_same_or_edit1=win))
with open("out/tables/repeats.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rep_rows[0].keys()); w.writeheader(); w.writerows(rep_rows)
for r in rep_rows: print({k: (round(v, 4) if isinstance(v, float) else v) for k, v in r.items()})
fig, ax = plt.subplots(figsize=(5.6, 2.8))
names = [r["corpus"] for r in rep_rows]
import numpy as np
x = np.arange(len(names))
ax.bar(x - 0.2, [r["adjacent_exact"] * 100 for r in rep_rows], 0.4, color="#222222", label="identical to previous word")
ax.bar(x + 0.2, [r["adjacent_edit1"] * 100 for r in rep_rows], 0.4, color="#aaaaaa", label="one edit from previous word")
ax.set_xticks(x); ax.set_xticklabels(names, rotation=30, ha="right", fontsize=7); ax.set_ylabel("% of word tokens")
ax.legend(fontsize=7, frameon=False); ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout(); fig.savefig("out/figs/repeats.pdf")

# ---------- A8 agreement between ZL and IT (Takahashi) on shared loci
T = Corpus(IT)
tl = {(l.page, l.num): l for l in T.loci}
same_tok = diff_tok = 0; same_loc = 0; nloc = 0; wc_mismatch = 0
for l in C.loci:
    k = (l.page, l.num)
    if k not in tl: continue
    a = ivtff.words(l.text); b = ivtff.words(tl[k].text)
    nloc += 1
    if a == b: same_loc += 1
    if len(a) != len(b): wc_mismatch += 1
    for x, y in zip(a, b):
        if x == y: same_tok += 1
        else: diff_tok += 1
ag = dict(shared_loci=nloc, loci_identical=same_loc / nloc, loci_word_count_differs=wc_mismatch / nloc, aligned_tokens_identical=same_tok / (same_tok + diff_tok))
print(ag)
with open("out/tables/agreement.csv", "w", newline="") as f:
    w = csv.writer(f); w.writerow(["measure", "value"])
    for k, v in ag.items(): w.writerow([k, v])