"""A9. Decipherment experiments, reported honestly.

E1  Monoalphabetic substitution: for each reference language, find the
    one-to-one glyph->letter mapping that maximises the bigram log-likelihood of
    the mapped Voynich text under that language's bigram model (hill-climbing
    with restarts). Compare the attained fit with (a) the language's own text,
    (b) a random-substitution encipherment of the language's text solved by the
    same method (control: the solver should recover ~the plaintext score), and
    (c) the Voynich text with a random mapping.
E2  Same, against vowel-less (abjad-style) versions of the languages.
E3  Verbose-cipher collapse: split each Voynich word into slot fillers and treat
    each filler as one symbol; recompute conditional entropy.
E4  Auto-copy test: share of words with an exact or edit-1 twin earlier on the
    same page, vs. the same words in shuffled order within the page.
Outputs: out/tables/decipher_E1.csv, E2.csv, E3.csv, E4.csv
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import math, random, collections, csv, re, os
import numpy as np
from corpus import Corpus, tokenize_glyphs
import refcorpora

random.seed(7); np.random.seed(7)
C = Corpus()
V = C.running_text()
N = len(V)
refs = refcorpora.all_refs(N)

# ---- symbol streams
def glyph_stream(words):
    out = []
    for w in words:
        out += tokenize_glyphs(w) + [" "]
    return out


def letter_stream(words):
    out = []
    for w in words:
        out += list(w) + [" "]
    return out


def devowel(words):
    return [re.sub(r"[aeiouyäöü]", "", w) for w in words if re.sub(r"[aeiouyäöü]", "", w)]


def bigram_logp(stream, alphabet, k=0.5):
    idx = {a: i for i, a in enumerate(alphabet)}
    M = np.full((len(alphabet), len(alphabet)), k)
    for a, b in zip(stream, stream[1:]):
        if a in idx and b in idx: M[idx[a], idx[b]] += 1
    P = M / M.sum(axis=1, keepdims=True)
    return np.log(P), idx


def count_matrix(stream, alphabet):
    idx = {a: i for i, a in enumerate(alphabet)}
    M = np.zeros((len(alphabet), len(alphabet)))
    for a, b in zip(stream, stream[1:]):
        if a in idx and b in idx: M[idx[a], idx[b]] += 1
    return M


def solve(Cm, logP, n_src, n_tgt, restarts=12, iters=6000):
    """One-to-one assignment of the n_src source symbols (space excluded, index
    n_src is the fixed space) onto the n_tgt target letters (space at index n_tgt),
    maximising sum Cm[i,j]*logP[m(i),m(j)]. n_src must be <= n_tgt."""
    assert n_src <= n_tgt
    best, best_m = -1e18, None
    for r in range(restarts):
        m = np.append(np.random.permutation(n_tgt)[:n_src], n_tgt)   # last = space -> space
        def score(m): return (Cm * logP[np.ix_(m, m)]).sum()
        s = score(m)
        for it in range(iters):
            m2 = m.copy()
            i = random.randrange(n_src)
            if random.random() < 0.6:
                j = random.randrange(n_src); m2[i], m2[j] = m2[j], m2[i]
            else:
                t = random.randrange(n_tgt)
                if t in m2[:n_src]: continue
                m2[i] = t
            s2 = score(m2)
            if s2 >= s: m, s = m2, s2
        if s > best: best, best_m = s, m
    return best, best_m


def experiment(name, ref_words, voy_stream, voy_alpha_all, letters_fn, rows):
    ref_stream = letters_fn(ref_words)
    ref_alpha = [a for a, c in collections.Counter(ref_stream).most_common() if c >= 20 and a != " "] + [" "]
    n_tgt = len(ref_alpha) - 1
    voy_alpha = voy_alpha_all[:n_tgt] + [" "]
    n_src = len(voy_alpha) - 1
    logP, idx = bigram_logp(ref_stream, ref_alpha)
    # (a) own text
    Cr = count_matrix(ref_stream, ref_alpha)
    own = (Cr * logP).sum() / Cr.sum()
    # (b) control: random encipherment of ref text, then solve
    perm = np.append(np.random.permutation(n_tgt), n_tgt)
    enc_stream = [ref_alpha[perm[idx[a]]] if a in idx else a for a in ref_stream]
    Ce = count_matrix(enc_stream, ref_alpha)
    sc, _ = solve(Ce, logP, n_tgt, n_tgt)
    ctrl = sc / Ce.sum()
    # (c) Voynich solved
    Cv = count_matrix(voy_stream, voy_alpha)
    sv, mv = solve(Cv, logP, n_src, n_tgt)
    voy = sv / Cv.sum()
    # (d) Voynich random mapping
    mr = np.append(np.random.permutation(n_tgt)[:n_src], n_tgt)
    rnd = (Cv * logP[np.ix_(mr, mr)]).sum() / Cv.sum()
    mapping = {voy_alpha[i]: ref_alpha[mv[i]] for i in range(n_src)}
    sample = "".join(mapping.get(s, "?") if s != " " else " " for s in voy_stream[:120])
    rows.append(dict(experiment=name, own_text_logp=own, control_recovered_logp=ctrl, voynich_best_logp=voy, voynich_random_logp=rnd,
                     voy_alphabet=len(voy_alpha), ref_alphabet=len(ref_alpha), mapping=" ".join(f"{k}>{v}" for k, v in mapping.items()), sample=sample.strip()))
    print(f"{name:32s} own {own:6.3f}  control {ctrl:6.3f}  voynich {voy:6.3f}  random {rnd:6.3f}")
    print("   sample:", sample[:110])


vs = glyph_stream(V)
valpha = [a for a, c in collections.Counter(vs).most_common() if c >= 100 and a != " "]
rows1, rows2 = [], []
for lang, words in refs.items():
    experiment(f"{lang}", words, vs, valpha, letter_stream, rows1)
for lang, words in refs.items():
    experiment(f"{lang}, vowel-less", devowel(words), vs, valpha, letter_stream, rows2)
for fn, rows in (("out/tables/decipher_E1.csv", rows1), ("out/tables/decipher_E2.csv", rows2)):
    with open(fn, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)

# ---- E3 verbose-cipher collapse
PRE = r"(qo|o|d|s|ch|sh|y|a|l|r|)"; GALL = r"(cth|ckh|cph|cfh|t|k|p|f|)"; MID = r"(ch|sh|e{1,3}|eo|o|a|)"; X = r"(d|l|r|s|)"
SUF = r"(y|dy|ain|aiin|aiiin|air|aiir|oiin|an|ar|al|or|ol|am|ey|eey|edy|eedy|s|l|r|n|m|d|o|a|e|)"
one = f"{PRE}{GALL}{MID}{X}{SUF}"
G1 = re.compile(f"^{one}$"); G2 = re.compile(f"^{one}{one}$")


def slots(w):
    m = G1.match(w)
    if m: return [x for x in m.groups() if x]
    m = G2.match(w)
    if m: return [x for x in m.groups() if x]
    return None


def entropy2(seq):
    uni = collections.Counter(seq); n = len(seq)
    h1 = -sum(c / n * math.log2(c / n) for c in uni.values())
    bi = collections.Counter(zip(seq, seq[1:])); nb = n - 1
    h2 = -sum(c / nb * math.log2(c / uni[a]) for (a, b), c in bi.items())
    return len(uni), h1, h2


tok, covered = [], 0
for w in V:
    s = slots(w)
    if s: tok += s + [" "]; covered += 1
    else: tok += ["#", " "]
k, h1, h2 = entropy2(tok)
k0, h10, h20 = entropy2(vs)
rows3 = [dict(stream="EVA glyph units", symbols=k0, h1=h10, h2=h20), dict(stream="slot fillers as symbols", symbols=k, h1=h1, h2=h2, coverage=covered / N)]
# word-level entropy for reference
wk, wh1, wh2 = entropy2(V)
rows3.append(dict(stream="whole words as symbols", symbols=wk, h1=wh1, h2=wh2))
for lang, words in refs.items():
    k, h1, h2 = entropy2(letter_stream(words)); rows3.append(dict(stream=f"{lang} letters", symbols=k, h1=h1, h2=h2))
    k, h1, h2 = entropy2(words); rows3.append(dict(stream=f"{lang} words", symbols=k, h1=h1, h2=h2))
with open("out/tables/decipher_E3.csv", "w", newline="") as f:
    keys = ["stream", "symbols", "h1", "h2", "coverage"]
    w = csv.DictWriter(f, fieldnames=keys); w.writeheader()
    for r in rows3: w.writerow({k: r.get(k, "") for k in keys})
for r in rows3: print(r)

# ---- E4 auto-copy: earlier twin on the same page
def lev1(a, b):
    if a == b or abs(len(a) - len(b)) > 1: return False
    if len(a) == len(b): return sum(x != y for x, y in zip(a, b)) == 1
    s, t = (a, b) if len(a) < len(b) else (b, a)
    i = 0
    while i < len(s) and s[i] == t[i]: i += 1
    return s[i:] == t[i + 1:]


def twin_rate(pages_words):
    tot = exact = near = 0
    for ws in pages_words:
        seen = []
        for w in ws:
            tot += 1
            if w in seen: exact += 1
            elif any(lev1(w, s) for s in seen[-60:]): near += 1
            seen.append(w)
    return exact / tot, near / tot


pages = [C.running_text(pages={p}) for p in C.pages]
pages = [ws for ws in pages if len(ws) >= 30]
e, n = twin_rate(pages)
sh = [random.sample(ws, len(ws)) for ws in pages]
e2, n2 = twin_rate(sh)
# reference: Latin split into pseudo-pages of 85 words
lat = refs["Latin (Vulgate)"]; lp = [lat[i:i + 85] for i in range(0, len(lat), 85)]
e3, n3 = twin_rate(lp)
rows4 = [dict(corpus="Voynich, page order", earlier_exact_twin=e, earlier_edit1_twin=n), dict(corpus="Voynich, words shuffled within page", earlier_exact_twin=e2, earlier_edit1_twin=n2), dict(corpus="Latin (Vulgate), 85-word pages", earlier_exact_twin=e3, earlier_edit1_twin=n3)]
with open("out/tables/decipher_E4.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows4[0].keys()); w.writeheader(); w.writerows(rows4)
for r in rows4: print(r)


# ---- E1b: dictionary hit rate of the best substitution (added)
def dict_hits(rows, refs, letters_fn, voy_words):
    for r in rows:
        lang = r["experiment"].replace(", vowel-less", "")
        words = refs[lang]
        if "vowel-less" in r["experiment"]: words = devowel(words)
        vocab = set(words)
        mapping = dict(p.split(">") for p in r["mapping"].split(" "))
        hits = tot = 0
        for w in voy_words:
            g = tokenize_glyphs(w)
            if any(x not in mapping for x in g): continue
            tot += 1; hits += "".join(mapping[x] for x in g) in vocab
        r["dictionary_hit_rate"] = hits / tot
        # baseline: random one-to-one mappings
        keys = list(mapping.keys()); vals = list(mapping.values()); base = []
        for _ in range(5):
            random.shuffle(vals); rm = dict(zip(keys, vals)); h = t = 0
            for w in voy_words:
                g = tokenize_glyphs(w)
                if any(x not in rm for x in g): continue
                t += 1; h += "".join(rm[x] for x in g) in vocab
            base.append(h / t)
        r["dictionary_hit_rate_random_mapping"] = sum(base) / len(base)
        print(f"{r['experiment']:36s} mapped words found in that language's vocabulary: {hits/tot:.3f}  (random mapping {r['dictionary_hit_rate_random_mapping']:.3f})")


dict_hits(rows1, refs, letter_stream, V); dict_hits(rows2, refs, letter_stream, V)
for fn, rows in (("out/tables/decipher_E1.csv", rows1), ("out/tables/decipher_E2.csv", rows2)):
    with open(fn, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)

# ---- E4b: near-twin rate by distance window
def near_by_window(pages_words, win):
    tot = hit = 0
    for ws in pages_words:
        for i, w in enumerate(ws):
            tot += 1
            hit += any(w == ws[j] or lev1(w, ws[j]) for j in range(max(0, i - win), i))
    return hit / tot
rows5 = []
for win in (1, 2, 5, 10, 20, 50):
    rows5.append(dict(window=win, voynich=near_by_window(pages, win), voynich_shuffled=near_by_window(sh, win), latin=near_by_window(lp, win)))
    print(rows5[-1])
with open("out/tables/decipher_E4b.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows5[0].keys()); w.writeheader(); w.writerows(rows5)