"""T23 (exploratory). Cicco Simonetta's rules for breaking a cipher (Pavia, 1474), applied to the
manuscript as a Milanese secretary would have: (1) how many distinct signs end words, and how much of
the text the five commonest endings cover (his rule: Italian has five or fewer endings, Latin more);
(2) a sign always followed by the same sign is q with its u; (3) the two-sign words.
A key with vowel homophones spreads the endings over more signs; plain text concentrates them.
Writes out/simonetta.csv.
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import collections, csv
from harness import structured, flatten, languages
from corpus import tokenize_glyphs

voy = flatten(structured()); L = languages(); rows = []


def endings(name, words, parse):
    c = collections.Counter(parse(w)[-1] for w in words if parse(w)); n = sum(c.values()); top = c.most_common()
    rows.append(dict(text=name, endings_above_1pct=len([k for k, v in top if v / n >= 0.01]), top5_share=round(sum(v for k, v in top[:5]) / n, 3),
                     top_endings=" ".join(f"{k}:{v / n:.2f}" for k, v in top[:6])))


endings("Voynich", voy, tokenize_glyphs)
for lang in ("Latin", "Italian"): endings(lang, L[lang][:34000], list)
nxt = collections.defaultdict(collections.Counter)
for w in voy:
    u = tokenize_glyphs(w)
    for a, b in zip(u, u[1:]): nxt[a][b] += 1
q = nxt["q"]; rows[0]["q_followed_by_o"] = round(q["o"] / sum(q.values()), 3)
two = collections.Counter(w for w in voy if len(tokenize_glyphs(w)) == 2)
rows[0]["two_unit_words"] = " ".join(f"{w}:{c}" for w, c in two.most_common(6))
with open("out/simonetta.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=sorted({k for r in rows for k in r}, key=lambda k: k != "text")); w.writeheader(); w.writerows(rows)
for r in rows: print(r)