"""T17. Are list-like texts where the manuscript sits?
Latin texts of list, recipe, index and glossary genres (Apicius; Cato, De agri cultura; Pliny,
Natural History book 1 = the table of contents; Isidore, Etymologiae books 12 and 17; Hyginus,
Astronomica 2-3) against Latin prose (Pliny NH 2; Vulgate sample) and the manuscript, on the
word-level battery, at matched length (each text's length; the manuscript sampled as contiguous
page runs of that length, 5 draws).  Writes out/lists.csv; logs to results.csv.
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import csv, glob, random, re, sys
import numpy as np
from harness import structured, flatten, languages, log
import battery as B

LAT = os.path.join(REF, "latlib")
TEXTS = {
    "Apicius (recipes)": glob.glob(f"{LAT}/apicius/*.txt"),
    "Cato, De agri cultura": [f"{LAT}/cato/cato.agri.txt"],
    "Pliny NH 1 (index)": [f"{LAT}/pliny.nh1.txt"],
    "Isidore, Etym. 12 (animals)": [f"{LAT}/isidore/12.txt"],
    "Isidore, Etym. 17 (plants)": [f"{LAT}/isidore/17.txt"],
    "Hyginus, Astronomica 2-3": [f"{LAT}/hyginus/hyginus2.txt", f"{LAT}/hyginus/hyginus3.txt"],
    "Pliny NH 2 (prose)": [f"{LAT}/pliny.nh2.txt"],
}


def clean(paths):
    ws = []
    for p in paths:
        t = open(p, encoding="utf-8", errors="ignore").read().lower()
        t = re.sub(r"\[[^\]]*\]", " ", t); t = re.sub(r"[^a-z\s]", " ", t)
        ws += t.split()
    return ws


def paginate(words, line=9, lines=25, name="x"):
    pages = []; i = 0; k = 0
    while i < len(words):
        ls = []
        for _ in range(lines):
            ls.append(words[i:i + line]); i += line
            if i >= len(words): break
        pages.append(dict(page=f"{name}{k}", quire=None, hand=None, lang=None, I=None, lines=[l for l in ls if l])); k += 1
    return pages


STATS = ["ttr", "hapax", "h2_chars", "order_gain", "heldout_bigram_gain", "ff_rho", "ff_nn5", "rep3", "rep4", "twin10", "twin10_shuffled"]
rows = []
voy = structured(); vw = flatten(voy)
rng = random.Random(0)


def voy_sample(n, seed):
    """contiguous run of pages with about n words"""
    r = random.Random(seed); start = r.randrange(0, len(voy)); out = []; tot = 0; i = start
    while tot < n:
        p = voy[i % len(voy)]; out.append(p); tot += sum(len(l) for l in p["lines"]); i += 1
    return out


for name, paths in TEXTS.items():
    ws = clean(paths); n = len(ws)
    st = B.all_stats(paginate(ws, name=name[:3]), STATS); st.update(text=name, n_words=n); rows.append(st); print(st, flush=True)
    vs = [B.all_stats(voy_sample(n, s), STATS) for s in range(5)]
    m = {k: float(np.mean([v[k] for v in vs])) for k in STATS}; m.update(text=f"Voynich @ {n} words (mean of 5 runs)", n_words=n); rows.append(m); print(m, flush=True)
vul = languages()["Latin"][:12000]
st = B.all_stats(paginate(vul, name="vul"), STATS); st.update(text="Vulgate (prose), 12k", n_words=12000); rows.append(st); print(st)
with open("out/lists.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=["text", "n_words"] + STATS); w.writeheader()
    for r in rows: w.writerow({k: (round(v, 4) if isinstance(v, float) else v) for k, v in r.items()})
lists = [r for r in rows if not r["text"].startswith("Voynich") and "prose" not in r["text"] and "Vulgate" not in r["text"]]
log("H-TEXT-LIST", "T17-lists", "order gain (bits/word) of six Latin list-genre texts vs the manuscript at matched length",
    "; ".join(f"{r['text'].split(' (')[0].split(',')[0]} {r['order_gain']:.2f}" for r in lists),
    "manuscript at matched lengths: " + "; ".join(f"{r['order_gain']:.2f}" for r in rows if r["text"].startswith("Voynich")), "see note", "out/lists.csv")