"""T15. Generator competition / procedure fingerprinting.
For each simulator in generators.py: (1) fit its free parameters on the FIT statistics of the
training quires (grid search, distance in units of the manuscript's values); (2) with the fitted
parameters, generate `n_sims` corpora into the manuscript's layout; (3) compute the HELD-OUT
battery on each and on the manuscript; (4) report, per generator x statistic, the manuscript's
z-score against the simulated distribution (|z| > 2 = the generator fails that statistic).
Writes out/generators_fit.csv, out/generators_scores.csv, out/generators_grid.csv.
Usage: python3 t_generators.py [n_sims]
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import csv, itertools, json, sys, time
import numpy as np
from harness import structured, split, flatten, log
import generators as G, battery as B

n_sims = int(sys.argv[1]) if len(sys.argv) > 1 else 20
pages = structured(); tr, te = split(pages); layout = G.layout_of(pages); trw = flatten(tr)
target_fit = {k: fn(tr) for k, fn in B.FIT.items()}
target_all = B.all_stats(pages)
print("manuscript:", {k: round(v, 3) for k, v in target_all.items()}, flush=True)

GRIDS = {
    "slot": [dict()],
    "autocopy": [dict(window=w, p_mod=m, p_new=n) for w in (6, 12, 24) for m in (0.5, 0.8, 0.95) for n in (0.02, 0.1, 0.25)],
    "grille": [dict(rows=r, refill=f, p_skip=s) for r in (40, 120, 400) for f in (30, 120, 500) for s in (0.1, 0.3)],
    "naibbe": [dict(k=k, p_bigram=p, n_bigrams=nb) for k in (10, 25, 60) for p in (0.3, 0.6) for nb in (60, 200)],
    "naibbe_novowel": [dict(k=k, p_bigram=p, n_bigrams=nb) for k in (10, 25, 60) for p in (0.3, 0.6) for nb in (60, 200)],
    "homonull": [dict(k=k, p_null=p) for k in (2, 4) for p in (0.0, 0.1, 0.3)],
    "wordletter": [dict(k=k) for k in (10, 50, 200, 400)],
}


def fit_dist(st):
    return sum(((st[k] - target_fit[k]) / (abs(target_fit[k]) + 1e-9)) ** 2 for k in target_fit) ** 0.5


grid_rows = []; fit_rows = []; score_rows = []
for name, gen in G.GENERATORS.items():
    best = None
    for params in GRIDS[name]:
        pg = gen(trw, G.layout_of(tr), seed=0, **params)
        st = {k: fn(pg) for k, fn in B.FIT.items()}; d = fit_dist(st)
        grid_rows.append(dict(generator=name, params=json.dumps(params), distance=round(d, 3), **{k: round(v, 3) for k, v in st.items()}))
        if best is None or d < best[0]: best = (d, params, st)
    d, params, st = best
    fit_rows.append(dict(generator=name, params=json.dumps(params), distance=round(d, 3), **{k: round(v, 3) for k, v in st.items()}))
    print(f"fitted {name}: {params} distance {d:.3f} {st}", flush=True)
    sims = []
    t0 = time.time()
    for s in range(n_sims):
        pg = gen(trw, layout, seed=100 + s, **params); sims.append(B.all_stats(pg))
    for k in target_all:
        vals = np.array([x[k] for x in sims]); mu, sd = vals.mean(), vals.std() + 1e-9
        z = (target_all[k] - mu) / sd
        score_rows.append(dict(generator=name, statistic=k, manuscript=round(target_all[k], 4), sim_mean=round(mu, 4), sim_sd=round(sd, 4), z=round(z, 2), fails=abs(z) > 2))
    print(f"  scored {name} in {time.time()-t0:.0f}s: " + " ".join(f"{r['statistic']}={r['z']:+.1f}" for r in score_rows if r['generator'] == name), flush=True)

for fn, rows in (("out/generators_grid.csv", grid_rows), ("out/generators_fit.csv", fit_rows), ("out/generators_scores.csv", score_rows)):
    with open(fn, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)
held = [k for k in B.HELD]
summary = {}
for name in G.GENERATORS:
    fails = [r["statistic"] for r in score_rows if r["generator"] == name and r["fails"] and r["statistic"] in held]
    summary[name] = f"{len(fails)}/{len(held)} held-out statistics failed ({', '.join(fails)})"
    print(name, summary[name])
# ---- which generator is the manuscript nearest to, and can the battery tell the generators apart?
held_z = {}  # generator -> list of z-vectors of its own simulations against the pooled sd
allsims = {}
import json as _j
pooled_sd = {k: np.std([r["sim_mean"] for r in score_rows if r["statistic"] == k] + [target_all[k]]) + 1e-9 for k in held}
cent = {name: {r["statistic"]: r["sim_mean"] for r in score_rows if r["generator"] == name} for name in G.GENERATORS}
def dist(stats, name): return sum(((stats[k] - cent[name][k]) / pooled_sd[k]) ** 2 for k in held) ** 0.5
near = sorted((dist(target_all, n), n) for n in G.GENERATORS)
print("manuscript nearest generators (standardised distance):", [(n, round(d, 2)) for d, n in near])
log("H-PROCESS-GENERATORS", "T15-nearest", "nearest generator to the manuscript in standardised held-out space", near[0][1], "; ".join(f"{n} {d:.2f}" for d, n in near), "see note", "distance in pooled-sd units over held-out statistics")
log("H-PROCESS-GENERATORS", "T15-generators", "held-out statistics failed per generator (|z|>2)", "; ".join(f"{k}: {v.split(' ')[0]}" for k, v in summary.items()), f"{n_sims} simulations each, parameters fitted on training quires", "see note", "out/generators_scores.csv")