"""T14. Did the scribes execute a procedure, or copy an exemplar? Three statistics, each compared with a null in which scribe labels are shuffled across pages within the same section and Currier language (so section and language cannot drive the result). (a) Variant choice. For the commonest glyph alternations (ch/sh, k/t, o/a, e/ee, y/dy, in/iin, ain/aiin, l/r, d/s, ol/or), every word pair that differs only by that alternation is a minimal pair; per scribe, the share taken by the first member. Statistic: G-test of scribe x variant over all pairs of the alternation (summed), expressed as z against the null. (b) Transition fingerprint. Glyph-unit bigram distribution per scribe; mean Jensen-Shannon divergence between scribes, z against the null. (c) Error typology. Near-twin pairs (a word followed within ten words by a word one edit away): the distribution of the substituted glyph pair per scribe; JS divergence between scribes, z against the null. Also: visual-confusion share (ch/sh, k/t, a/o, e/o, r/s, in/iin) of all near-twin substitutions per scribe. Strata: herbal B (scribes 2, 3, 5) is the only section with three scribes in one language; all-B (scribes 2, 3, 5 across sections) is reported as a looser check. Writes out/scribes.csv, out/scribes_variants.csv; logs to results.csv. """ import os, sys _here = os.path.dirname(os.path.abspath(__file__)) # data/analysis, or a subfolder of it sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here)) from paths import * # noqa: repository locations import collections, csv, math, random, sys from corpus import Corpus, tokenize_glyphs from harness import log c = Corpus() ALT = [("ch", "sh"), ("k", "t"), ("o", "a"), ("e", "ee"), ("y", "dy"), ("in", "iin"), ("ain", "aiin"), ("l", "r"), ("d", "s"), ("ol", "or")] VISUAL = {frozenset(x) for x in [("ch", "sh"), ("k", "t"), ("a", "o"), ("e", "o"), ("r", "s"), ("in", "iin"), ("i", "ii"), ("f", "p"), ("cth", "ckh")]} def pages_for(section, lang, scribes): out = [] for p in c.pages: v = c.pagevars[p] if (section is None or c.section_of(p) == section) and v.get("L") == lang and v.get("H") in scribes: ws = c.running_text(pages=[p]) if len(ws) >= 20: out.append((p, v["H"], ws)) return out def variant_counts(pages_ws): """per scribe: for each alternation, count of words containing variant a vs b position-matched pairs.""" vocab = collections.Counter(w for _, _, ws in pages_ws for w in ws) cnt = collections.defaultdict(lambda: collections.Counter()) for a, b in ALT: pairs = set() for w in vocab: if a in w: for i in range(len(w)): if w.startswith(a, i): w2 = w[:i] + b + w[i + len(a):] if w2 in vocab: pairs.add((w, w2)) for p, h, ws in pages_ws: wc = collections.Counter(ws) for w1, w2 in pairs: cnt[(a, b)][(h, "a")] += wc[w1]; cnt[(a, b)][(h, "b")] += wc[w2] return cnt def gstat(table): """G statistic for scribe x variant table dict[(h,v)] -> n""" hs = sorted({h for h, v in table}); vs = ["a", "b"] N = sum(table.values()) if N == 0: return 0.0 g = 0.0 for h in hs: rh = sum(table[(h, v)] for v in vs) for v in vs: cv = sum(table[(h2, v)] for h2 in hs); o = table[(h, v)]; e = rh * cv / N if o > 0 and e > 0: g += 2 * o * math.log(o / e) return g def bigram_dist(ws): d = collections.Counter() for w in ws: u = tokenize_glyphs(w) for a, b in zip(["^"] + u, u + ["$"]): d[(a, b)] += 1 return d def js(p, q): keys = set(p) | set(q); sp, sq = sum(p.values()), sum(q.values()) if not sp or not sq: return 0.0 m = {k: 0.5 * (p.get(k, 0) / sp + q.get(k, 0) / sq) for k in keys} def kl(x, sx): return sum(x[k] / sx * math.log2((x[k] / sx) / m[k]) for k in x if x[k]) return 0.5 * kl(p, sp) + 0.5 * kl(q, sq) def mean_js(dists): hs = list(dists); vals = [] for i in range(len(hs)): for j in range(i + 1, len(hs)): vals.append(js(dists[hs[i]], dists[hs[j]])) return sum(vals) / len(vals) if vals else 0.0 def twin_subst(ws, window=10): """substituted glyph pairs among near-twin (edit-1 on glyph units) pairs within the window""" out = collections.Counter(); vis = tot = 0 U = [tokenize_glyphs(w) for w in ws] for i in range(len(U)): for j in range(i + 1, min(len(U), i + 1 + window)): a, b = U[i], U[j] if a == b: continue if len(a) == len(b): diff = [(x, y) for x, y in zip(a, b) if x != y] if len(diff) == 1: out[frozenset(diff[0])] += 1; tot += 1; vis += frozenset(diff[0]) in VISUAL elif abs(len(a) - len(b)) == 1: s, l = (a, b) if len(a) < len(b) else (b, a) for k in range(len(l)): if l[:k] + l[k + 1:] == s: out[frozenset((l[k], "-"))] += 1; tot += 1; vis += frozenset((l[k], "-")) in VISUAL or l[k] in ("i",); break return out, (vis / tot if tot else 0.0), tot def run(name, pages_ws, nperm=300, seed=0): rng = random.Random(seed) def stats(pw): by = collections.defaultdict(list) for p, h, ws in pw: by[h] += ws vc = variant_counts(pw); G = sum(gstat(t) for t in vc.values()) bj = mean_js({h: bigram_dist(ws) for h, ws in by.items()}) tw = {h: twin_subst(ws) for h, ws in by.items()} tj = mean_js({h: tw[h][0] for h in tw}) return G, bj, tj, {h: tw[h][1] for h in tw}, {h: tw[h][2] for h in tw}, vc G, bj, tj, vis, ntw, vc = stats(pages_ws) null = [] labels = [h for _, h, _ in pages_ws] for _ in range(nperm): rng.shuffle(labels) pw = [(p, h, ws) for (p, _, ws), h in zip(pages_ws, labels)] g2, b2, t2, _, _, _ = stats(pw); null.append((g2, b2, t2)) def z(x, i): xs = [n[i] for n in null]; mu = sum(xs) / len(xs); sd = (sum((v - mu) ** 2 for v in xs) / len(xs)) ** 0.5 or 1e-9 return (x - mu) / sd, sum(v >= x for v in xs) / len(xs) r = dict(stratum=name, n_pages=len(pages_ws), n_words=sum(len(ws) for _, _, ws in pages_ws), scribes=" ".join(sorted({h for _, h, _ in pages_ws})), variant_G=round(G, 1), variant_z=round(z(G, 0)[0], 2), variant_p=round(z(G, 0)[1], 3), bigram_js=round(bj, 4), bigram_z=round(z(bj, 1)[0], 2), bigram_p=round(z(bj, 1)[1], 3), twin_js=round(tj, 4), twin_z=round(z(tj, 2)[0], 2), twin_p=round(z(tj, 2)[1], 3), visual_share=" ".join(f"{h}:{v:.2f}(n={ntw[h]})" for h, v in sorted(vis.items()))) print(r, flush=True) return r, vc rows = []; vrows = [] for name, sec, lang, scr in [("herbal B (scribes 2,3,5)", "Herbal", "B", {"2", "3", "5"}), ("all B (scribes 2,3,5)", None, "B", {"2", "3", "5"}), ("herbal A vs pharmaceutical A (scribe 1 only; section control)", None, "A", {"1"})]: pw = pages_for(sec, lang, scr) if name.startswith("herbal A"): # relabel scribe-1 pages by section to show what a section difference looks like on the same scale pw = [(p, c.section_of(p)[:4], ws) for p, h, ws in pw if c.section_of(p) in ("Herbal", "Pharmaceutical")] r, vc = run(name, pw); rows.append(r) for (a, b), t in vc.items(): for h in sorted({h for h, v in t}): na, nb = t[(h, "a")], t[(h, "b")] vrows.append(dict(stratum=name, alternation=f"{a}/{b}", scribe=h, n_a=na, n_b=nb, share_a=round(na / (na + nb), 3) if na + nb else "")) with open("out/scribes.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows) with open("out/scribes_variants.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=vrows[0].keys()); w.writeheader(); w.writerows(vrows) r = rows[0] log("H-HANDS-PROCEDURE", "T14-scribes", "herbal B, scribes 2/3/5: variant-choice G (z vs page-shuffled null); bigram JS z; near-twin substitution JS z", f"G z={r['variant_z']} p={r['variant_p']}; bigram z={r['bigram_z']} p={r['bigram_p']}; twin z={r['twin_z']} p={r['twin_p']}", "null: scribe labels shuffled across pages within section and language (300 draws)", "see note", "out/scribes.csv, out/scribes_variants.csv")