"""T9/T10. Two cheap structural tests. T9 Null glyphs: for each glyph unit g, delete every occurrence and recompute conditional character entropy h2 and mean word length. Hypothesis H-CIPHER-NULLS predicts that removing the right unit lifts h2 into the attested-language range (>= 2.51 bits, the minimum of the 103 samples incl. vowel-less). Also tried: removing the commonest unit from each language sample as a control for what deletion does to h2 in general. T10 Canonical order: how much of each word's letter ORDER is predictable from its letter SET? Fit one global ranking of glyph units that maximises the share of words whose sequence is non-decreasing in rank; report that share for the manuscript, its within-word shuffle surrogate, and language samples. Hypothesis H-CIPHER-ANAGRAM (words are letter sets written in a fixed order) predicts a share near 1 for the manuscript and far lower for languages. Writes out/nulls.csv, out/order.csv; logs to results.csv. """ import os, sys _here = os.path.dirname(os.path.abspath(__file__)) # data/analysis, or a subfolder of it sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here)) from paths import * # noqa: repository locations import collections, csv, math, random, sys pass from corpus import tokenize_glyphs from harness import structured, flatten, languages, entropies, log pages = structured(); words = flatten(pages) UNITS = [tokenize_glyphs(w) for w in words] def h2_of(seqs): chars = [u for s in seqs for u in s + [" "]] return entropies(chars)[2] # ---------------- T9 nulls base_h2 = h2_of(UNITS); base_len = sum(len(s) for s in UNITS) / len(UNITS) cnt = collections.Counter(u for s in UNITS for u in s) rows = [dict(removed="(none)", share=0.0, h2=round(base_h2, 3), mean_len=round(base_len, 2))] for g, c in cnt.most_common(20): seqs = [[u for u in s if u != g] for s in UNITS]; seqs = [s for s in seqs if s] rows.append(dict(removed=g, share=round(c / sum(cnt.values()), 3), h2=round(h2_of(seqs), 3), mean_len=round(sum(len(s) for s in seqs) / len(seqs), 2))) # pairs of the two commonest for a, b in [("o", "e"), ("o", "y"), ("e", "y"), ("o", "a"), ("e", "i")]: seqs = [[u for u in s if u not in (a, b)] for s in UNITS]; seqs = [s for s in seqs if s] rows.append(dict(removed=a + "+" + b, share=round((cnt[a] + cnt[b]) / sum(cnt.values()), 3), h2=round(h2_of(seqs), 3), mean_len=round(sum(len(s) for s in seqs) / len(seqs), 2))) # control: languages with their commonest letter removed lang_rows = [] LANGS12 = ["Latin", "Italian", "German", "French", "Czech", "Hebrew", "Arabic", "Greek", "Hungarian", "Finnish", "Malagasy", "Latin.novowel"] L = languages() def langs12(): for n in LANGS12: if n in L: yield n, L[n] for name, sample in langs12(): ws = [list(w) for w in sample][:40000] c2 = collections.Counter(u for s in ws for u in s); top = c2.most_common(1)[0][0] seqs = [[u for u in s if u != top] for s in ws]; seqs = [s for s in seqs if s] lang_rows.append(dict(removed=f"{name}: {top}", share=round(c2[top] / sum(c2.values()), 3), h2=round(h2_of(seqs), 3), mean_len=round(sum(len(s) for s in seqs) / len(seqs), 2), base_h2=round(h2_of(ws), 3))) with open("out/nulls.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=["removed", "share", "h2", "mean_len", "base_h2"]); w.writeheader() for r in rows: w.writerow({**r, "base_h2": round(base_h2, 3)}) for r in lang_rows: w.writerow(r) best = max(rows[1:], key=lambda r: r["h2"]) print("T9 base h2 %.3f; best single deletion %s -> %.3f; languages' h2 after deleting commonest letter: %s" % (base_h2, best["removed"], best["h2"], [(r["removed"], r["base_h2"], r["h2"]) for r in lang_rows[:6]])) log("H-CIPHER-NULLS", "T9-nulls", "max h2 after deleting one glyph unit", best["h2"], "attested minimum 2.51 (languages lose 0.1-0.4 bits when their commonest letter is deleted)", "not supported" if best["h2"] < 2.51 else "consistent", f"deleted {best['removed']}; out/nulls.csv") # ---------------- T10 canonical order def fit_rank(seqs, iters=3): units = sorted(set(u for s in seqs for u in s)) prec = collections.Counter() for s in seqs: for i in range(len(s)): for j in range(i + 1, len(s)): if s[i] != s[j]: prec[(s[i], s[j])] += 1 score = {u: sum(prec[(u, v)] - prec[(v, u)] for v in units) for u in units} rank = sorted(units, key=lambda u: -score[u]) def share(rank): pos = {u: i for i, u in enumerate(rank)} ok = sum(all(pos[a] <= pos[b] for a, b in zip(s, s[1:])) for s in seqs if len(s) >= 2) return ok / sum(1 for s in seqs if len(s) >= 2) best = share(rank) for _ in range(iters): # local improvement by adjacent swaps improved = False for i in range(len(rank) - 1): r2 = rank[:]; r2[i], r2[i + 1] = r2[i + 1], r2[i] s2 = share(r2) if s2 > best: rank, best, improved = r2, s2, True if not improved: break return rank, best random.seed(0) orows = [] rank, sh = fit_rank(UNITS); orows.append(dict(text="Voynich (glyph units)", share=round(sh, 3), ranking=" ".join(rank))) shuf = [random.sample(s, len(s)) for s in UNITS]; r2, s2 = fit_rank(shuf); orows.append(dict(text="Voynich, letters shuffled within words", share=round(s2, 3), ranking=" ".join(r2))) eva = [list(w) for w in words]; r3, s3 = fit_rank(eva); orows.append(dict(text="Voynich (EVA letters)", share=round(s3, 3), ranking=" ".join(r3))) for name, sample in langs12(): ws = [list(w) for w in sample][:34000] r, s = fit_rank(ws, iters=1); orows.append(dict(text=name, share=round(s, 3), ranking=" ".join(r))) with open("out/order.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=orows[0].keys()); w.writeheader(); w.writerows(orows) for r in orows: print("T10 %-40s %.3f" % (r["text"], r["share"])) log("H-CIPHER-ANAGRAM", "T10-order", "share of words monotone under the best single ranking of units", orows[0]["share"], f"within-word shuffle {orows[1]['share']}; languages {min(r['share'] for r in orows[3:])}-{max(r['share'] for r in orows[3:])}", "see note", "out/order.csv")