"""T6. Form vs function: do words that look alike behave alike?
For the 300 commonest words of each corpus, compute (a) string similarity (1 - normalised
edit distance) and (b) distributional similarity (cosine of left+right neighbour profiles)
for all pairs; report the Spearman correlation and the share of a word's 5 nearest
distributional neighbours that are also within edit distance 1 of it. In inflecting
languages the two are mildly related; in a verbose cipher whose word-level neighbours are
plaintext letters they should be unrelated to word form; in an auto-copied text they should
be strongly related. Outputs: out/form_function.csv
"""
import os, sys
_here = os.path.dirname(os.path.abspath(__file__))  # data/analysis, or a subfolder of it
sys.path.insert(0, _here if os.path.exists(os.path.join(_here, "paths.py")) else os.path.dirname(_here))
from paths import *  # noqa: repository locations
import csv, collections, itertools
import numpy as np
from scipy.stats import spearmanr
from harness import *


def edit(a, b):
    if a == b: return 0
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i]
        for j, cb in enumerate(b, 1):
            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
        prev = cur
    return prev[-1]


def analyse(words, K=300):
    cnt = collections.Counter(words); top = [w for w, c in cnt.most_common(K)]; ti = {w: i for i, w in enumerate(top)}
    ctx = np.zeros((K, 2 * K))
    for i in range(1, len(words) - 1):
        w = words[i]
        if w in ti:
            if words[i - 1] in ti: ctx[ti[w], ti[words[i - 1]]] += 1
            if words[i + 1] in ti: ctx[ti[w], K + ti[words[i + 1]]] += 1
    ctx = ctx / (np.linalg.norm(ctx, axis=1, keepdims=True) + 1e-9); D = ctx @ ctx.T
    form = np.zeros((K, K))
    for i, j in itertools.combinations(range(K), 2):
        form[i, j] = form[j, i] = 1 - edit(top[i], top[j]) / max(len(top[i]), len(top[j]))
    iu = np.triu_indices(K, 1)
    rho = spearmanr(form[iu], D[iu]).correlation
    # nearest distributional neighbours within edit distance 1
    near = 0; tot = 0
    for i in range(K):
        nn = np.argsort(-D[i])[1:6]
        for j in nn:
            tot += 1; near += edit(top[i], top[j]) <= 1
    return rho, near / tot


rows = []
pages = structured()
V = flatten(pages)
rho, nn = analyse(V); rows.append(dict(corpus="Voynich", spearman=rho, nn5_edit1=nn)); print(rows[-1])
rho, nn = analyse(flatten(shuffle_words_global(pages))); rows.append(dict(corpus="Voynich, words shuffled", spearman=rho, nn5_edit1=nn)); print(rows[-1])
langs = languages()
for L in ["Latin", "Italian", "German", "Czech", "Hungarian", "Hebrew", "Nahuatl-NT", "Swahili-NT", "Quichua-NT", "Malagasy", "Turkish", "Finnish"]:
    rho, nn = analyse(langs[L][:len(V)]); rows.append(dict(corpus=L, spearman=rho, nn5_edit1=nn)); print(rows[-1])
with open("out/form_function.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)