"""Eğitilen tokenizer'ları FLORES-200 (dev+devtest, 2009 paralel cümle) üzerinde ölçer, hazır tokenizer'larla karşılaştırır.
Çıktı: results/eval.json, results/per_sentence.csv, results/vocab.json
"""
import csv, hashlib, json, re, statistics, time
from pathlib import Path
from huggingface_hub import hf_hub_download
from tokenizers import Tokenizer
import tiktoken

ROOT = Path(__file__).resolve().parent
F = ROOT / 'data/flores200_dataset'
load = lambda sp, l: (F / sp / f'{l}.{sp}').read_text(encoding='utf-8').splitlines()
TR = load('dev', 'tur_Latn') + load('devtest', 'tur_Latn')
EN = load('dev', 'eng_Latn') + load('devtest', 'eng_Latn')
assert len(TR) == len(EN) == 2009
PREV = {r['label']: r for r in json.load(open(ROOT / '../turkce-token-2026-10-21/results/tokens.json'))['results']}
sha = lambda p: hashlib.sha256(Path(p).read_bytes()).hexdigest()

REFS = [
    ('GPT-4o ailesi (o200k)', 'OpenAI o200k (GPT-4o, GPT-4.1, o3/o4)', 'tiktoken', 'o200k_base', None),
    ('Qwen3.8', 'Qwen3.8', 'local', '/home/zeka/ai/vllm/hf-cache/hub/models--nvidia--Qwen3.8-27B-NVFP4/snapshots/482ca0f3832238542f8f5295dde86b5f22711d80/tokenizer.json', None),
    ('Llama 4', 'Llama 4', 'hf', 'unsloth/Llama-4-Scout-17B-16E-Instruct', 'afd8e498c87bda51c7ea8ec68ea2f7c066e6340b'),
    ('Kumru (VNGRS)', 'Kumru 2B (VNGRS)', 'hf', 'vngrs-ai/Kumru-2B', '0b3cf37f657ef4013bdc99598545f5d317bd7397'),
    ('Türkçe GPT-2 (YTÜ)', 'Türkçe GPT-2 (YTÜ Cosmos)', 'hf', 'ytu-ce-cosmos/turkish-gpt2-large', 'c4ddd0f3808fb7bf23ab63d9fea6ec98c1899934'),
]


def ref(kind, src, rev):
    if kind == 'tiktoken':
        e = tiktoken.get_encoding(src)
        return (lambda s: e.encode(s, disallowed_special=())), (lambda ids: e.decode(ids)), \
            (lambda i: e.decode_single_token_bytes(i).decode('utf-8', 'replace')), {'source': f'tiktoken {tiktoken.__version__} {src}', 'vocab': e.n_vocab}
    path = hf_hub_download(src, 'tokenizer.json', revision=rev) if kind == 'hf' else src
    return own(path, {'source': f'huggingface.co/{src}' if kind == 'hf' else 'yerel Qwen3.8 NVFP4 snapshot 482ca0f', 'revision': rev})


def own(path, meta=None):
    t = Tokenizer.from_file(str(path))
    m = dict(meta or {}); m['vocab'] = t.get_vocab_size(); m['sha256'] = sha(path)
    return (lambda s: t.encode(s, add_special_tokens=False).ids), (lambda ids: t.decode(ids)), (lambda i: t.decode([i])), m


WORDS = ['yapabileceklerimizden', 'gözlükçülerden', 'Çekoslovakyalılaştıramadıklarımızdan', 'teşekkür ederim',
         'kitaplarımızdaki', 'thank you', 'üniversitesinde', 'computer']
SENT = TR[0]
nw = lambda L: sum(len(s.split()) for s in L)
train = {r['name']: r for r in json.load(open(ROOT / 'results/train.json'))['runs']}
out, per = [], {}


def measure(label, group, enc, dec, piece, meta):
    tr = [len(enc(s)) for s in TR]; en = [len(enc(s)) for s in EN]
    used = set(); [used.update(enc(s)) for s in TR]
    lossless = sum(dec(enc(s)) == s for s in TR + EN)
    r = dict(label=label, group=group, **meta, tr_tokens=sum(tr), en_tokens=sum(en), ratio_total=round(sum(tr) / sum(en), 4),
             tr_chars_per_token=round(sum(map(len, TR)) / sum(tr), 3), en_chars_per_token=round(sum(map(len, EN)) / sum(en), 3),
             tr_tokens_per_word=round(sum(tr) / nw(TR), 3), en_tokens_per_word=round(sum(en) / nw(EN), 3),
             tr_distinct_ids_used=len(used), lossless_roundtrip=f'{lossless}/{len(TR) + len(EN)}',
             splits={w: [piece(i) for i in enc(w)] for w in WORDS}, sentence0=[piece(i) for i in enc(SENT)])
    per[label] = (tr, en); out.append(r)
    print(f"{label:28s} TR {r['tr_tokens']:6d} EN {r['en_tokens']:6d} oran {r['ratio_total']:.3f} kayıpsız {r['lossless_roundtrip']}", flush=True)
    return r


for label, prev, kind, src, rev in REFS:
    r = measure(label, 'hazır', *ref(kind, src, rev))
    p = PREV[prev]; r['matches_prev_article'] = (p['tr_tokens'], p['en_tokens']) == (r['tr_tokens'], r['en_tokens'])
for name in train:
    r = measure(name, 'benim', *own(ROOT / 'tokenizers' / f'{name}.json', {'source': f'tokenizers/{name}.json'}))
    r['train'] = train[name]

# Sözlük içeriği: 32k Türkçe tokenizer'ın öğrendiği parçalar
voc = {}
for name in ('tr_100M_32k', 'en_100M_32k', 'mix_100M_32k'):
    t = Tokenizer.from_file(str(ROOT / 'tokenizers' / f'{name}.json'))
    j = json.loads(t.to_str()); merges = j['model']['merges']
    pieces = [t.decode([i]) for i in range(t.get_vocab_size())]
    words = [p for p in pieces if re.fullmatch(r' ?[^\W\d_]+', p)]
    tr_chars = sum(1 for p in pieces if re.search('[çğıöşüÇĞİÖŞÜ]', p))
    first = [''.join(m) if isinstance(m, list) else m.replace(' ', '') for m in merges[:40]]
    first = [t.decode(t.encode(''.join(x for x in m), add_special_tokens=False).ids) if False else m for m in first]
    longest = sorted({p.strip() for p in words}, key=len, reverse=True)[:25]
    suffixes = ['lar', 'ler', 'ları', 'leri', 'ların', 'lerin', 'ında', 'inde', 'ından', 'inden', 'dır', 'dir', 'mış', 'miş', 'yor', 'acak', 'ecek']
    vid = t.get_vocab()
    bl = __import__('tokenizers').decoders.ByteLevel()
    voc[name] = dict(size=len(pieces), with_turkish_letter=tr_chars, mean_piece_chars=round(statistics.mean(len(p) for p in pieces), 2),
                     first_merges=[bl.decode([m if isinstance(m, str) else ''.join(m)]) for m in first],
                     longest_word_pieces=longest, suffix_tokens_present={s: (s in [p for p in pieces]) for s in suffixes})
    print(name, voc[name]['with_turkish_letter'], voc[name]['first_merges'][:15], longest[:5])

json.dump(dict(corpus='FLORES-200 dev+devtest (2009 cümle)', sentence0=SENT, tr_words=nw(TR), en_words=nw(EN), tr_chars=sum(map(len, TR)),
               results=out, vocab=voc, measured_at=time.strftime('%Y-%m-%d %H:%M %z')),
          open(ROOT / 'results/eval.json', 'w'), ensure_ascii=False, indent=1)
with open(ROOT / 'results/per_sentence.csv', 'w', newline='') as f:
    w = csv.writer(f); w.writerow(['idx', 'tokenizer', 'tr_tokens', 'en_tokens'])
    for label, (tr, en) in per.items():
        for i, (a, b) in enumerate(zip(tr, en)): w.writerow([i, label, a, b])
