"""Türkçe / İngilizce token sayımı — FLORES-200 (dev+devtest, 2009 paralel cümle).

Her tokenizer için: cümle başına token (özel tokenlar hariç), toplam, TR/EN oranı,
karakter/token, kelime/token; ayrıca Türkçe harfleri ASCII'ye çevrilmiş metin ve örnek kelime bölünmeleri.
Çıktı: results/tokens.json, results/per_sentence.csv, results/splits.json
"""
import csv, json, os, statistics, sys, time
from pathlib import Path
from huggingface_hub import hf_hub_download, HfApi
from tokenizers import Tokenizer
import tiktoken

ROOT = Path(__file__).resolve().parent
DATA = ROOT / 'data/flores200_dataset'
OUT = ROOT / 'results'; OUT.mkdir(exist_ok=True)

def load(split, lang):
    return (DATA / split / f'{lang}.{split}').read_text(encoding='utf-8').splitlines()

TR = load('dev', 'tur_Latn') + load('devtest', 'tur_Latn')
EN = load('dev', 'eng_Latn') + load('devtest', 'eng_Latn')
assert len(TR) == len(EN) == 2009

FOLD = str.maketrans('çğıöşüÇĞİÖŞÜâîûÂÎÛ', 'cgiosuCGIOSUaiuAIU')
TR_ASCII = [s.translate(FOLD) for s in TR]

# (etiket, grup, kaynak türü, kaynak, revizyon notu)
SPECS = [
    ('OpenAI o200k (GPT-4o, GPT-4.1, o3/o4)', 'Kapalı model', 'tiktoken', 'o200k_base'),
    ('OpenAI cl100k (GPT-4, GPT-3.5)', 'Kapalı model', 'tiktoken', 'cl100k_base'),
    ('gpt-oss', 'Açık model', 'local', '/home/zeka/ai/models/gpt-oss-120b/tokenizer.json'),
    ('Qwen3.8', 'Açık model', 'local', '/home/zeka/ai/vllm/hf-cache/hub/models--nvidia--Qwen3.8-27B-NVFP4/snapshots/482ca0f3832238542f8f5295dde86b5f22711d80/tokenizer.json'),
    ('GLM-5.3 Flash', 'Açık model', 'hf', 'zai-org/GLM-5.3-Flash'),
    ('DeepSeek V3.2', 'Açık model', 'hf', 'deepseek-ai/DeepSeek-V3.2'),
    ('Gemma 4', 'Açık model', 'hf', 'google/gemma-4-31b-it'),
    ('Llama 4', 'Açık model', 'hf', 'unsloth/Llama-4-Scout-17B-16E-Instruct'),
    ('Mistral Small 3.2', 'Açık model', 'hf', 'unsloth/Mistral-Small-3.2-24B-Instruct-2506'),
    ('MiniMax M2', 'Açık model', 'hf', 'MiniMaxAI/MiniMax-M2'),
    ('Nemotron 3 Super', 'Açık model', 'local', '/home/zeka/ai/models/nvidia-nemotron-3-super-120b-a12b-nvfp4/tokenizer.json'),
    ('Kumru 2B (VNGRS)', 'Türkçe model', 'hf', 'vngrs-ai/Kumru-2B'),
    ('Turkcell LLM 7B', 'Türkçe model', 'hf', 'TURKCELL/Turkcell-LLM-7b-v1'),
    ('Trendyol LLM 7B v4.1', 'Türkçe model', 'hf', 'Trendyol/Trendyol-LLM-7B-chat-v4.1.0'),
    ('Türkçe GPT-2 (YTÜ Cosmos)', 'Türkçe model', 'hf', 'ytu-ce-cosmos/turkish-gpt2-large'),
]

api = HfApi()

def make(kind, src):
    if kind == 'tiktoken':
        enc = tiktoken.get_encoding(src)
        return (lambda s: enc.encode(s, disallowed_special=())), (lambda ids: [enc.decode_single_token_bytes(i).decode('utf-8', 'replace') for i in ids]), {'source': f'tiktoken {tiktoken.__version__} {src}', 'vocab': enc.n_vocab}
    if kind == 'hf':
        rev = api.model_info(src).sha
        path = hf_hub_download(src, 'tokenizer.json', revision=rev)
        meta = {'source': f'huggingface.co/{src}', 'revision': rev}
    else:
        path = src; meta = {'source': src}
    tok = Tokenizer.from_file(path)
    meta['vocab'] = tok.get_vocab_size()
    meta['sha256'] = __import__('hashlib').sha256(Path(path).read_bytes()).hexdigest()
    return (lambda s: tok.encode(s, add_special_tokens=False).ids), (lambda ids: [tok.decode([i]) for i in ids]), meta

WORDS = ['yapabileceklerimizden', 'gözlükçülerden', 'teşekkür ederim', 'thank you', 'bilgisayar', 'computer',
         'Çekoslovakyalılaştıramadıklarımızdan']

def nwords(lines):
    return sum(len(s.split()) for s in lines)

results, per = [], {}
for label, group, kind, src in SPECS:
    t0 = time.time()
    enc, pieces, meta = make(kind, src)
    tr = [len(enc(s)) for s in TR]; en = [len(enc(s)) for s in EN]; ta = [len(enc(s)) for s in TR_ASCII]
    ratios = [a / b for a, b in zip(tr, en)]
    r = dict(label=label, group=group, **meta,
             tr_tokens=sum(tr), en_tokens=sum(en), tr_ascii_tokens=sum(ta),
             ratio_total=round(sum(tr) / sum(en), 4), ratio_median_sentence=round(statistics.median(ratios), 4),
             tr_chars_per_token=round(sum(map(len, TR)) / sum(tr), 3), en_chars_per_token=round(sum(map(len, EN)) / sum(en), 3),
             tr_tokens_per_word=round(sum(tr) / nwords(TR), 3), en_tokens_per_word=round(sum(en) / nwords(EN), 3),
             ascii_vs_tr=round(sum(ta) / sum(tr), 4))
    r['splits'] = {w: [p if isinstance(p, str) else p.decode('utf-8', 'replace') for p in pieces(enc(w))] for w in WORDS}
    per[label] = (tr, en)
    results.append(r)
    print(f"{label:40s} TR {sum(tr):6d} EN {sum(en):6d} oran {r['ratio_total']:.3f} medyan {r['ratio_median_sentence']:.3f} ascii {r['ascii_vs_tr']:.3f} ({time.time()-t0:.1f}s)", flush=True)

corpus = dict(name='FLORES-200 dev+devtest', sentences=len(TR), tr_chars=sum(map(len, TR)), en_chars=sum(map(len, EN)),
              tr_words=nwords(TR), en_words=nwords(EN), license='CC BY-SA 4.0',
              url='https://dl.fbaipublicfiles.com/nllb/flores200_dataset.tar.gz',
              tar_sha256='b8b0b76783024b85797e5cc75064eb83fc5288b41e9654dabc7be6ae944011f6')
json.dump(dict(corpus=corpus, results=results, measured_at=time.strftime('%Y-%m-%d %H:%M %z')), open(OUT / 'tokens.json', 'w'), ensure_ascii=False, indent=1)
with open(OUT / 'per_sentence.csv', 'w', newline='') as f:
    w = csv.writer(f); w.writerow(['idx', 'tokenizer', 'tr_tokens', 'en_tokens'])
    for label, (tr, en) in per.items():
        for i, (a, b) in enumerate(zip(tr, en)): w.writerow([i, label, a, b])
print(json.dumps(corpus, ensure_ascii=False))
