"""Ek grafikler: dağılım, kelime başı token, sözlük büyüklüğü, cümle parçalama, iki yönlü maliyet, aynı bütçeye sığan cümle."""
import csv, json, statistics
from collections import defaultdict
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
import tiktoken
from tokenizers import Tokenizer

ROOT = Path(__file__).resolve().parent
OUT = ROOT / 'charts'
D = json.load(open(ROOT / 'results/tokens.json'))
R = {r['label']: r for r in D['results']}
W, H = 1600, 900
BG, FG, MUTED, GRID = (24, 26, 22), (240, 234, 220), (170, 165, 150), (60, 62, 56)
BLUE, AMBER, GREEN, RED = (110, 170, 230), (240, 170, 70), (120, 200, 120), (225, 95, 85)
B = '/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf'
RG = '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf'
MONO = '/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf'
f = lambda p, s: ImageFont.truetype(p, s)
c = lambda x, n=2: f'{x:.{n}f}'.replace('.', ',')
th = lambda x: f'{x:,}'.replace(',', '.')
SHORT = {'OpenAI o200k (GPT-4o, GPT-4.1, o3/o4)': 'GPT-4o ailesi', 'OpenAI cl100k (GPT-4, GPT-3.5)': 'GPT-4 / 3.5 (eski)',
         'Qwen3.8': 'Qwen3.8', 'GLM-5.3 Flash': 'GLM-5.3', 'DeepSeek V3.2': 'DeepSeek V3.2', 'Gemma 4': 'Gemma 4',
         'Llama 4': 'Llama 4', 'Mistral Small 3.2': 'Mistral Small 3.2', 'MiniMax M2': 'MiniMax M2',
         'Nemotron 3 Super': 'Nemotron 3 Super', 'Kumru 2B (VNGRS)': 'Kumru (VNGRS)', 'Turkcell LLM 7B': 'Turkcell LLM',
         'Trendyol LLM 7B v4.1': 'Trendyol LLM', 'Türkçe GPT-2 (YTÜ Cosmos)': 'Türkçe GPT-2 (YTÜ)'}
COL = {'Kapalı model': BLUE, 'Açık model': AMBER, 'Türkçe model': GREEN}
ALL = [r for r in D['results'] if r['label'] != 'gpt-oss']


def canvas(title, sub):
    im = Image.new('RGB', (W, H), BG); d = ImageDraw.Draw(im)
    d.text((60, 44), title, font=f(B, 48), fill=FG)
    d.text((60, 108), sub, font=f(RG, 24), fill=MUTED)
    return im, d


def sign(d):
    d.text((W - 48, H - 36), '@zekayapar', font=f(B, 30), fill=FG, anchor='rd')


per = defaultdict(list)
for row in csv.DictReader(open(ROOT / 'results/per_sentence.csv')):
    per[row['tokenizer']].append(int(row['tr_tokens']) / int(row['en_tokens']))


def distribution():
    picks = [('OpenAI o200k (GPT-4o, GPT-4.1, o3/o4)', BLUE), ('DeepSeek V3.2', RED), ('Turkcell LLM 7B', GREEN)]
    im, d = canvas('Her cümlede durum aynı mı?', '2009 cümlenin her biri için Türkçe/İngilizce token oranı · kaç cümle hangi aralıkta')
    x0, x1, y0, y1 = 140, 1500, 200, 760
    bins = [i / 10 for i in range(5, 31)]  # 0.5..3.0
    hists = []
    for lab, col in picks:
        h = [0] * (len(bins) - 1)
        for v in per[lab]:
            for i in range(len(bins) - 1):
                if bins[i] <= v < bins[i + 1]: h[i] += 1; break
        hists.append(h)
    mx = max(max(h) for h in hists)
    bw = (x1 - x0) / (len(bins) - 1)
    for t in range(0, mx + 1, 200):
        y = y1 - t / mx * (y1 - y0)
        d.line((x0, y, x1, y), fill=GRID); d.text((x0 - 12, y), str(t), font=f(RG, 20), fill=MUTED, anchor='rm')
    for i, v in enumerate(bins):
        x = x0 + i * bw
        if abs(v * 2 - round(v * 2)) < 1e-6:
            d.text((x, y1 + 12), c(v, 1) + '×', font=f(RG, 20), fill=MUTED, anchor='mt')
    xe = x0 + (1.0 - 0.5) / 0.1 * bw
    d.line((xe, y0 - 10, xe, y1), fill=FG, width=3)
    d.text((xe + 8, y0 - 10), 'eşit', font=f(RG, 20), fill=FG)
    for (lab, col), h in zip(picks, hists):
        pts = [(x0 + (i + 0.5) * bw, y1 - v / mx * (y1 - y0)) for i, v in enumerate(h)]
        d.line([(x0, y1)] + pts + [(x1, y1)], fill=col, width=5, joint='curve')
    lx = 140
    for lab, col in picks:
        med = statistics.median(per[lab])
        d.rectangle((lx, 815, lx + 26, 841), fill=col)
        d.text((lx + 36, 828), f'{SHORT[lab]} · ortanca {c(med)}×', font=f(RG, 24), fill=FG, anchor='lm')
        lx += 470
    sign(d); im.save(OUT / 'dagilim.png')


def per_word():
    rows = sorted(ALL, key=lambda r: r['tr_tokens_per_word'])
    im, d = canvas('Bir kelime kaç token?', 'ortalama · Türkçe kelime (34.301) ve İngilizce kelime (42.855) aynı metinden')
    x0, x1, top, rh = 420, 1440, 170, 42
    sc = (x1 - x0) / 3.4
    for v in (0, 1, 2, 3):
        x = x0 + v * sc
        d.line((x, top - 6, x, top + len(rows) * rh), fill=GRID)
        d.text((x, top + len(rows) * rh + 8), str(v), font=f(RG, 20), fill=MUTED, anchor='mt')
    for i, r in enumerate(rows):
        y = top + i * rh + rh / 2 - 3
        a, b = r['en_tokens_per_word'], r['tr_tokens_per_word']
        d.text((x0 - 16, y), SHORT[r['label']], font=f(RG, 24), fill=FG, anchor='rm')
        d.line((x0 + min(a, b) * sc, y, x0 + max(a, b) * sc, y), fill=GRID, width=6)
        d.ellipse((x0 + a * sc - 10, y - 10, x0 + a * sc + 10, y + 10), fill=BLUE)
        d.ellipse((x0 + b * sc - 10, y - 10, x0 + b * sc + 10, y + 10), fill=RED)
        d.text((x0 + max(a, b) * sc + 18, y), c(b), font=f(B, 22), fill=RED, anchor='lm')
    d.ellipse((60, H - 60, 82, H - 38), fill=BLUE); d.text((92, H - 49), 'İngilizce', font=f(RG, 22), fill=FG, anchor='lm')
    d.ellipse((250, H - 60, 272, H - 38), fill=RED); d.text((282, H - 49), 'Türkçe (sayı)', font=f(RG, 22), fill=FG, anchor='lm')
    sign(d); im.save(OUT / 'kelime.png')


def vocab():
    im, d = canvas('Sözlük büyüdükçe Türkçe ucuzluyor mu?', 'yatay: tokenizer sözlüğündeki parça sayısı · dikey: Türkçe/İngilizce oranı')
    x0, x1, y0, y1 = 170, 1480, 190, 760
    vx = lambda v: x0 + (v - 40000) / (270000 - 40000) * (x1 - x0)
    vy = lambda r: y1 - (r - 0.5) / (2.2 - 0.5) * (y1 - y0)
    for t in (50000, 100000, 150000, 200000, 250000):
        d.line((vx(t), y0, vx(t), y1), fill=GRID); d.text((vx(t), y1 + 12), f'{t // 1000} bin', font=f(RG, 20), fill=MUTED, anchor='mt')
    for t in (0.5, 1.0, 1.5, 2.0):
        d.line((x0, vy(t), x1, vy(t)), fill=MUTED if t == 1.0 else GRID, width=2 if t == 1.0 else 1)
        d.text((x0 - 12, vy(t)), c(t, 1) + '×', font=f(RG, 20), fill=MUTED, anchor='rm')
    offs = {'GPT-4o ailesi': (14, -30), 'Qwen3.8': (-16, -36), 'Llama 4': (14, 4), 'Gemma 4': (-16, 10), 'MiniMax M2': (14, -12),
            'Nemotron 3 Super': (-16, 8), 'Mistral Small 3.2': (-16, -34), 'Trendyol LLM': (16, 10), 'GLM-5.3': (16, -36),
            'DeepSeek V3.2': (14, -8), 'GPT-4 / 3.5 (eski)': (14, -26), 'Turkcell LLM': (14, -10), 'Kumru (VNGRS)': (14, -26), 'Türkçe GPT-2 (YTÜ)': (14, 6)}
    for r in ALL:
        x, y = vx(r['vocab']), vy(r['ratio_total']); s = SHORT[r['label']]
        d.ellipse((x - 11, y - 11, x + 11, y + 11), fill=COL[r['group']])
        ox, oy = offs.get(s, (14, -10))
        d.text((x + ox, y + oy), s, font=f(RG, 21), fill=FG, anchor='ra' if ox < 0 else 'la')
    d.text((60, H - 36), 'sözlük büyüklüğü tek başına belirlemiyor; sözlüğün hangi dillerle kurulduğu daha belirleyici', font=f(RG, 22), fill=MUTED, anchor='ld')
    sign(d); im.save(OUT / 'sozluk.png')


def sentence_boxes():
    tr = 'Daha önce diyabetik olup şimdi diyabetik olmayan 4 aylık farelerimiz var, diye ekledi.'
    en = '"We now have 4-month-old mice that are non-diabetic that used to be diabetic," he added.'
    enc = tiktoken.get_encoding('o200k_base')
    im, d = canvas('Aynı cümle, iki dil', 'GPT-4o ailesinin tokenizer’ı · her renkli kutu bir token')
    pal = [(70, 110, 150), (150, 110, 60), (80, 130, 80), (130, 80, 130), (150, 80, 70), (70, 130, 130)]

    def draw(text, y, label):
        ids = enc.encode(text); parts = [enc.decode_single_token_bytes(i).decode('utf-8', 'replace') for i in ids]
        d.text((60, y), f'{label} · {len(parts)} token', font=f(B, 30), fill=AMBER)
        x, yy = 60, y + 52
        for j, p in enumerate(parts):
            t = p.replace(' ', '·')
            tw = d.textlength(t, font=f(MONO, 26))
            if x + tw + 14 > W - 60: x = 60; yy += 56
            d.rounded_rectangle((x, yy, x + tw + 14, yy + 42), radius=6, fill=pal[j % len(pal)])
            d.text((x + 7, yy + 21), t, font=f(MONO, 26), fill=FG, anchor='lm')
            x += tw + 20
        return yy + 70
    y = draw(tr, 170, 'Türkçe')
    draw(en, y + 30, 'İngilizce')
    d.text((60, H - 36), '· = boşluk · cümle FLORES-200’den', font=f(RG, 22), fill=MUTED, anchor='ld')
    sign(d); im.save(OUT / 'cumle.png')


def both_ways():
    rows = sorted(ALL, key=lambda r: r['tr_tokens'])
    im, d = canvas('Aynı metin için toplam token', '2009 cümle · Türkçe ve İngilizce hali ayrı ayrı · Türkçe modellerde İngilizce pahalanıyor')
    x0, x1, top, rh = 420, 1440, 170, 42
    sc = (x1 - x0) / 110000
    for v in (0, 25000, 50000, 75000, 100000):
        x = x0 + v * sc
        d.line((x, top - 6, x, top + len(rows) * rh), fill=GRID)
        d.text((x, top + len(rows) * rh + 8), f'{v // 1000} bin', font=f(RG, 20), fill=MUTED, anchor='mt')
    for i, r in enumerate(rows):
        y = top + i * rh
        d.text((x0 - 16, y + rh / 2 - 3), SHORT[r['label']], font=f(RG, 24), fill=FG, anchor='rm')
        d.rectangle((x0, y + 6, x0 + r['tr_tokens'] * sc, y + 19), fill=RED)
        d.rectangle((x0, y + 21, x0 + r['en_tokens'] * sc, y + 34), fill=BLUE)
        d.text((x0 + max(r['tr_tokens'], r['en_tokens']) * sc + 10, y + 20), f"{th(r['tr_tokens'])} / {th(r['en_tokens'])}", font=f(RG, 19), fill=FG, anchor='lm')
    d.rectangle((60, H - 60, 82, H - 38), fill=RED); d.text((92, H - 49), 'Türkçe', font=f(RG, 22), fill=FG, anchor='lm')
    d.rectangle((230, H - 60, 252, H - 38), fill=BLUE); d.text((262, H - 49), 'İngilizce', font=f(RG, 22), fill=FG, anchor='lm')
    sign(d); im.save(OUT / 'toplam.png')


def budget():
    budget_tokens = 8192
    picks = ['Turkcell LLM 7B', 'Llama 4', 'OpenAI o200k (GPT-4o, GPT-4.1, o3/o4)', 'Qwen3.8', 'GLM-5.3 Flash', 'DeepSeek V3.2']
    im, d = canvas('8 bin token’a kaç cümle sığar?', 'FLORES ortalamasıyla · aynı bütçe, iki dil · uzun çubuk = daha çok içerik')
    x0, x1, top, rh = 420, 1440, 190, 100
    n = D['corpus']['sentences']
    sc = (x1 - x0) / 330
    for v in (0, 100, 200, 300):
        x = x0 + v * sc
        d.line((x, top - 6, x, top + len(picks) * rh), fill=GRID)
        d.text((x, top + len(picks) * rh + 8), str(v), font=f(RG, 20), fill=MUTED, anchor='mt')
    for i, lab in enumerate(picks):
        r = R[lab]; y = top + i * rh
        tr_n = budget_tokens / (r['tr_tokens'] / n); en_n = budget_tokens / (r['en_tokens'] / n)
        d.text((x0 - 16, y + 40), SHORT[lab], font=f(B, 26), fill=FG, anchor='rm')
        d.rounded_rectangle((x0, y + 8, x0 + tr_n * sc, y + 38), radius=5, fill=RED)
        d.rounded_rectangle((x0, y + 44, x0 + en_n * sc, y + 74), radius=5, fill=BLUE)
        d.text((x0 + tr_n * sc + 10, y + 23), f'{tr_n:.0f} Türkçe cümle', font=f(RG, 21), fill=FG, anchor='lm')
        d.text((x0 + en_n * sc + 10, y + 59), f'{en_n:.0f} İngilizce', font=f(RG, 21), fill=MUTED, anchor='lm')
    d.text((60, H - 36), 'hesap: 8192 / cümle başına ortalama token (FLORES cümleleri ortalama ~130 karakter)', font=f(RG, 22), fill=MUTED, anchor='ld')
    sign(d); im.save(OUT / 'butce.png')


distribution(); per_word(); vocab(); sentence_boxes(); both_ways(); budget()
print('ok')
