"""Küçük Türkçe GPT'yi sıfırdan eğitir (tek RTX 5090, çalışan servislerin yanında boş VRAM'de).
Süre bütçeli: öğrenme oranı (warmup + cosine) geçen EĞİTİM süresinin bütçeye oranına göre azalır.
Doğrulama ve örnek üretme sırasında saat durdurulur. Kayıtlar results/<ad>/ altında.
"""
import argparse, json, math, os, time
from pathlib import Path
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from tokenizers import Tokenizer

ROOT = Path(__file__).resolve().parent
p = argparse.ArgumentParser()
p.add_argument('--name', required=True)
p.add_argument('--d', type=int, default=512)
p.add_argument('--L', type=int, default=8)
p.add_argument('--H', type=int, default=8)
p.add_argument('--ctx', type=int, default=512)
p.add_argument('--batch', type=int, default=64)
p.add_argument('--lr', type=float, default=6e-4)
p.add_argument('--budget_min', type=float, default=60)
p.add_argument('--seed', type=int, default=1337)
p.add_argument('--mem_gb', type=float, default=6.5)
p.add_argument('--sample_at', default='0,1,3,10,30,60')
a = p.parse_args()

OUT = ROOT / 'results' / a.name; OUT.mkdir(parents=True, exist_ok=True)
dev = 'cuda'
torch.cuda.set_per_process_memory_fraction(a.mem_gb * 1024**3 / torch.cuda.get_device_properties(0).total_memory)
torch.manual_seed(a.seed); np.random.seed(a.seed)
torch.backends.cuda.matmul.allow_tf32 = True; torch.backends.cudnn.allow_tf32 = True

tok = Tokenizer.from_file(str(ROOT.parent / 'turkce-tokenizer-egitimi-2026-10-21/tokenizers/tr_100M_32k.json'))
V = tok.get_vocab_size()
train = np.memmap(ROOT / 'data/train.bin', dtype=np.uint16, mode='r')
val = np.memmap(ROOT / 'data/val.bin', dtype=np.uint16, mode='r')
rng = np.random.default_rng(a.seed)


def batch(src, idx=None):
    if idx is None: idx = rng.integers(0, len(src) - a.ctx - 1, a.batch)
    x = torch.from_numpy(np.stack([src[i:i + a.ctx].astype(np.int64) for i in idx]))
    y = torch.from_numpy(np.stack([src[i + 1:i + 1 + a.ctx].astype(np.int64) for i in idx]))
    return x.pin_memory().to(dev, non_blocking=True), y.pin_memory().to(dev, non_blocking=True)


VAL_IDX = [np.random.default_rng(7 + k).integers(0, len(val) - a.ctx - 1, a.batch) for k in range(40)]


class Block(nn.Module):
    def __init__(s):
        super().__init__()
        s.ln1, s.ln2 = nn.LayerNorm(a.d), nn.LayerNorm(a.d)
        s.qkv, s.proj = nn.Linear(a.d, 3 * a.d, bias=False), nn.Linear(a.d, a.d, bias=False)
        s.fc, s.out = nn.Linear(a.d, 4 * a.d, bias=False), nn.Linear(4 * a.d, a.d, bias=False)

    def forward(s, x):
        B, T, C = x.shape
        q, k, v = s.qkv(s.ln1(x)).split(C, 2)
        q, k, v = (t.view(B, T, a.H, C // a.H).transpose(1, 2) for t in (q, k, v))
        y = F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2).reshape(B, T, C)
        x = x + s.proj(y)
        return x + s.out(F.gelu(s.fc(s.ln2(x))))


class GPT(nn.Module):
    def __init__(s):
        super().__init__()
        s.wte, s.wpe = nn.Embedding(V, a.d), nn.Embedding(a.ctx, a.d)
        s.blocks = nn.ModuleList(Block() for _ in range(a.L))
        s.ln = nn.LayerNorm(a.d)
        s.head = nn.Linear(a.d, V, bias=False); s.head.weight = s.wte.weight  # bağlı gömme
        s.apply(s._init)
        for n, prm in s.named_parameters():
            if n.endswith('proj.weight') or n.endswith('out.weight'):
                nn.init.normal_(prm, 0, 0.02 / math.sqrt(2 * a.L))

    def _init(s, m):
        if isinstance(m, (nn.Linear, nn.Embedding)): nn.init.normal_(m.weight, 0, 0.02)

    def forward(s, idx, y=None):
        x = s.wte(idx) + s.wpe(torch.arange(idx.shape[1], device=idx.device))
        for b in s.blocks: x = b(x)
        logits = s.head(s.ln(x))
        if y is None: return logits
        return F.cross_entropy(logits.float().view(-1, V), y.view(-1))


model = GPT().to(dev)
n_params = sum(prm.numel() for prm in model.parameters())
n_emb = model.wte.weight.numel() + model.wpe.weight.numel()
decay = [prm for prm in model.parameters() if prm.dim() >= 2]
nodecay = [prm for prm in model.parameters() if prm.dim() < 2]
opt = torch.optim.AdamW([{'params': decay, 'weight_decay': 0.1}, {'params': nodecay, 'weight_decay': 0.0}],
                        lr=a.lr, betas=(0.9, 0.95), fused=True)
cmodel = torch.compile(model)

PROMPTS = ["Türkiye'nin başkenti", "Fotosentez, bitkilerin", "İstanbul Boğazı", "1923 yılında", "Yapay zekâ"]


@torch.no_grad()
def evaluate():
    model.eval(); tot = 0.0
    for idx in VAL_IDX:
        x, y = batch(val, idx)
        with torch.autocast('cuda', dtype=torch.bfloat16): tot += cmodel(x, y).item()
    model.train(); return tot / len(VAL_IDX)


@torch.no_grad()
def sample(n_new=48):
    model.eval(); outs = []
    g = torch.Generator(device=dev).manual_seed(2026)
    for pr in PROMPTS:
        ids = torch.tensor([tok.encode(pr).ids], device=dev)
        for _ in range(n_new):
            with torch.autocast('cuda', dtype=torch.bfloat16): lg = model(ids[:, -a.ctx:])[:, -1].float() / 0.8
            v, _ = torch.topk(lg, 50); lg[lg < v[:, [-1]]] = -float('inf')
            nxt = torch.multinomial(F.softmax(lg, -1), 1, generator=g)
            if nxt.item() == 0: break
            ids = torch.cat([ids, nxt], 1)
        outs.append({'prompt': pr, 'output': tok.decode(ids[0].tolist())})
    model.train(); return outs


cfg = vars(a) | {'params': n_params, 'params_non_embedding': n_params - n_emb, 'vocab': V,
                 'tokens_per_step': a.batch * a.ctx, 'train_tokens_available': len(train),
                 'torch': torch.__version__, 'gpu': torch.cuda.get_device_name(0)}
json.dump(cfg, open(OUT / 'config.json', 'w'), indent=1)
print(json.dumps(cfg), flush=True)
log = open(OUT / 'log.jsonl', 'w')
W = lambda r: (log.write(json.dumps(r) + '\n'), log.flush())

budget = a.budget_min * 60
marks = sorted(float(m) for m in a.sample_at.split(','))
samples = [{'minute': 0.0, 'step': 0, 'tokens': 0, 'val_loss': None, 'items': sample()}]
marks = [m for m in marks if m > 0]
t_train, step, tokens = 0.0, 0, 0
next_eval = 0.0
compile_s = None
wall0 = time.time()
while t_train < budget:
    t0 = time.time()
    frac = t_train / budget
    lr = a.lr * min(1.0, (step + 1) / 300) * (0.1 + 0.9 * 0.5 * (1 + math.cos(math.pi * frac)))
    for gp in opt.param_groups: gp['lr'] = lr
    x, y = batch(train)
    with torch.autocast('cuda', dtype=torch.bfloat16): loss = cmodel(x, y)
    loss.backward()
    gn = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
    opt.step(); opt.zero_grad(set_to_none=True)
    step += 1; tokens += a.batch * a.ctx
    if step % 20 == 0 or step <= 3:
        lv = loss.item()  # senkronize eder
    else:
        lv = None
    dt = time.time() - t0
    if step == 1: compile_s = dt; dt = 0.0  # derleme süresi ayrı kaydedilir
    t_train += dt
    if lv is not None:
        W({'step': step, 'tokens': tokens, 'train_s': t_train, 'loss': lv, 'lr': lr, 'gnorm': float(gn)})
    if t_train >= next_eval or t_train >= budget:
        vl = evaluate(); next_eval += 60
        W({'step': step, 'tokens': tokens, 'train_s': t_train, 'val_loss': vl,
           'mem_gb': torch.cuda.max_memory_allocated() / 1024**3})
        print(f'{a.name} step {step} {t_train/60:.1f}dk tok {tokens/1e6:.0f}M val {vl:.3f}', flush=True)
    if marks and t_train >= marks[0] * 60:
        samples.append({'minute': marks.pop(0), 'step': step, 'tokens': tokens, 'val_loss': evaluate(),
                        'items': sample()})
        json.dump(samples, open(OUT / 'samples.json', 'w'), ensure_ascii=False, indent=1)
final_val = evaluate()
samples.append({'minute': t_train / 60, 'step': step, 'tokens': tokens, 'val_loss': final_val, 'items': sample(),
                'final': True})
json.dump(samples, open(OUT / 'samples.json', 'w'), ensure_ascii=False, indent=1)
summary = cfg | {'steps': step, 'tokens_seen': tokens, 'epochs': tokens / len(train), 'train_seconds': t_train,
                 'wall_seconds': time.time() - wall0, 'compile_first_step_s': compile_s, 'final_val_loss': final_val,
                 'final_val_ppl': math.exp(final_val), 'tokens_per_s': tokens / t_train,
                 'max_mem_gb': torch.cuda.max_memory_allocated() / 1024**3,
                 'train_flops_6ND': 6 * n_params * tokens}
json.dump(summary, open(OUT / 'summary.json', 'w'), indent=1)
torch.save({'model': model.state_dict(), 'cfg': cfg}, ROOT / 'ckpt' / f'{a.name}.pt')
print(json.dumps(summary), flush=True)
