"""BPE tokenizer eğitimi (HF tokenizers, GPT tarzı byte-level BPE).
Her eğitim ayrı süreçte çalışır; duvar saati süresi ve tepe RSS (ru_maxrss) kaydedilir.
Kullanım: python train.py  → tokenizers/*.json, results/train.json
"""
import json, resource, subprocess, sys, time
from pathlib import Path

ROOT = Path(__file__).resolve().parent
TOK = ROOT / 'tokenizers'; TOK.mkdir(exist_ok=True)
K = 1024
RUNS = [('tr_100M.txt', v) for v in (8 * K, 16 * K, 32 * K, 64 * K, 128 * K, 256 * K)]
RUNS += [(f'tr_{m}M.txt', 32 * K) for m in (1, 3, 10, 30, 300)]
RUNS += [('en_100M.txt', 32 * K), ('mix_100M.txt', 32 * K), ('mix_100M.txt', 64 * K)]

CHILD = r'''
import sys, time, resource
from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders
src, vocab, out = sys.argv[1], int(sys.argv[2]), sys.argv[3]
tok = Tokenizer(models.BPE())
tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=True)
tok.decoder = decoders.ByteLevel()
tr = trainers.BpeTrainer(vocab_size=vocab, min_frequency=2, special_tokens=['<|endoftext|>'],
                         initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), show_progress=False)
t0 = time.time(); tok.train([src], tr); dt = time.time() - t0
tok.save(out)
print(dt, tok.get_vocab_size(), resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
'''

res = []
for src, vocab in RUNS:
    name = f"{src[:-4]}_{vocab // K}k"
    out = TOK / f'{name}.json'
    t0 = time.time()
    p = subprocess.run([sys.executable, '-c', CHILD, str(ROOT / 'data/text' / src), str(vocab), str(out)],
                       capture_output=True, text=True, check=True)
    dt, vs, rss = p.stdout.split()
    r = dict(name=name, corpus=src, vocab_target=vocab, vocab_actual=int(vs), train_seconds=round(float(dt), 2),
             wall_seconds=round(time.time() - t0, 2), peak_rss_mib=round(int(rss) / 1024, 1))
    res.append(r); print(r, flush=True)
    json.dump(dict(runs=res, threads='RAYON varsayılan (32 mantıksal çekirdek)', cpu='AMD Ryzen Threadripper PRO 5955WX',
                   library='tokenizers ' + __import__('tokenizers').__version__, measured_at=time.strftime('%Y-%m-%d %H:%M %z')),
              open(ROOT / 'results/train.json', 'w'), ensure_ascii=False, indent=1)
