"""Türkçe Wikipedia (wikimedia/wikipedia 20231101.tr, iki parquet parçası, CC BY-SA 4.0) →
kendi 32k Türkçe tokenizer'ımla (21 Ekim yazısı, tr_100M_32k) token dizisi.
Makaleler sabit tohumla karıştırılır; %0,5'i doğrulama için ayrılır (eğitimde hiç görülmez).
Çıktı: data/train.bin, data/val.bin (uint16), results/data.json
"""
import hashlib, json, random, time
from pathlib import Path
import numpy as np
import pyarrow.parquet as pq
from tokenizers import Tokenizer

ROOT = Path(__file__).resolve().parent
TOK = ROOT.parent / 'turkce-tokenizer-egitimi-2026-10-21/tokenizers/tr_100M_32k.json'
REV = 'b04c8d1ceb2f5cd4588862100d08de323dccfbaa'
FILES = [ROOT / 'data/wiki/tr-0.parquet', ROOT / 'data/wiki/tr-1.parquet']
sha = lambda p: hashlib.sha256(Path(p).read_bytes()).hexdigest()

t0 = time.time()
arts = []
for f in FILES:
    arts += [a for a in pq.read_table(f, columns=['text']).column('text').to_pylist() if len(a) > 200]
random.Random(20261023).shuffle(arts)
nval = len(arts) // 200
val, train = arts[:nval], arts[nval:]
tok = Tokenizer.from_file(str(TOK))
EOT = tok.token_to_id('<|endoftext|>')


def encode(docs, out):
    # Akışlı yazım: Python int listesi tutmaz (eski sürüm cron'un 4 GB sınırında OOM oldu).
    n = 0
    with open(out, 'wb') as fh:
        for i in range(0, len(docs), 2000):
            enc = tok.encode_batch(docs[i:i + 2000])
            a = np.concatenate([np.append(np.asarray(e.ids, dtype=np.uint16), np.uint16(EOT)) for e in enc])
            a.tofile(fh); n += len(a)
    return n


meta = {'source': 'huggingface.co/datasets/wikimedia/wikipedia 20231101.tr', 'revision': REV,
        'license': 'CC BY-SA 4.0',
        'files': {f.name: {'sha256': sha(f), 'bytes': f.resolve().stat().st_size} for f in FILES},
        'tokenizer': {'path': str(TOK.relative_to(ROOT.parent)), 'sha256': sha(TOK), 'vocab': tok.get_vocab_size()},
        'shuffle_seed': 20261023, 'min_article_chars': 200,
        'articles_total': len(arts), 'articles_train': len(train), 'articles_val': len(val),
        'chars_train': sum(map(len, train)), 'chars_val': sum(map(len, val))}
meta['tokens_train'] = encode(train, ROOT / 'data/train.bin')
meta['tokens_val'] = encode(val, ROOT / 'data/val.bin')
meta['chars_per_token_train'] = meta['chars_train'] / meta['tokens_train']
meta['seconds'] = time.time() - t0
json.dump(meta, open(ROOT / 'results/data.json', 'w'), ensure_ascii=False, indent=1)
print(json.dumps(meta, ensure_ascii=False, indent=1))
