"""Eğitim metinleri: Wikipedia (wikimedia/wikipedia 20231101, CC BY-SA 4.0) ilk parquet parçaları.
Makaleler sabit tohumla karıştırılır, karakter bütçesine göre ilk N makale alınır.
Çıktı: data/text/{tr,en}_{N}M.txt ve data/text/mix_100M.txt (50M TR + 50M EN), results/data.json
FLORES-200 cümlelerinin eğitim metnine birebir sızıp sızmadığı da kontrol edilir.
"""
import hashlib, json, random
from pathlib import Path
import pyarrow.parquet as pq

ROOT = Path(__file__).resolve().parent
TXT = ROOT / 'data/text'; TXT.mkdir(parents=True, exist_ok=True)
REV = 'b04c8d1ceb2f5cd4588862100d08de323dccfbaa'
SRC = {'tr': ROOT / 'data/wiki/20231101.tr/train-00000-of-00002.parquet',
       'en': ROOT / 'data/wiki/20231101.en/train-00000-of-00041.parquet'}
sha = lambda p: hashlib.sha256(p.read_bytes()).hexdigest()


def articles(lang):
    t = pq.read_table(SRC[lang], columns=['text'])
    arts = [a for a in t.column('text').to_pylist() if len(a) > 200]
    random.Random(20261021).shuffle(arts)
    return arts


def take(arts, nchars):
    out, n = [], 0
    for a in arts:
        if n >= nchars: break
        out.append(a); n += len(a) + 1
    return '\n'.join(out)


meta = {'source': 'huggingface.co/datasets/wikimedia/wikipedia', 'revision': REV, 'license': 'CC BY-SA 4.0',
        'files': {k: {'path': str(v.relative_to(ROOT)), 'sha256': sha(v), 'bytes': v.stat().st_size} for k, v in SRC.items()},
        'shuffle_seed': 20261021, 'min_article_chars': 200, 'subsets': {}}
A = {k: articles(k) for k in SRC}
for k in A: meta[f'{k}_articles_available'] = len(A[k]); meta[f'{k}_chars_available'] = sum(map(len, A[k]))
plan = [('tr', 1), ('tr', 3), ('tr', 10), ('tr', 30), ('tr', 100), ('tr', 300), ('en', 100)]
for lang, m in plan:
    s = take(A[lang], m * 1_000_000)
    p = TXT / f'{lang}_{m}M.txt'; p.write_text(s, encoding='utf-8')
    meta['subsets'][p.name] = {'chars': len(s), 'bytes': p.stat().st_size, 'sha256': sha(p)}
    print(p.name, len(s), flush=True)
mix = take(A['tr'], 50_000_000) + '\n' + take(A['en'], 50_000_000)
p = TXT / 'mix_100M.txt'; p.write_text(mix, encoding='utf-8')
meta['subsets'][p.name] = {'chars': len(mix), 'bytes': p.stat().st_size, 'sha256': sha(p)}

# FLORES sızıntı kontrolü (eğitimde kullanılan en büyük TR alt kümesinde birebir cümle araması)
F = ROOT / 'data/flores200_dataset'
fl = (F / 'dev/tur_Latn.dev').read_text().splitlines() + (F / 'devtest/tur_Latn.devtest').read_text().splitlines()
big = (TXT / 'tr_300M.txt').read_text(encoding='utf-8')
meta['flores_tr_sentences'] = len(fl)
meta['flores_tr_exact_in_tr_300M'] = sum(1 for s in fl if s in big)
json.dump(meta, open(ROOT / 'results/data.json', 'w'), ensure_ascii=False, indent=1)
print(json.dumps({k: v for k, v in meta.items() if k != 'subsets'}, ensure_ascii=False))
