refactor: switch eval datasets to HuggingFace source

- Replace GitHub/berkeley direct downloads with HF datasets API
- MMLU: cais/mmlu (all config), map val->validation split, write per-subject CSV
- HumanEval: openai/openai_humaneval
- IFEval: google/IFEval
- Enables HF_ENDPOINT mirror for faster downloads in CN

scripts/eval/evaluate_humaneval.py

@@ -20,6 +20,7 @@ from typing import Dict, Iterator, List, Optional, Sequence, Tuple
 import numpy as np
 import torch
 import tqdm
+from datasets import load_dataset
 
 from astrai.inference import InferenceEngine
 from astrai.model import AutoModel
@@ -29,9 +30,7 @@ from astrai.tokenize import AutoTokenizer
 # Config
 # ---------------------------------------------------------------------------
 
-HUMANEVAL_URL = (
-    "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz"
-)
+HUMANEVAL_HF_DATASET = "openai/openai_humaneval"
 
 STOP_SEQUENCES = [
     "\nclass ",
@@ -64,21 +63,16 @@ class EvalConfig:
     problem_indices: Optional[List[int]] = None
 
 
-def download(url: str, path: str):
+def download(path: str):
     if os.path.exists(path):
         return
-    import gzip
-    import urllib.request
-
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    print(f"Downloading {url} ...")
-    tmp = path + ".tmp"
-    urllib.request.urlretrieve(url, tmp)
-    with gzip.open(tmp, "rb") as f_in:
-        with open(path, "wb") as f_out:
-            f_out.write(f_in.read())
-    os.remove(tmp)
-    print(f"  saved to {path}")
+    print(f"Downloading HumanEval from HuggingFace ({HUMANEVAL_HF_DATASET}) ...")
+    ds = load_dataset(HUMANEVAL_HF_DATASET, split="test")
+    with open(path, "w", encoding="utf-8") as f:
+        for item in ds:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+    print(f"  saved {len(ds)} problems to {path}")
 
 
 def load_jsonl(path: str) -> List[dict]:
@@ -318,7 +312,7 @@ def run_pipeline(cfg: EvalConfig) -> Dict:
         with open(cfg.test_only, encoding="utf-8") as f:
             generated = json.load(f)
     else:
-        download(HUMANEVAL_URL, cfg.data_path)
+        download(cfg.data_path)
 
         problems = load_jsonl(cfg.data_path)
         if cfg.problem_indices:

scripts/eval/evaluate_ifeval.py

@@ -14,21 +14,17 @@ import argparse
 import json
 import os
 import re
-import urllib.request
 from typing import Callable, Dict, List, Optional
 
 import torch
 import tqdm
+from datasets import load_dataset
 
 from astrai.inference import InferenceEngine
 from astrai.model import AutoModel
 from astrai.tokenize import AutoTokenizer
 
-IFEVAL_URL = (
-    "https://raw.githubusercontent.com/google-research/"
-    "google-research/master/instruction_following_eval/data/input_data.jsonl"
-)
-
+IFEVAL_HF_DATASET = "google/IFEval"
 CONSTRAINT_VERIFIERS: Dict[str, Callable[[str, dict], bool]] = {}
 
 
@@ -310,15 +306,12 @@ def download_ifeval(data_path: str):
     if os.path.exists(data_path):
         return
     os.makedirs(os.path.dirname(data_path) or ".", exist_ok=True)
-    print(f"Downloading IFEval from {IFEVAL_URL} ...")
-    tmp = data_path + ".tmp"
-    urllib.request.urlretrieve(IFEVAL_URL, tmp)
-    with open(tmp, "rb") as f_in:
-        content = f_in.read()
-    with open(data_path, "wb") as f_out:
-        f_out.write(content)
-    os.remove(tmp)
-    print(f"  saved to {data_path}")
+    print(f"Downloading IFEval from HuggingFace ({IFEVAL_HF_DATASET}) ...")
+    ds = load_dataset(IFEVAL_HF_DATASET, split="train")
+    with open(data_path, "w", encoding="utf-8") as f:
+        for item in ds:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+    print(f"  saved {len(ds)} items to {data_path}")
 
 
 def load_problems(data_path: str) -> List[dict]:

scripts/eval/evaluate_mmlu.py

@@ -4,18 +4,17 @@ import argparse
 import csv
 import json
 import os
-import shutil
-import tarfile
+from collections import defaultdict
 
-import requests
 import torch
 import torch.nn.functional as F
 import tqdm
+from datasets import load_dataset
 
 from astrai.model import AutoModel
 from astrai.tokenize import AutoTokenizer
 
-MMLU_URL = "https://people.eecs.berkeley.edu/~hendrycks/data.tar"
+MMLU_HF_DATASET = "cais/mmlu"
 MMLU_SUBJECTS = [
     "abstract_algebra",
     "anatomy",
@@ -77,38 +76,40 @@ MMLU_SUBJECTS = [
 ]
 
 
-def _download_and_extract(url: str, data_dir: str):
-    tar_path = os.path.join(data_dir, "data.tar")
-    os.makedirs(data_dir, exist_ok=True)
-    print(f"Downloading MMLU data from {url}...")
-    resp = requests.get(url, stream=True, timeout=300)
-    resp.raise_for_status()
-    total = int(resp.headers.get("content-length", 0))
-    with tqdm.tqdm(total=total, unit="B", unit_scale=True, desc="  Download") as bar:
-        with open(tar_path, "wb") as f:
-            for chunk in resp.iter_content(chunk_size=8192):
-                f.write(chunk)
-                bar.update(len(chunk))
-    print("Extracting...")
-    with tarfile.open(tar_path, "r") as tf:
-        tf.extractall(data_dir)
-    os.remove(tar_path)
+def _write_subject_csv(data_dir: str, split: str, subject: str, rows: list[dict]):
+    split_dir = os.path.join(data_dir, split)
+    os.makedirs(split_dir, exist_ok=True)
+    path = os.path.join(split_dir, f"{subject}_{split}.csv")
+    with open(path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f)
+        for row in rows:
+            writer.writerow(row)
 
 
 def download_mmlu(data_dir: str):
-    _download_and_extract(MMLU_URL, data_dir)
-    src = os.path.join(data_dir, "data")
-    if os.path.exists(src):
-        for item in os.listdir(src):
-            src_item = os.path.join(src, item)
-            dst_item = os.path.join(data_dir, item)
-            if os.path.exists(dst_item):
-                if os.path.isdir(dst_item):
-                    shutil.rmtree(dst_item)
-                else:
-                    os.remove(dst_item)
-            os.rename(src_item, dst_item)
-        os.rmdir(src)
+    print(f"Downloading MMLU from HuggingFace ({MMLU_HF_DATASET}) ...")
+    letters = ("A", "B", "C", "D")
+    split_map = {"dev": "dev", "val": "validation", "test": "test"}
+    for local_split, hf_split in split_map.items():
+        ds = load_dataset(MMLU_HF_DATASET, "all", split=hf_split)
+        grouped: dict[str, list[dict]] = defaultdict(list)
+        for item in tqdm.tqdm(ds, desc=f"  {local_split}", leave=False):
+            subject = item["subject"]
+            choices = item["choices"]
+            ans_letter = letters[item["answer"]]
+            grouped[subject].append(
+                [
+                    item["question"],
+                    f"A){choices[0]}",
+                    f"B){choices[1]}",
+                    f"C){choices[2]}",
+                    f"D){choices[3]}",
+                    ans_letter,
+                ]
+            )
+        for subject, rows in grouped.items():
+            _write_subject_csv(data_dir, local_split, subject, rows)
+        print(f"  {local_split}: {len(ds)} items, {len(grouped)} subjects")
     print(f"MMLU data saved to {data_dir}")