From 2565755e45da5ba47843af72b8ef928088ec6830 Mon Sep 17 00:00:00 2001 From: ViperEkura <3081035982@qq.com> Date: Sat, 18 Jul 2026 00:07:01 +0800 Subject: [PATCH] refactor: switch eval datasets to HuggingFace source - Replace GitHub/berkeley direct downloads with HF datasets API - MMLU: cais/mmlu (all config), map val->validation split, write per-subject CSV - HumanEval: openai/openai_humaneval - IFEval: google/IFEval - Enables HF_ENDPOINT mirror for faster downloads in CN --- scripts/eval/evaluate_humaneval.py | 26 +++++------- scripts/eval/evaluate_ifeval.py | 23 ++++------ scripts/eval/evaluate_mmlu.py | 67 +++++++++++++++--------------- 3 files changed, 52 insertions(+), 64 deletions(-) diff --git a/scripts/eval/evaluate_humaneval.py b/scripts/eval/evaluate_humaneval.py index e7feb12..2ae5712 100644 --- a/scripts/eval/evaluate_humaneval.py +++ b/scripts/eval/evaluate_humaneval.py @@ -20,6 +20,7 @@ from typing import Dict, Iterator, List, Optional, Sequence, Tuple import numpy as np import torch import tqdm +from datasets import load_dataset from astrai.inference import InferenceEngine from astrai.model import AutoModel @@ -29,9 +30,7 @@ from astrai.tokenize import AutoTokenizer # Config # --------------------------------------------------------------------------- -HUMANEVAL_URL = ( - "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz" -) +HUMANEVAL_HF_DATASET = "openai/openai_humaneval" STOP_SEQUENCES = [ "\nclass ", @@ -64,21 +63,16 @@ class EvalConfig: problem_indices: Optional[List[int]] = None -def download(url: str, path: str): +def download(path: str): if os.path.exists(path): return - import gzip - import urllib.request - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - print(f"Downloading {url} ...") - tmp = path + ".tmp" - urllib.request.urlretrieve(url, tmp) - with gzip.open(tmp, "rb") as f_in: - with open(path, "wb") as f_out: - f_out.write(f_in.read()) - os.remove(tmp) - print(f" saved to {path}") + print(f"Downloading HumanEval from HuggingFace ({HUMANEVAL_HF_DATASET}) ...") + ds = load_dataset(HUMANEVAL_HF_DATASET, split="test") + with open(path, "w", encoding="utf-8") as f: + for item in ds: + f.write(json.dumps(item, ensure_ascii=False) + "\n") + print(f" saved {len(ds)} problems to {path}") def load_jsonl(path: str) -> List[dict]: @@ -318,7 +312,7 @@ def run_pipeline(cfg: EvalConfig) -> Dict: with open(cfg.test_only, encoding="utf-8") as f: generated = json.load(f) else: - download(HUMANEVAL_URL, cfg.data_path) + download(cfg.data_path) problems = load_jsonl(cfg.data_path) if cfg.problem_indices: diff --git a/scripts/eval/evaluate_ifeval.py b/scripts/eval/evaluate_ifeval.py index 304641f..f924613 100644 --- a/scripts/eval/evaluate_ifeval.py +++ b/scripts/eval/evaluate_ifeval.py @@ -14,21 +14,17 @@ import argparse import json import os import re -import urllib.request from typing import Callable, Dict, List, Optional import torch import tqdm +from datasets import load_dataset from astrai.inference import InferenceEngine from astrai.model import AutoModel from astrai.tokenize import AutoTokenizer -IFEVAL_URL = ( - "https://raw.githubusercontent.com/google-research/" - "google-research/master/instruction_following_eval/data/input_data.jsonl" -) - +IFEVAL_HF_DATASET = "google/IFEval" CONSTRAINT_VERIFIERS: Dict[str, Callable[[str, dict], bool]] = {} @@ -310,15 +306,12 @@ def download_ifeval(data_path: str): if os.path.exists(data_path): return os.makedirs(os.path.dirname(data_path) or ".", exist_ok=True) - print(f"Downloading IFEval from {IFEVAL_URL} ...") - tmp = data_path + ".tmp" - urllib.request.urlretrieve(IFEVAL_URL, tmp) - with open(tmp, "rb") as f_in: - content = f_in.read() - with open(data_path, "wb") as f_out: - f_out.write(content) - os.remove(tmp) - print(f" saved to {data_path}") + print(f"Downloading IFEval from HuggingFace ({IFEVAL_HF_DATASET}) ...") + ds = load_dataset(IFEVAL_HF_DATASET, split="train") + with open(data_path, "w", encoding="utf-8") as f: + for item in ds: + f.write(json.dumps(item, ensure_ascii=False) + "\n") + print(f" saved {len(ds)} items to {data_path}") def load_problems(data_path: str) -> List[dict]: diff --git a/scripts/eval/evaluate_mmlu.py b/scripts/eval/evaluate_mmlu.py index 25b2eba..4732488 100644 --- a/scripts/eval/evaluate_mmlu.py +++ b/scripts/eval/evaluate_mmlu.py @@ -4,18 +4,17 @@ import argparse import csv import json import os -import shutil -import tarfile +from collections import defaultdict -import requests import torch import torch.nn.functional as F import tqdm +from datasets import load_dataset from astrai.model import AutoModel from astrai.tokenize import AutoTokenizer -MMLU_URL = "https://people.eecs.berkeley.edu/~hendrycks/data.tar" +MMLU_HF_DATASET = "cais/mmlu" MMLU_SUBJECTS = [ "abstract_algebra", "anatomy", @@ -77,38 +76,40 @@ MMLU_SUBJECTS = [ ] -def _download_and_extract(url: str, data_dir: str): - tar_path = os.path.join(data_dir, "data.tar") - os.makedirs(data_dir, exist_ok=True) - print(f"Downloading MMLU data from {url}...") - resp = requests.get(url, stream=True, timeout=300) - resp.raise_for_status() - total = int(resp.headers.get("content-length", 0)) - with tqdm.tqdm(total=total, unit="B", unit_scale=True, desc=" Download") as bar: - with open(tar_path, "wb") as f: - for chunk in resp.iter_content(chunk_size=8192): - f.write(chunk) - bar.update(len(chunk)) - print("Extracting...") - with tarfile.open(tar_path, "r") as tf: - tf.extractall(data_dir) - os.remove(tar_path) +def _write_subject_csv(data_dir: str, split: str, subject: str, rows: list[dict]): + split_dir = os.path.join(data_dir, split) + os.makedirs(split_dir, exist_ok=True) + path = os.path.join(split_dir, f"{subject}_{split}.csv") + with open(path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + for row in rows: + writer.writerow(row) def download_mmlu(data_dir: str): - _download_and_extract(MMLU_URL, data_dir) - src = os.path.join(data_dir, "data") - if os.path.exists(src): - for item in os.listdir(src): - src_item = os.path.join(src, item) - dst_item = os.path.join(data_dir, item) - if os.path.exists(dst_item): - if os.path.isdir(dst_item): - shutil.rmtree(dst_item) - else: - os.remove(dst_item) - os.rename(src_item, dst_item) - os.rmdir(src) + print(f"Downloading MMLU from HuggingFace ({MMLU_HF_DATASET}) ...") + letters = ("A", "B", "C", "D") + split_map = {"dev": "dev", "val": "validation", "test": "test"} + for local_split, hf_split in split_map.items(): + ds = load_dataset(MMLU_HF_DATASET, "all", split=hf_split) + grouped: dict[str, list[dict]] = defaultdict(list) + for item in tqdm.tqdm(ds, desc=f" {local_split}", leave=False): + subject = item["subject"] + choices = item["choices"] + ans_letter = letters[item["answer"]] + grouped[subject].append( + [ + item["question"], + f"A){choices[0]}", + f"B){choices[1]}", + f"C){choices[2]}", + f"D){choices[3]}", + ans_letter, + ] + ) + for subject, rows in grouped.items(): + _write_subject_csv(data_dir, local_split, subject, rows) + print(f" {local_split}: {len(ds)} items, {len(grouped)} subjects") print(f"MMLU data saved to {data_dir}")