feat: 新增 IFEval 指令遵循评测
- 实现 25 种正则约束 verifier - 将评测脚本从 scripts/tools/ 移至 scripts/eval/
This commit is contained in:
@@ -0,0 +1,334 @@
|
||||
"""HumanEval code generation benchmark.
|
||||
|
||||
Generates n completions per problem, extracts function bodies, executes
|
||||
against hidden tests, and computes pass@k.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/tools/evaluate_humaneval.py --param_path ./params \
|
||||
--data_path HumanEval.jsonl.gz --output results.json \
|
||||
--num_samples 200 --temperature 0.8 --max_tokens 512
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from math import prod
|
||||
from multiprocessing import Process, Queue
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import tqdm
|
||||
|
||||
from astrai.inference import InferenceEngine
|
||||
from astrai.model import AutoModel
|
||||
from astrai.tokenize import AutoTokenizer
|
||||
|
||||
HUMANEVAL_URL = (
|
||||
"https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz"
|
||||
)
|
||||
|
||||
_STOP_SEQUENCES = [
|
||||
"\nclass ",
|
||||
"\ndef ",
|
||||
"\n# ",
|
||||
"\nif __name__",
|
||||
"\nprint(",
|
||||
"\n\n\n",
|
||||
]
|
||||
|
||||
|
||||
def _download_humaneval(data_path: str):
|
||||
if os.path.exists(data_path):
|
||||
return
|
||||
import gzip
|
||||
import urllib.request
|
||||
|
||||
os.makedirs(os.path.dirname(data_path) or ".", exist_ok=True)
|
||||
print(f"Downloading HumanEval from {HUMANEVAL_URL} ...")
|
||||
tmp = data_path + ".tmp"
|
||||
urllib.request.urlretrieve(HUMANEVAL_URL, tmp)
|
||||
with gzip.open(tmp, "rb") as f_in:
|
||||
with open(data_path, "wb") as f_out:
|
||||
f_out.write(f_in.read())
|
||||
os.remove(tmp)
|
||||
print(f" saved to {data_path}")
|
||||
|
||||
|
||||
def _load_problems(data_path: str) -> List[dict]:
|
||||
problems = []
|
||||
with open(data_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
problems.append(json.loads(line))
|
||||
return problems
|
||||
|
||||
|
||||
def _extract_function_body(code: str, entry_point: str) -> Optional[str]:
|
||||
"""Extract the function body from a completion."""
|
||||
pattern = rf"def\s+{re.escape(entry_point)}\b[^:]*:"
|
||||
match = re.search(pattern, code)
|
||||
if not match:
|
||||
# Use the full code as-is if we can't find the function
|
||||
return code
|
||||
|
||||
body_start = match.end()
|
||||
lines = code[body_start:].split("\n")
|
||||
body_lines = []
|
||||
started = False
|
||||
|
||||
for line in lines:
|
||||
stripped = line.rstrip()
|
||||
if not stripped and not started:
|
||||
continue
|
||||
if not stripped and started:
|
||||
body_lines.append("")
|
||||
continue
|
||||
if not started:
|
||||
started = True
|
||||
if stripped.lstrip() == stripped and started:
|
||||
break
|
||||
body_lines.append(stripped)
|
||||
|
||||
body = "\n".join(body_lines)
|
||||
if not body.strip():
|
||||
return None
|
||||
return body
|
||||
|
||||
|
||||
def _trim_stop_sequences(text: str) -> str:
|
||||
for stop in _STOP_SEQUENCES:
|
||||
idx = text.find(stop)
|
||||
if idx != -1:
|
||||
text = text[:idx]
|
||||
return text
|
||||
|
||||
|
||||
def _execute_code(problem: dict, completion: str, timeout: float = 3.0) -> bool:
|
||||
"""Run the completion against hidden tests in a subprocess."""
|
||||
|
||||
def _worker(queue, full_code):
|
||||
try:
|
||||
namespace = {}
|
||||
exec(full_code, namespace)
|
||||
check = namespace.get("check")
|
||||
if check is None:
|
||||
queue.put(False)
|
||||
return
|
||||
check(namespace.get(problem["entry_point"]))
|
||||
queue.put(True)
|
||||
except Exception:
|
||||
queue.put(False)
|
||||
|
||||
full_code = problem["prompt"] + completion + "\n" + problem["test"]
|
||||
|
||||
queue: Queue = Queue()
|
||||
proc = Process(target=_worker, args=(queue, full_code))
|
||||
proc.start()
|
||||
proc.join(timeout)
|
||||
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
proc.join()
|
||||
return False
|
||||
|
||||
try:
|
||||
return queue.get_nowait()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _pass_at_k(n: int, c: int, k: int) -> float:
|
||||
"""Unbiased estimator of pass@k."""
|
||||
if n - c < k:
|
||||
return 1.0
|
||||
return 1.0 - float(prod(1.0 - k / np.arange(n - c + 1, n + 1)))
|
||||
|
||||
|
||||
def _deduplicate(completions: List[str]) -> List[str]:
|
||||
seen = set()
|
||||
unique = []
|
||||
for c in completions:
|
||||
if c not in seen:
|
||||
seen.add(c)
|
||||
unique.append(c)
|
||||
return unique
|
||||
|
||||
|
||||
def _generate(
|
||||
engine: InferenceEngine,
|
||||
prompt: str,
|
||||
num_samples: int,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
batch_size: int,
|
||||
) -> List[str]:
|
||||
batches = [prompt] * min(batch_size, num_samples)
|
||||
completions = []
|
||||
remaining = num_samples
|
||||
|
||||
while remaining > 0:
|
||||
current = min(batch_size, remaining)
|
||||
batch_prompts = batches[:current]
|
||||
outputs = engine.generate(
|
||||
prompt=batch_prompts,
|
||||
stream=False,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
)
|
||||
if isinstance(outputs, str):
|
||||
outputs = [outputs]
|
||||
completions.extend(outputs)
|
||||
remaining -= current
|
||||
|
||||
return _deduplicate(completions)
|
||||
|
||||
|
||||
def evaluate(
|
||||
engine: InferenceEngine,
|
||||
problems: List[dict],
|
||||
num_samples: int,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
batch_size: int,
|
||||
k_values: Tuple[int, ...] = (1, 10, 100),
|
||||
) -> Dict:
|
||||
results = {}
|
||||
all_pass_at_k = {k: [] for k in k_values}
|
||||
|
||||
for problem in tqdm.tqdm(problems, desc="HumanEval", unit="problem"):
|
||||
task_id = problem["task_id"]
|
||||
prompt = problem["prompt"]
|
||||
entry_point = problem["entry_point"]
|
||||
|
||||
raw_completions = _generate(
|
||||
engine,
|
||||
prompt,
|
||||
num_samples,
|
||||
max_tokens,
|
||||
temperature,
|
||||
top_p,
|
||||
top_k,
|
||||
batch_size,
|
||||
)
|
||||
|
||||
completions = []
|
||||
for raw in raw_completions:
|
||||
trimmed = _trim_stop_sequences(raw)
|
||||
body = _extract_function_body(trimmed, entry_point)
|
||||
if body:
|
||||
completions.append(body)
|
||||
|
||||
passed = 0
|
||||
for comp in completions:
|
||||
if _execute_code(problem, comp):
|
||||
passed += 1
|
||||
|
||||
n = len(completions)
|
||||
c = passed
|
||||
result = {"task_id": task_id, "n": n, "passed": c}
|
||||
for k in k_values:
|
||||
result[f"pass@{k}"] = round(_pass_at_k(n, c, k), 4)
|
||||
all_pass_at_k[k].append(_pass_at_k(n, c, k))
|
||||
results[task_id] = result
|
||||
|
||||
summary = {}
|
||||
for k in k_values:
|
||||
vals = all_pass_at_k[k]
|
||||
summary[f"pass@{k}"] = round(float(np.mean(vals)), 4)
|
||||
results["_summary"] = summary
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="HumanEval benchmark")
|
||||
parser.add_argument(
|
||||
"--param_path", type=str, default="./params", help="Model directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--data_path",
|
||||
type=str,
|
||||
default="./humaneval/HumanEval.jsonl",
|
||||
help="HumanEval JSONL file (auto-download if missing)",
|
||||
)
|
||||
parser.add_argument("--output", type=str, default=None, help="Output JSON path")
|
||||
parser.add_argument(
|
||||
"--num_samples",
|
||||
type=int,
|
||||
default=200,
|
||||
help="Completions per problem",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max_tokens", type=int, default=512, help="Max generation tokens"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--temperature", type=float, default=0.8, help="Sampling temperature"
|
||||
)
|
||||
parser.add_argument("--top_p", type=float, default=0.95, help="Top-p sampling")
|
||||
parser.add_argument("--top_k", type=int, default=50, help="Top-k sampling")
|
||||
parser.add_argument(
|
||||
"--batch_size", type=int, default=1, help="Inference batch size"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--problems",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Specific problem indices (0-based)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
_download_humaneval(args.data_path)
|
||||
problems = _load_problems(args.data_path)
|
||||
if args.problems:
|
||||
problems = [problems[i] for i in args.problems if i < len(problems)]
|
||||
|
||||
model = AutoModel.from_pretrained(args.param_path)
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.param_path)
|
||||
model.to(device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
engine = InferenceEngine(
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
max_batch_size=args.batch_size,
|
||||
)
|
||||
|
||||
results = evaluate(
|
||||
engine=engine,
|
||||
problems=problems,
|
||||
num_samples=args.num_samples,
|
||||
max_tokens=args.max_tokens,
|
||||
temperature=args.temperature,
|
||||
top_p=args.top_p,
|
||||
top_k=args.top_k,
|
||||
batch_size=args.batch_size,
|
||||
k_values=(1, 10, 100),
|
||||
)
|
||||
|
||||
summary = results.pop("_summary")
|
||||
print(f"\n{'=' * 60}")
|
||||
for k, v in summary.items():
|
||||
print(f" {k}: {v:.2%}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
if args.output:
|
||||
results["_summary"] = summary
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||||
print(f"Results saved to {args.output}")
|
||||
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,600 @@
|
||||
"""IFEval instruction-following evaluation benchmark.
|
||||
|
||||
Evaluates model responses against regex-based constraint verifiers.
|
||||
Supports all IFEval constraint types except language detection.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/tools/evaluate_ifeval.py --param_path ./params \
|
||||
--data_path ifeval.jsonl --output results.json \
|
||||
--temperature 0.1 --max_tokens 512
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
import torch
|
||||
import tqdm
|
||||
|
||||
from astrai.inference import InferenceEngine
|
||||
from astrai.model import AutoModel
|
||||
from astrai.tokenize import AutoTokenizer
|
||||
|
||||
IFEVAL_URL = (
|
||||
"https://raw.githubusercontent.com/google-research/"
|
||||
"google-research/master/instruction_following_eval/data/input_data.jsonl"
|
||||
)
|
||||
|
||||
CONSTRAINT_VERIFIERS: Dict[str, Callable[[str, dict], bool]] = {}
|
||||
|
||||
|
||||
def register(instruction_id: str):
|
||||
def decorator(fn):
|
||||
CONSTRAINT_VERIFIERS[instruction_id] = fn
|
||||
return fn
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
@register("keywords:existence")
|
||||
def check_keyword_existence(response: str, kwargs: dict) -> bool:
|
||||
for kw in kwargs["keywords"]:
|
||||
if not re.search(re.escape(kw), response, re.IGNORECASE):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@register("keywords:frequency")
|
||||
def check_keyword_frequency(response: str, kwargs: dict) -> bool:
|
||||
keyword = kwargs["keyword"]
|
||||
frequency = kwargs.get("frequency", 1)
|
||||
relation = kwargs.get("relation", "at least")
|
||||
count = len(re.findall(re.escape(keyword), response, re.IGNORECASE))
|
||||
if relation == "less than":
|
||||
return count < frequency
|
||||
return count >= frequency
|
||||
|
||||
|
||||
@register("keywords:forbidden_words")
|
||||
def check_forbidden_words(response: str, kwargs: dict) -> bool:
|
||||
for word in kwargs["forbidden_words"]:
|
||||
if re.search(r"\b" + re.escape(word) + r"\b", response, re.IGNORECASE):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@register("keywords:letter_frequency")
|
||||
def check_letter_frequency(response: str, kwargs: dict) -> bool:
|
||||
letter = kwargs["letter"].lower()
|
||||
frequency = kwargs.get("let_frequency", 1)
|
||||
relation = kwargs.get("let_relation", "at least")
|
||||
count = response.lower().count(letter)
|
||||
if relation == "less than":
|
||||
return count < frequency
|
||||
return count >= frequency
|
||||
|
||||
|
||||
@register("detectable_content:number_placeholders")
|
||||
def check_placeholders(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_placeholders", 1)
|
||||
placeholders = re.findall(r"\[.*?\]", response)
|
||||
return len(placeholders) >= num
|
||||
|
||||
|
||||
@register("detectable_content:postscript")
|
||||
def check_postscript(response: str, kwargs: dict) -> bool:
|
||||
marker = kwargs.get("postscript_marker", "P.S.")
|
||||
response_lower = response.lower()
|
||||
if marker == "P.P.S":
|
||||
return bool(re.search(r"p\.\s?p\.\s?s", response_lower))
|
||||
elif marker == "P.S.":
|
||||
return bool(re.search(r"p\.\s?s\.", response_lower))
|
||||
else:
|
||||
return bool(re.search(re.escape(marker.lower()), response_lower))
|
||||
|
||||
|
||||
@register("detectable_format:number_bullet_lists")
|
||||
def check_bullet_lists(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_bullets", 1)
|
||||
bullets = re.findall(r"^\s*\*[^\*].*$", response, re.MULTILINE)
|
||||
dashes = re.findall(r"^\s*-.*$", response, re.MULTILINE)
|
||||
return len(bullets) + len(dashes) == num
|
||||
|
||||
|
||||
@register("detectable_format:number_highlighted_sections")
|
||||
def check_highlighted_sections(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_highlights", 1)
|
||||
highlights = re.findall(r"\*[^\n\*]+\*", response)
|
||||
count = 0
|
||||
for h in highlights:
|
||||
if h.strip("*").strip():
|
||||
count += 1
|
||||
return count >= num
|
||||
|
||||
|
||||
@register("detectable_format:multiple_sections")
|
||||
def check_multiple_sections(response: str, kwargs: dict) -> bool:
|
||||
splitter = kwargs.get("section_spliter", "Section")
|
||||
num = kwargs.get("num_sections", 1)
|
||||
pattern = r"\s?" + re.escape(splitter) + r"\s?\d+\s?"
|
||||
sections = re.split(pattern, response)
|
||||
return len(sections) - 1 >= num
|
||||
|
||||
|
||||
@register("detectable_format:title")
|
||||
def check_title(response: str, kwargs: dict) -> bool:
|
||||
titles = re.findall(r"<<[^>\n]+>>", response)
|
||||
for title in titles:
|
||||
if title.strip("<>").strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@register("detectable_format:json_format")
|
||||
def check_json_format(response: str, kwargs: dict) -> bool:
|
||||
value = response.strip()
|
||||
for prefix in ("```json", "```Json", "```JSON", "```"):
|
||||
if value.lower().startswith(prefix.lower()):
|
||||
value = value[len(prefix) :].strip()
|
||||
if value.endswith("```"):
|
||||
value = value[:-3].strip()
|
||||
try:
|
||||
json.loads(value)
|
||||
return True
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
return False
|
||||
|
||||
|
||||
@register("detectable_format:general_punctuation")
|
||||
def check_general_punctuation(response: str, kwargs: dict) -> bool:
|
||||
punctuation_blacklist = kwargs.get("punctuation_blacklist", [])
|
||||
for punct in punctuation_blacklist:
|
||||
if punct in response:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@register("detectable_format:number_highlighted_words")
|
||||
def check_highlighted_words(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_highlights", 1)
|
||||
highlights = re.findall(r"\*[^\s\*][^\*]*[^\s\*]\*", response)
|
||||
return len(highlights) >= num
|
||||
|
||||
|
||||
@register("startend:end_checker")
|
||||
def check_end_checker(response: str, kwargs: dict) -> bool:
|
||||
end_phrase = kwargs["end_phrase"]
|
||||
return (
|
||||
response.strip()
|
||||
.rstrip('"')
|
||||
.rstrip()
|
||||
.lower()
|
||||
.endswith(end_phrase.strip().lower())
|
||||
)
|
||||
|
||||
|
||||
@register("startend:quotation")
|
||||
def check_quotation(response: str, kwargs: dict) -> bool:
|
||||
value = response.strip()
|
||||
return value.startswith('"') and value.endswith('"')
|
||||
|
||||
|
||||
@register("startend:start_checker")
|
||||
def check_start_checker(response: str, kwargs: dict) -> bool:
|
||||
starter = kwargs["starter"]
|
||||
return bool(re.search(r"^\s*" + re.escape(starter), response, re.MULTILINE))
|
||||
|
||||
|
||||
@register("change_case:english_capital")
|
||||
def check_english_capital(response: str, kwargs: dict) -> bool:
|
||||
return response.isupper()
|
||||
|
||||
|
||||
@register("change_case:english_lowercase")
|
||||
def check_english_lowercase(response: str, kwargs: dict) -> bool:
|
||||
return response.islower()
|
||||
|
||||
|
||||
@register("change_case:capital_word_frequency")
|
||||
def check_capital_word_frequency(response: str, kwargs: dict) -> bool:
|
||||
frequency = kwargs.get("capital_frequency", 1)
|
||||
relation = kwargs.get("capital_relation", "at least")
|
||||
capital_words = re.findall(r"\b[A-Z]{2,}\b", response)
|
||||
count = len(capital_words)
|
||||
if relation == "less than":
|
||||
return count < frequency
|
||||
return count >= frequency
|
||||
|
||||
|
||||
@register("punctuation:no_comma")
|
||||
def check_no_comma(response: str, kwargs: dict) -> bool:
|
||||
return "," not in response
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
return len(re.findall(r"\b\w+\b", text))
|
||||
|
||||
|
||||
def count_sentences(text: str) -> int:
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return 0
|
||||
sentences = re.split(r"(?<=[.!?])\s+", text)
|
||||
return len([s for s in sentences if s.strip()])
|
||||
|
||||
|
||||
@register("length_constraints:number_words")
|
||||
def check_number_words(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_words", 100)
|
||||
relation = kwargs.get("relation", "at least")
|
||||
cnt = count_words(response)
|
||||
if relation == "less than":
|
||||
return cnt < num
|
||||
return cnt >= num
|
||||
|
||||
|
||||
@register("length_constraints:number_sentences")
|
||||
def check_number_sentences(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_sentences", 5)
|
||||
relation = kwargs.get("relation", "at least")
|
||||
cnt = count_sentences(response)
|
||||
if relation == "less than":
|
||||
return cnt < num
|
||||
return cnt >= num
|
||||
|
||||
|
||||
@register("length_constraints:number_paragraphs")
|
||||
def check_number_paragraphs(response: str, kwargs: dict) -> bool:
|
||||
num = kwargs.get("num_paragraphs", 1)
|
||||
if "***" in response:
|
||||
paragraphs = re.split(r"\s?\*\*\*\s?", response)
|
||||
else:
|
||||
paragraphs = re.split(r"\n\n+", response)
|
||||
actual = len([p for p in paragraphs if p.strip()])
|
||||
return actual == num
|
||||
|
||||
|
||||
@register("length_constraints:nth_paragraph_first_word")
|
||||
def check_nth_paragraph_first_word(response: str, kwargs: dict) -> bool:
|
||||
num_paragraphs = kwargs.get("num_paragraphs", 1)
|
||||
nth = kwargs.get("nth_paragraph", 1)
|
||||
first_word = kwargs.get("first_word", "").lower()
|
||||
|
||||
paragraphs = re.split(r"\n\n+", response)
|
||||
paragraphs = [p.strip() for p in paragraphs if p.strip()]
|
||||
|
||||
if len(paragraphs) != num_paragraphs:
|
||||
return False
|
||||
if nth > len(paragraphs):
|
||||
return False
|
||||
|
||||
target = paragraphs[nth - 1]
|
||||
words = target.split()
|
||||
if not words:
|
||||
return False
|
||||
|
||||
word = words[0].strip().lstrip("'\"").rstrip(".,!?:;\"'")
|
||||
return word.lower() == first_word
|
||||
|
||||
|
||||
@register("length_constraints:nth_word_checker")
|
||||
def check_nth_word(response: str, kwargs: dict) -> bool:
|
||||
nth = kwargs.get("nth_word", 1)
|
||||
target = kwargs.get("target_word", "").lower()
|
||||
words = re.findall(r"\b\w+\b", response)
|
||||
if nth > len(words):
|
||||
return False
|
||||
return words[nth - 1].lower() == target
|
||||
|
||||
|
||||
@register("combination:repeat_prompt")
|
||||
def check_repeat_prompt(response: str, kwargs: dict) -> bool:
|
||||
prompt = kwargs["prompt_to_repeat"]
|
||||
return response.strip().lower().startswith(prompt.strip().lower())
|
||||
|
||||
|
||||
@register("combination:two_responses")
|
||||
def check_two_responses(response: str, kwargs: dict) -> bool:
|
||||
parts = response.split("******")
|
||||
valid = [p for p in parts if p.strip()]
|
||||
if len(valid) != 2:
|
||||
return False
|
||||
return valid[0].strip() != valid[1].strip()
|
||||
|
||||
|
||||
def download_ifeval(data_path: str):
|
||||
if os.path.exists(data_path):
|
||||
return
|
||||
os.makedirs(os.path.dirname(data_path) or ".", exist_ok=True)
|
||||
print(f"Downloading IFEval from {IFEVAL_URL} ...")
|
||||
tmp = data_path + ".tmp"
|
||||
urllib.request.urlretrieve(IFEVAL_URL, tmp)
|
||||
with open(tmp, "rb") as f_in:
|
||||
content = f_in.read()
|
||||
with open(data_path, "wb") as f_out:
|
||||
f_out.write(content)
|
||||
os.remove(tmp)
|
||||
print(f" saved to {data_path}")
|
||||
|
||||
|
||||
def load_problems(data_path: str) -> List[dict]:
|
||||
problems = []
|
||||
with open(data_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
problems.append(json.loads(line))
|
||||
return problems
|
||||
|
||||
|
||||
def verify_response(response: str, instruction_id: str, kwargs: dict) -> Optional[bool]:
|
||||
verifier = CONSTRAINT_VERIFIERS.get(instruction_id)
|
||||
if verifier is None:
|
||||
return None
|
||||
try:
|
||||
return verifier(response, kwargs)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def generate_one(
|
||||
engine: InferenceEngine,
|
||||
prompt: str,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
) -> str:
|
||||
output = engine.generate(
|
||||
prompt=prompt,
|
||||
stream=False,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
)
|
||||
if isinstance(output, list):
|
||||
return output[0]
|
||||
return output
|
||||
|
||||
|
||||
def evaluate(
|
||||
engine: InferenceEngine,
|
||||
problems: List[dict],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
num_samples: int = 1,
|
||||
) -> Dict:
|
||||
results = {}
|
||||
constraint_stats: Dict[str, Dict[str, int]] = {}
|
||||
total_constraints = 0
|
||||
total_passed = 0
|
||||
|
||||
for problem in tqdm.tqdm(problems, desc="IFEval", unit="problem"):
|
||||
key = problem["key"]
|
||||
prompt = problem["prompt"]
|
||||
instruction_ids = problem["instruction_id_list"]
|
||||
kwargs_list = problem["kwargs"]
|
||||
|
||||
samples = []
|
||||
for _ in range(num_samples):
|
||||
response = generate_one(
|
||||
engine, prompt, max_tokens, temperature, top_p, top_k
|
||||
)
|
||||
samples.append(response)
|
||||
|
||||
constraint_results = []
|
||||
passed = 0
|
||||
verified = 0
|
||||
|
||||
for idx, instruction_id in enumerate(instruction_ids):
|
||||
kwargs = kwargs_list[idx] if idx < len(kwargs_list) else {}
|
||||
best_pass = False
|
||||
for response in samples:
|
||||
result = verify_response(response, instruction_id, kwargs)
|
||||
if result is None:
|
||||
continue
|
||||
if result:
|
||||
best_pass = True
|
||||
break
|
||||
|
||||
verifier_exists = instruction_id in CONSTRAINT_VERIFIERS
|
||||
if verifier_exists:
|
||||
verified += 1
|
||||
if best_pass:
|
||||
passed += 1
|
||||
|
||||
constraint_results.append(
|
||||
{
|
||||
"instruction_id": instruction_id,
|
||||
"passed": best_pass,
|
||||
"supported": verifier_exists,
|
||||
"kwargs": kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
if verifier_exists:
|
||||
if instruction_id not in constraint_stats:
|
||||
constraint_stats[instruction_id] = {
|
||||
"total": 0,
|
||||
"passed": 0,
|
||||
}
|
||||
constraint_stats[instruction_id]["total"] += 1
|
||||
if best_pass:
|
||||
constraint_stats[instruction_id]["passed"] += 1
|
||||
|
||||
total_constraints += verified
|
||||
total_passed += passed
|
||||
|
||||
accuracy = passed / verified if verified > 0 else None
|
||||
results[str(key)] = {
|
||||
"key": key,
|
||||
"prompt": prompt,
|
||||
"response": samples[0],
|
||||
"num_samples": num_samples,
|
||||
"num_constraints": len(instruction_ids),
|
||||
"num_verified": verified,
|
||||
"num_passed": passed,
|
||||
"accuracy": round(accuracy, 4) if accuracy is not None else None,
|
||||
"constraints": constraint_results,
|
||||
}
|
||||
|
||||
overall_accuracy = (
|
||||
round(total_passed / total_constraints, 4) if total_constraints > 0 else 0.0
|
||||
)
|
||||
|
||||
type_summary = {}
|
||||
for inst_id, stats in sorted(constraint_stats.items()):
|
||||
type_summary[inst_id] = {
|
||||
"total": stats["total"],
|
||||
"passed": stats["passed"],
|
||||
"accuracy": round(stats["passed"] / stats["total"], 4)
|
||||
if stats["total"] > 0
|
||||
else 0.0,
|
||||
}
|
||||
|
||||
unsupported_count = sum(
|
||||
1
|
||||
for p in problems
|
||||
for iid in p["instruction_id_list"]
|
||||
if iid not in CONSTRAINT_VERIFIERS
|
||||
)
|
||||
|
||||
results["_summary"] = {
|
||||
"total_problems": len(problems),
|
||||
"total_constraints": total_constraints,
|
||||
"total_passed": total_passed,
|
||||
"overall_accuracy": overall_accuracy,
|
||||
"unsupported_constraints": unsupported_count,
|
||||
"supported_types": sorted(CONSTRAINT_VERIFIERS.keys()),
|
||||
"per_type_accuracy": type_summary,
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="IFEval benchmark")
|
||||
parser.add_argument(
|
||||
"--param_path", type=str, default="./params", help="Model directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--data_path",
|
||||
type=str,
|
||||
default="./ifeval/input_data.jsonl",
|
||||
help="IFEval JSONL file (auto-download if missing)",
|
||||
)
|
||||
parser.add_argument("--output", type=str, default=None, help="Output JSON path")
|
||||
parser.add_argument(
|
||||
"--max_tokens", type=int, default=512, help="Max generation tokens"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=0.1,
|
||||
help="Sampling temperature",
|
||||
)
|
||||
parser.add_argument("--top_p", type=float, default=0.95, help="Top-p sampling")
|
||||
parser.add_argument("--top_k", type=int, default=50, help="Top-k sampling")
|
||||
parser.add_argument(
|
||||
"--num_samples",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of samples per problem (best-of-n scoring)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch_size", type=int, default=1, help="Inference batch size"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Limit to first N problems (for quick testing)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dump_responses",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to dump raw model responses (JSONL)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
download_ifeval(args.data_path)
|
||||
problems = load_problems(args.data_path)
|
||||
if args.limit:
|
||||
problems = problems[: args.limit]
|
||||
|
||||
print(f"Loaded {len(problems)} problems")
|
||||
print(f"Supported constraint types: {len(CONSTRAINT_VERIFIERS)}")
|
||||
|
||||
model = AutoModel.from_pretrained(args.param_path)
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.param_path)
|
||||
model.to(device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
engine = InferenceEngine(
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
max_batch_size=args.batch_size,
|
||||
)
|
||||
|
||||
results = evaluate(
|
||||
engine=engine,
|
||||
problems=problems,
|
||||
max_tokens=args.max_tokens,
|
||||
temperature=args.temperature,
|
||||
top_p=args.top_p,
|
||||
top_k=args.top_k,
|
||||
num_samples=args.num_samples,
|
||||
)
|
||||
|
||||
summary = results.pop("_summary")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" Problems: {summary['total_problems']}")
|
||||
print(f" Constraints: {summary['total_constraints']}")
|
||||
print(f" Passed: {summary['total_passed']}")
|
||||
print(f" Accuracy: {summary['overall_accuracy']:.2%}")
|
||||
print(f" Unsupported: {summary['unsupported_constraints']}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
print(f"\nPer-type accuracy:")
|
||||
for inst_id, stats in sorted(summary["per_type_accuracy"].items()):
|
||||
print(
|
||||
f" {inst_id:50s} {stats['accuracy']:.2%} "
|
||||
f"({stats['passed']}/{stats['total']})"
|
||||
)
|
||||
|
||||
if args.output:
|
||||
results["_summary"] = summary
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||||
print(f"\nResults saved to {args.output}")
|
||||
|
||||
if args.dump_responses:
|
||||
with open(args.dump_responses, "w", encoding="utf-8") as f:
|
||||
for k, v in results.items():
|
||||
if k.startswith("_"):
|
||||
continue
|
||||
f.write(
|
||||
json.dumps(
|
||||
{
|
||||
"key": v["key"],
|
||||
"prompt": v["prompt"],
|
||||
"response": v["response"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
print(f"Responses dumped to {args.dump_responses}")
|
||||
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,319 @@
|
||||
"""MMLU evaluation via log-likelihood ranking."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import tqdm
|
||||
|
||||
from astrai.model import AutoModel
|
||||
from astrai.tokenize import AutoTokenizer
|
||||
|
||||
MMLU_URL = "https://people.eecs.berkeley.edu/~hendrycks/data.tar"
|
||||
MMLU_SUBJECTS = [
|
||||
"abstract_algebra",
|
||||
"anatomy",
|
||||
"astronomy",
|
||||
"business_ethics",
|
||||
"clinical_knowledge",
|
||||
"college_biology",
|
||||
"college_chemistry",
|
||||
"college_computer_science",
|
||||
"college_mathematics",
|
||||
"college_medicine",
|
||||
"college_physics",
|
||||
"computer_security",
|
||||
"conceptual_physics",
|
||||
"econometrics",
|
||||
"electrical_engineering",
|
||||
"elementary_mathematics",
|
||||
"formal_logic",
|
||||
"global_facts",
|
||||
"high_school_biology",
|
||||
"high_school_chemistry",
|
||||
"high_school_computer_science",
|
||||
"high_school_european_history",
|
||||
"high_school_geography",
|
||||
"high_school_government_and_politics",
|
||||
"high_school_macroeconomics",
|
||||
"high_school_mathematics",
|
||||
"high_school_microeconomics",
|
||||
"high_school_physics",
|
||||
"high_school_psychology",
|
||||
"high_school_statistics",
|
||||
"high_school_us_history",
|
||||
"high_school_world_history",
|
||||
"human_aging",
|
||||
"human_sexuality",
|
||||
"international_law",
|
||||
"jurisprudence",
|
||||
"logical_fallacies",
|
||||
"machine_learning",
|
||||
"management",
|
||||
"marketing",
|
||||
"medical_genetics",
|
||||
"miscellaneous",
|
||||
"moral_disputes",
|
||||
"moral_scenarios",
|
||||
"nutrition",
|
||||
"philosophy",
|
||||
"prehistory",
|
||||
"professional_accounting",
|
||||
"professional_law",
|
||||
"professional_medicine",
|
||||
"professional_psychology",
|
||||
"public_relations",
|
||||
"security_studies",
|
||||
"sociology",
|
||||
"us_foreign_policy",
|
||||
"virology",
|
||||
"world_religions",
|
||||
]
|
||||
|
||||
|
||||
def _download_and_extract(url: str, data_dir: str):
|
||||
tar_path = os.path.join(data_dir, "data.tar")
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
print(f"Downloading MMLU data from {url}...")
|
||||
resp = requests.get(url, stream=True, timeout=300)
|
||||
resp.raise_for_status()
|
||||
total = int(resp.headers.get("content-length", 0))
|
||||
with tqdm.tqdm(total=total, unit="B", unit_scale=True, desc=" Download") as bar:
|
||||
with open(tar_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
bar.update(len(chunk))
|
||||
print("Extracting...")
|
||||
with tarfile.open(tar_path, "r") as tf:
|
||||
tf.extractall(data_dir)
|
||||
os.remove(tar_path)
|
||||
|
||||
|
||||
def download_mmlu(data_dir: str):
|
||||
_download_and_extract(MMLU_URL, data_dir)
|
||||
src = os.path.join(data_dir, "data")
|
||||
if os.path.exists(src):
|
||||
for item in os.listdir(src):
|
||||
src_item = os.path.join(src, item)
|
||||
dst_item = os.path.join(data_dir, item)
|
||||
if os.path.exists(dst_item):
|
||||
if os.path.isdir(dst_item):
|
||||
shutil.rmtree(dst_item)
|
||||
else:
|
||||
os.remove(dst_item)
|
||||
os.rename(src_item, dst_item)
|
||||
os.rmdir(src)
|
||||
print(f"MMLU data saved to {data_dir}")
|
||||
|
||||
|
||||
def _strip_prefix(text: str, prefix: str) -> str:
|
||||
if text.startswith(prefix):
|
||||
return text[len(prefix) :].strip()
|
||||
return text
|
||||
|
||||
|
||||
def load_csv(path: str) -> list[dict]:
|
||||
data = []
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for row in csv.reader(f):
|
||||
if len(row) < 6:
|
||||
continue
|
||||
if row[0].strip().lower() == "question":
|
||||
continue
|
||||
data.append(
|
||||
{
|
||||
"question": row[0].strip(),
|
||||
"A": _strip_prefix(row[1].strip(), "A)"),
|
||||
"B": _strip_prefix(row[2].strip(), "B)"),
|
||||
"C": _strip_prefix(row[3].strip(), "C)"),
|
||||
"D": _strip_prefix(row[4].strip(), "D)"),
|
||||
"answer": row[5].strip(),
|
||||
}
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def build_prompt(
|
||||
question: str, choices: dict, subject: str, n_shot: int, dev_data: list[dict]
|
||||
) -> str:
|
||||
prompt = ""
|
||||
if n_shot > 0 and dev_data:
|
||||
prompt = f"The following are multiple choice questions (with answers) about {subject}.\n\n"
|
||||
for item in dev_data[:n_shot]:
|
||||
prompt += f"Question: {item['question']}\n"
|
||||
for k in ("A", "B", "C", "D"):
|
||||
prompt += f"{k}. {item[k]}\n"
|
||||
prompt += f"Answer: {item['answer']}\n\n"
|
||||
prompt += f"Question: {question}\n"
|
||||
for k in ("A", "B", "C", "D"):
|
||||
prompt += f"{k}. {choices[k]}\n"
|
||||
prompt += "Answer:"
|
||||
return prompt
|
||||
|
||||
|
||||
def apply_chat(
|
||||
tokenizer, raw_prompt: str, n_shot: int, dev_data: list[dict] | None
|
||||
) -> str:
|
||||
"""Wrap raw MMLU prompt in the model's chat template format.
|
||||
|
||||
For few-shot, prepend example Q&A pairs as a second user/assistant exchange.
|
||||
"""
|
||||
messages = []
|
||||
if n_shot > 0 and dev_data:
|
||||
for item in dev_data[:n_shot]:
|
||||
q = f"Question: {item['question']}\n"
|
||||
for k in ("A", "B", "C", "D"):
|
||||
q += f"{k}. {item[k]}\n"
|
||||
q += "Answer:"
|
||||
messages.append({"role": "user", "content": q})
|
||||
messages.append({"role": "assistant", "content": item["answer"]})
|
||||
messages.append({"role": "user", "content": raw_prompt})
|
||||
return tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
|
||||
|
||||
def choice_logprob(
|
||||
model, tokenizer, context_ids: list[int], choice_letter: str, device: str
|
||||
) -> float:
|
||||
choice_text = choice_letter
|
||||
choice_ids = tokenizer.encode(choice_text, add_special_tokens=False)
|
||||
input_ids = context_ids + choice_ids
|
||||
max_len = model.config.max_len
|
||||
if len(input_ids) > max_len:
|
||||
overflow = len(input_ids) - max_len
|
||||
input_ids = input_ids[overflow:]
|
||||
ctx_len = len(input_ids) - len(choice_ids)
|
||||
else:
|
||||
ctx_len = len(context_ids)
|
||||
|
||||
input_tensor = torch.tensor([input_ids], device=device, dtype=torch.long)
|
||||
with torch.inference_mode():
|
||||
logits = model(input_tensor)["logits"][0]
|
||||
|
||||
score = 0.0
|
||||
for i, tid in enumerate(choice_ids):
|
||||
pos = ctx_len - 1 + i
|
||||
if pos >= len(logits):
|
||||
break
|
||||
score += F.log_softmax(logits[pos], dim=-1)[tid].item()
|
||||
return score
|
||||
|
||||
|
||||
def evaluate_subject(
|
||||
model,
|
||||
tokenizer,
|
||||
subject: str,
|
||||
test_data: list[dict],
|
||||
dev_data: list[dict] | None,
|
||||
device: str,
|
||||
n_shot: int,
|
||||
) -> tuple[float, int, int]:
|
||||
correct = 0
|
||||
total = 0
|
||||
for item in tqdm.tqdm(test_data, desc=f"{subject:40s}", leave=False):
|
||||
raw_prompt = build_prompt(
|
||||
item["question"], item, subject, n_shot, dev_data or []
|
||||
)
|
||||
context = apply_chat(tokenizer, raw_prompt, n_shot, dev_data or [])
|
||||
context_ids = tokenizer.encode(context)
|
||||
scores = {
|
||||
c: choice_logprob(model, tokenizer, context_ids, c, device)
|
||||
for c in ("A", "B", "C", "D")
|
||||
}
|
||||
if max(scores, key=scores.get) == item["answer"]:
|
||||
correct += 1
|
||||
total += 1
|
||||
return correct / total, correct, total
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="MMLU evaluation")
|
||||
parser.add_argument(
|
||||
"--param_path", type=str, default="./params", help="Model directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--data_dir", type=str, default="./mmlu_data", help="MMLU data directory"
|
||||
)
|
||||
parser.add_argument("--download", action="store_true", help="Download MMLU data")
|
||||
parser.add_argument(
|
||||
"--n_shot", type=int, default=5, help="Few-shot examples (0 for zero-shot)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--subjects", type=str, nargs="+", help="Specific subjects (default: all)"
|
||||
)
|
||||
parser.add_argument("--output", type=str, help="Output JSON path")
|
||||
parser.add_argument("--split", type=str, default="test", choices=["test", "val"])
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default="cuda" if torch.cuda.is_available() else "cpu",
|
||||
help="Device",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dtype",
|
||||
type=str,
|
||||
default="bfloat16" if torch.cuda.is_available() else "float32",
|
||||
help="Torch dtype",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.download or not os.path.exists(args.data_dir):
|
||||
download_mmlu(args.data_dir)
|
||||
|
||||
model = AutoModel.from_pretrained(args.param_path)
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.param_path)
|
||||
device = args.device
|
||||
dtype = getattr(torch, args.dtype)
|
||||
model.to(device=device, dtype=dtype)
|
||||
model.eval()
|
||||
|
||||
subjects = args.subjects or MMLU_SUBJECTS
|
||||
results = {}
|
||||
total_correct = 0
|
||||
total_questions = 0
|
||||
|
||||
for subject in subjects:
|
||||
dev_path = os.path.join(args.data_dir, "dev", f"{subject}_dev.csv")
|
||||
test_path = os.path.join(
|
||||
args.data_dir, args.split, f"{subject}_{args.split}.csv"
|
||||
)
|
||||
|
||||
if not os.path.exists(test_path):
|
||||
print(f" Skipping {subject}: test file not found")
|
||||
continue
|
||||
|
||||
dev_data = load_csv(dev_path) if os.path.exists(dev_path) else None
|
||||
test_data = load_csv(test_path)
|
||||
|
||||
acc, corr, tot = evaluate_subject(
|
||||
model, tokenizer, subject, test_data, dev_data, device, args.n_shot
|
||||
)
|
||||
results[subject] = {"accuracy": round(acc, 4), "correct": corr, "total": tot}
|
||||
total_correct += corr
|
||||
total_questions += tot
|
||||
print(f" {subject:40s} {acc:.2%} ({corr}/{tot})")
|
||||
|
||||
overall = total_correct / total_questions if total_questions else 0
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f" Overall: {overall:.2%} ({total_correct}/{total_questions})")
|
||||
results["_overall"] = {
|
||||
"accuracy": round(overall, 4),
|
||||
"correct": total_correct,
|
||||
"total": total_questions,
|
||||
}
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"Results saved to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user