fix(chinese-instruct): 修复 LaTex 公式在数据集中的显示问题
This commit is contained in:
+12
-3
@@ -1,11 +1,20 @@
|
|||||||
# Mxode/Chinese-Reasoning-Distil-Data
|
|
||||||
|
|
||||||
from datasets import DatasetDict
|
from datasets import DatasetDict
|
||||||
from datasets import load_dataset, concatenate_datasets
|
from datasets import load_dataset, concatenate_datasets
|
||||||
from utils import process_dataset
|
from utils import process_dataset
|
||||||
|
|
||||||
|
|
||||||
def build_prompt(query, response) -> str:
|
def build_prompt(query:str, response:str) -> str:
|
||||||
|
replacements = {
|
||||||
|
"\\[": "$$",
|
||||||
|
"\\]": "$$",
|
||||||
|
"\\(": "$",
|
||||||
|
"\\)": "$"
|
||||||
|
}
|
||||||
|
|
||||||
|
for old, new in replacements.items():
|
||||||
|
query = query.replace(old, new)
|
||||||
|
response = response.replace(old, new)
|
||||||
|
|
||||||
return f"<|user|> {query} <|system|> <bos>{response}<eos>\n"
|
return f"<|user|> {query} <|system|> <bos>{response}<eos>\n"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user