From 2d1c4e9d6cf8e4904166f1ce27178042a27abef3 Mon Sep 17 00:00:00 2001 From: ViperEkura <3081035982@qq.com> Date: Fri, 1 Aug 2025 17:41:49 +0800 Subject: [PATCH] =?UTF-8?q?feat(project=5Fstructure):=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E7=9B=AE=E5=BD=95=E5=B9=B6=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pre_train/__init__.py | 12 +++++ chinese-c4.py => pre_train/chinese-c4.py | 0 .../chinese-cosmopedia.py | 0 .../english-fineweb.py | 0 english-wiki.py => pre_train/english-wiki.py | 0 run.py | 47 +++++++++++++++++++ supervised_finetuning/__init__.py | 12 +++++ .../sft_belle.py | 0 .../sft_chinese_instruct.py | 0 .../sft_coder.py | 0 .../sft_magpie-pro-300k.py | 0 utils.py | 6 ++- 12 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 pre_train/__init__.py rename chinese-c4.py => pre_train/chinese-c4.py (100%) rename chinese-cosmopedia.py => pre_train/chinese-cosmopedia.py (100%) rename english-fineweb.py => pre_train/english-fineweb.py (100%) rename english-wiki.py => pre_train/english-wiki.py (100%) create mode 100644 run.py create mode 100644 supervised_finetuning/__init__.py rename sft_belle.py => supervised_finetuning/sft_belle.py (100%) rename sft_chinese_instruct.py => supervised_finetuning/sft_chinese_instruct.py (100%) rename sft_coder.py => supervised_finetuning/sft_coder.py (100%) rename sft_magpie-pro-300k.py => supervised_finetuning/sft_magpie-pro-300k.py (100%) diff --git a/pre_train/__init__.py b/pre_train/__init__.py new file mode 100644 index 0000000..c5a770d --- /dev/null +++ b/pre_train/__init__.py @@ -0,0 +1,12 @@ +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import utils +from tokenizer import BpeTokenizer + +__all__ = [ + "utils", + "BpeTokenizer", +] \ No newline at end of file diff --git a/chinese-c4.py b/pre_train/chinese-c4.py similarity index 100% rename from chinese-c4.py rename to pre_train/chinese-c4.py diff --git a/chinese-cosmopedia.py b/pre_train/chinese-cosmopedia.py similarity index 100% rename from chinese-cosmopedia.py rename to pre_train/chinese-cosmopedia.py diff --git a/english-fineweb.py b/pre_train/english-fineweb.py similarity index 100% rename from english-fineweb.py rename to pre_train/english-fineweb.py diff --git a/english-wiki.py b/pre_train/english-wiki.py similarity index 100% rename from english-wiki.py rename to pre_train/english-wiki.py diff --git a/run.py b/run.py new file mode 100644 index 0000000..432a48c --- /dev/null +++ b/run.py @@ -0,0 +1,47 @@ +# run_all.py + +import os +import sys +import importlib.util + +# 确保根目录在路径中 +PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(PROJECT_ROOT) + +def run_script(script_path): + """动态导入并运行一个 Python 脚本""" + if not os.path.exists(script_path): + print(f"[警告] 文件不存在: {script_path}") + return + + # 生成模块名 + module_name = os.path.splitext(os.path.basename(script_path))[0] + spec = importlib.util.spec_from_file_location(module_name, script_path) + module = importlib.util.module_from_spec(spec) + + # 插入到 sys.modules 避免重复导入 + sys.modules[module_name] = module + + # 执行脚本（相当于 __name__ == "__main__"） + print(f"\n{'='*50}") + print(f"运行: {script_path}") + print(f"{'='*50}") + spec.loader.exec_module(module) + +def main(): + # 运行 pre_train 下的所有脚本 + pre_train_dir = os.path.join(PROJECT_ROOT, 'pre_train') + for file in os.listdir(pre_train_dir): + if file.endswith('.py') and not file.startswith('__'): + script_path = os.path.join(pre_train_dir, file) + run_script(script_path) + + # 运行 supervised_finetuning 下的所有脚本 + sft_dir = os.path.join(PROJECT_ROOT, 'supervised_finetuning') + for file in os.listdir(sft_dir): + if file.endswith('.py') and not file.startswith('__'): + script_path = os.path.join(sft_dir, file) + run_script(script_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/supervised_finetuning/__init__.py b/supervised_finetuning/__init__.py new file mode 100644 index 0000000..c5a770d --- /dev/null +++ b/supervised_finetuning/__init__.py @@ -0,0 +1,12 @@ +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import utils +from tokenizer import BpeTokenizer + +__all__ = [ + "utils", + "BpeTokenizer", +] \ No newline at end of file diff --git a/sft_belle.py b/supervised_finetuning/sft_belle.py similarity index 100% rename from sft_belle.py rename to supervised_finetuning/sft_belle.py diff --git a/sft_chinese_instruct.py b/supervised_finetuning/sft_chinese_instruct.py similarity index 100% rename from sft_chinese_instruct.py rename to supervised_finetuning/sft_chinese_instruct.py diff --git a/sft_coder.py b/supervised_finetuning/sft_coder.py similarity index 100% rename from sft_coder.py rename to supervised_finetuning/sft_coder.py diff --git a/sft_magpie-pro-300k.py b/supervised_finetuning/sft_magpie-pro-300k.py similarity index 100% rename from sft_magpie-pro-300k.py rename to supervised_finetuning/sft_magpie-pro-300k.py diff --git a/utils.py b/utils.py index 86906b8..9174d24 100644 --- a/utils.py +++ b/utils.py @@ -116,14 +116,16 @@ def process_dataset( column_name: str = "text", process_func: Union[Callable[[dict], dict], Callable[[List[dict]], List[dict]]] = None, normalization_func=comprehensive_normalization, + output_dir: str = None, ): train_dataset = dataset_dict[split_name] total_samples = len(train_dataset) num_chunks = (total_samples // chunk_size) + 1 lim_chunks = min(max_chunk_num, num_chunks) if max_chunk_num else num_chunks - script_dir = os.path.dirname(os.path.abspath(__file__)) - output_dir = os.path.join(script_dir, "dataset", output_subdir) + if output_dir is None: + output_dir = os.path.join(os.getcwd(), "dataset", output_subdir) + os.makedirs(output_dir, exist_ok=True) for i in range(lim_chunks):