diff --git a/pre_train/__init__.py b/pre_train/__init__.py new file mode 100644 index 0000000..c5a770d --- /dev/null +++ b/pre_train/__init__.py @@ -0,0 +1,12 @@ +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import utils +from tokenizer import BpeTokenizer + +__all__ = [ + "utils", + "BpeTokenizer", +] \ No newline at end of file diff --git a/chinese-c4.py b/pre_train/chinese-c4.py similarity index 100% rename from chinese-c4.py rename to pre_train/chinese-c4.py diff --git a/chinese-cosmopedia.py b/pre_train/chinese-cosmopedia.py similarity index 100% rename from chinese-cosmopedia.py rename to pre_train/chinese-cosmopedia.py diff --git a/english-fineweb.py b/pre_train/english-fineweb.py similarity index 100% rename from english-fineweb.py rename to pre_train/english-fineweb.py diff --git a/english-wiki.py b/pre_train/english-wiki.py similarity index 100% rename from english-wiki.py rename to pre_train/english-wiki.py diff --git a/run.py b/run.py new file mode 100644 index 0000000..432a48c --- /dev/null +++ b/run.py @@ -0,0 +1,47 @@ +# run_all.py + +import os +import sys +import importlib.util + +# 确保根目录在路径中 +PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(PROJECT_ROOT) + +def run_script(script_path): + """动态导入并运行一个 Python 脚本""" + if not os.path.exists(script_path): + print(f"[警告] 文件不存在: {script_path}") + return + + # 生成模块名 + module_name = os.path.splitext(os.path.basename(script_path))[0] + spec = importlib.util.spec_from_file_location(module_name, script_path) + module = importlib.util.module_from_spec(spec) + + # 插入到 sys.modules 避免重复导入 + sys.modules[module_name] = module + + # 执行脚本(相当于 __name__ == "__main__") + print(f"\n{'='*50}") + print(f"运行: {script_path}") + print(f"{'='*50}") + spec.loader.exec_module(module) + +def main(): + # 运行 pre_train 下的所有脚本 + pre_train_dir = os.path.join(PROJECT_ROOT, 'pre_train') + for file in os.listdir(pre_train_dir): + if file.endswith('.py') and not file.startswith('__'): + script_path = os.path.join(pre_train_dir, file) + run_script(script_path) + + # 运行 supervised_finetuning 下的所有脚本 + sft_dir = os.path.join(PROJECT_ROOT, 'supervised_finetuning') + for file in os.listdir(sft_dir): + if file.endswith('.py') and not file.startswith('__'): + script_path = os.path.join(sft_dir, file) + run_script(script_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/supervised_finetuning/__init__.py b/supervised_finetuning/__init__.py new file mode 100644 index 0000000..c5a770d --- /dev/null +++ b/supervised_finetuning/__init__.py @@ -0,0 +1,12 @@ +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import utils +from tokenizer import BpeTokenizer + +__all__ = [ + "utils", + "BpeTokenizer", +] \ No newline at end of file diff --git a/sft_belle.py b/supervised_finetuning/sft_belle.py similarity index 100% rename from sft_belle.py rename to supervised_finetuning/sft_belle.py diff --git a/sft_chinese_instruct.py b/supervised_finetuning/sft_chinese_instruct.py similarity index 100% rename from sft_chinese_instruct.py rename to supervised_finetuning/sft_chinese_instruct.py diff --git a/sft_coder.py b/supervised_finetuning/sft_coder.py similarity index 100% rename from sft_coder.py rename to supervised_finetuning/sft_coder.py diff --git a/sft_magpie-pro-300k.py b/supervised_finetuning/sft_magpie-pro-300k.py similarity index 100% rename from sft_magpie-pro-300k.py rename to supervised_finetuning/sft_magpie-pro-300k.py diff --git a/utils.py b/utils.py index 86906b8..9174d24 100644 --- a/utils.py +++ b/utils.py @@ -116,14 +116,16 @@ def process_dataset( column_name: str = "text", process_func: Union[Callable[[dict], dict], Callable[[List[dict]], List[dict]]] = None, normalization_func=comprehensive_normalization, + output_dir: str = None, ): train_dataset = dataset_dict[split_name] total_samples = len(train_dataset) num_chunks = (total_samples // chunk_size) + 1 lim_chunks = min(max_chunk_num, num_chunks) if max_chunk_num else num_chunks - script_dir = os.path.dirname(os.path.abspath(__file__)) - output_dir = os.path.join(script_dir, "dataset", output_subdir) + if output_dir is None: + output_dir = os.path.join(os.getcwd(), "dataset", output_subdir) + os.makedirs(output_dir, exist_ok=True) for i in range(lim_chunks):