diff --git a/pre_train/__init__.py b/pre_train/__init__.py deleted file mode 100644 index c5a770d..0000000 --- a/pre_train/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys -import os - -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import utils -from tokenizer import BpeTokenizer - -__all__ = [ - "utils", - "BpeTokenizer", -] \ No newline at end of file diff --git a/run.py b/run.py index 432a48c..b3729d4 100644 --- a/run.py +++ b/run.py @@ -1,47 +1,41 @@ -# run_all.py - import os import sys -import importlib.util +import subprocess -# 确保根目录在路径中 PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) sys.path.append(PROJECT_ROOT) def run_script(script_path): - """动态导入并运行一个 Python 脚本""" if not os.path.exists(script_path): - print(f"[警告] 文件不存在: {script_path}") + print(f"[Warning] File does not exist: {script_path}") return - # 生成模块名 - module_name = os.path.splitext(os.path.basename(script_path))[0] - spec = importlib.util.spec_from_file_location(module_name, script_path) - module = importlib.util.module_from_spec(spec) - - # 插入到 sys.modules 避免重复导入 - sys.modules[module_name] = module - - # 执行脚本(相当于 __name__ == "__main__") print(f"\n{'='*50}") - print(f"运行: {script_path}") + print(f"Running: {script_path}") print(f"{'='*50}") - spec.loader.exec_module(module) -def main(): - # 运行 pre_train 下的所有脚本 - pre_train_dir = os.path.join(PROJECT_ROOT, 'pre_train') + try: + env = os.environ.copy() + env['PYTHONPATH'] = PROJECT_ROOT + subprocess.run( + [sys.executable, script_path], + check=True, + cwd=PROJECT_ROOT, + env=env + ) + except subprocess.CalledProcessError as e: + print(f"[Error] Script execution failed: {script_path}, Error code: {e.returncode}") + +def run_scripts(project_root: str, directory: str): + pre_train_dir = os.path.join(project_root, directory) for file in os.listdir(pre_train_dir): - if file.endswith('.py') and not file.startswith('__'): + if file.endswith('.py'): script_path = os.path.join(pre_train_dir, file) run_script(script_path) - # 运行 supervised_finetuning 下的所有脚本 - sft_dir = os.path.join(PROJECT_ROOT, 'supervised_finetuning') - for file in os.listdir(sft_dir): - if file.endswith('.py') and not file.startswith('__'): - script_path = os.path.join(sft_dir, file) - run_script(script_path) +def main(): + run_scripts(PROJECT_ROOT, 'pre_train') + run_scripts(PROJECT_ROOT, 'supervised_finetuning') if __name__ == "__main__": main() \ No newline at end of file diff --git a/supervised_finetuning/__init__.py b/supervised_finetuning/__init__.py deleted file mode 100644 index c5a770d..0000000 --- a/supervised_finetuning/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys -import os - -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import utils -from tokenizer import BpeTokenizer - -__all__ = [ - "utils", - "BpeTokenizer", -] \ No newline at end of file