refactor(run): 重构脚本执行方式并优化项目结构

pre_train/__init__.py

@@ -1,12 +0,0 @@
-import sys
-import os
-
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import utils
-from tokenizer import BpeTokenizer
-
-__all__ = [
-    "utils",
-    "BpeTokenizer",
-]

run.py

@@ -1,47 +1,41 @@
-# run_all.py
-
 import os
 import sys
-import importlib.util
+import subprocess
 
-# 确保根目录在路径中
 PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(PROJECT_ROOT)
 
 def run_script(script_path):
-    """动态导入并运行一个 Python 脚本"""
     if not os.path.exists(script_path):
-        print(f"[警告] 文件不存在: {script_path}")
+        print(f"[Warning] File does not exist: {script_path}")
         return
 
-    # 生成模块名
-    module_name = os.path.splitext(os.path.basename(script_path))[0]
-    spec = importlib.util.spec_from_file_location(module_name, script_path)
-    module = importlib.util.module_from_spec(spec)
-
-    # 插入到 sys.modules 避免重复导入
-    sys.modules[module_name] = module
-
-    # 执行脚本（相当于 __name__ == "__main__"）
     print(f"\n{'='*50}")
-    print(f"运行: {script_path}")
+    print(f"Running: {script_path}")
     print(f"{'='*50}")
-    spec.loader.exec_module(module)
 
-def main():
+    try:
-    # 运行 pre_train 下的所有脚本
+        env = os.environ.copy()
-    pre_train_dir = os.path.join(PROJECT_ROOT, 'pre_train')
+        env['PYTHONPATH'] = PROJECT_ROOT
+        subprocess.run(
+            [sys.executable, script_path], 
+            check=True, 
+            cwd=PROJECT_ROOT, 
+            env=env
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"[Error] Script execution failed: {script_path}, Error code: {e.returncode}")
+        
+def run_scripts(project_root: str, directory: str):
+    pre_train_dir = os.path.join(project_root, directory)
     for file in os.listdir(pre_train_dir):
-        if file.endswith('.py') and not file.startswith('__'):
+        if file.endswith('.py'):
             script_path = os.path.join(pre_train_dir, file)
             run_script(script_path)
 
-    # 运行 supervised_finetuning 下的所有脚本
+def main():
-    sft_dir = os.path.join(PROJECT_ROOT, 'supervised_finetuning')
+    run_scripts(PROJECT_ROOT, 'pre_train')
-    for file in os.listdir(sft_dir):
+    run_scripts(PROJECT_ROOT, 'supervised_finetuning')
-        if file.endswith('.py') and not file.startswith('__'):
-            script_path = os.path.join(sft_dir, file)
-            run_script(script_path)
 
 if __name__ == "__main__":
     main()

supervised_finetuning/__init__.py

@@ -1,12 +0,0 @@
-import sys
-import os
-
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import utils
-from tokenizer import BpeTokenizer
-
-__all__ = [
-    "utils",
-    "BpeTokenizer",
-]