refactor(data_processing): 重构数据处理脚本并添加 fetch_folders 函数

2025-08-01 22:28:40 +08:00
parent 9c3ecc252c
commit 931403a713
3 changed files with 15 additions and 16 deletions
@@ -14,6 +14,16 @@ import re
 def fetch_files(directory):
    return [os.path.join(root, f) 
            for root, _, files in os.walk(directory) for f in files]
+    
+
+def fetch_folders(root_dir, filter_func=None):
+    folders = []
+    for root, dirs, _ in os.walk(root_dir):
+        for dir_name in dirs:
+            folder_path = os.path.join(root, dir_name)
+            if filter_func is None or filter_func(folder_path):
+                folders.append(folder_path)
+    return folders


 def comprehensive_normalization(text):