ViperEkura
0b6a17330f
- New FSDP2Executor registers as 'fsdp2' in ExecutorFactory, using per-module fully_shard() instead of FSDP1 FlatParameter wrapper - FSDP2 preserves original Parameter objects as DTensors, eliminating use_orig_params=True hack - FSDP2Executor implements _no_sync via set_requires_gradient_sync, clip_grad_norm via unshard, unwrap_model via DTensor.full_tensor - Drop **_extra/**_ddp_only_kwargs fallbacks in BaseExecutor/FSDPExecutor/FSDP2Executor, replaced by parallel_mode-aware executor_kwargs dispatch in train.py (ddp-only kwargs only passed when parallel_mode=ddp) - Export FSDP2Executor in astrai.parallel.__init__
41 lines
830 B
Python
41 lines
830 B
Python
from astrai.parallel.executor import (
|
|
AccumOptimizer,
|
|
AccumScheduler,
|
|
BaseExecutor,
|
|
DDPExecutor,
|
|
ExecutorFactory,
|
|
FSDP2Executor,
|
|
FSDPExecutor,
|
|
GradientState,
|
|
NoneExecutor,
|
|
)
|
|
from astrai.parallel.module import ColumnParallelLinear, RowParallelLinear
|
|
from astrai.parallel.setup import (
|
|
get_current_device,
|
|
get_rank,
|
|
get_world_size,
|
|
only_on_rank,
|
|
setup_parallel,
|
|
spawn_parallel_fn,
|
|
)
|
|
|
|
__all__ = [
|
|
"get_world_size",
|
|
"get_rank",
|
|
"get_current_device",
|
|
"only_on_rank",
|
|
"setup_parallel",
|
|
"spawn_parallel_fn",
|
|
"RowParallelLinear",
|
|
"ColumnParallelLinear",
|
|
"ExecutorFactory",
|
|
"BaseExecutor",
|
|
"GradientState",
|
|
"AccumOptimizer",
|
|
"AccumScheduler",
|
|
"NoneExecutor",
|
|
"DDPExecutor",
|
|
"FSDPExecutor",
|
|
"FSDP2Executor",
|
|
]
|