fix: FSDP unwrap_model collective op and None guard

- unshard() and full_tensor() are collective ops, all ranks must participate
- Old code returned None on non-rank-0 before calling unshard, causing deadlock
- Fix: all ranks unshard/full_tensor, only rank-0 keeps the result
- Move create_ref_model to parallel/utils.py, accept executor+model directly
- Guard create_ref_model and sync_old_model against None on non-rank-0
This commit is contained in:
2026-07-29 23:41:10 +08:00
parent 8206afefd9
commit bcaa2d1ae0
5 changed files with 61 additions and 29 deletions
+13 -7
View File
@@ -326,21 +326,27 @@ class FSDPExecutor(BaseExecutor):
if not self.use_distributed:
return model.state_dict()
if get_rank() != 0:
return None
# unshard() and full_tensor() are collective ops — all ranks must
# participate. Non-rank-0 ranks still call them but discard results.
for module in model.modules():
if isinstance(module, FSDPModule):
module.unshard()
state_dict = model.state_dict()
result = {
k: (v.full_tensor() if isinstance(v, DTensor) else v)
for k, v in state_dict.items()
}
result = {}
for k, v in state_dict.items():
if isinstance(v, DTensor):
full = v.full_tensor()
if get_rank() == 0:
result[k] = full
elif get_rank() == 0:
result[k] = v
for module in model.modules():
if isinstance(module, FSDPModule):
module.reshard()
if get_rank() != 0:
return None
return result