hailin
/
jd_train
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

.

This commit is contained in:
hailin 2025-08-25 23:15:10 +08:00
parent af095b448a
commit a273c4e2d3
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
train_sft_ds.py
View File

@ -308,8 +308,8 @@ def main():
) )
# 多机/多卡分片（让每个全局 rank 读不同子流） # 多机/多卡分片（让每个全局 rank 读不同子流）
world_size = int(os.environ.get("WORLD_SIZE", "1")) # world_size = int(os.environ.get("WORLD_SIZE", "1"))
ds_stream2 = ds_stream2.shard(num_shards=world_size, index=rank, contiguous=True) # ds_stream2 = ds_stream2.shard(num_shards=world_size, index=rank, contiguous=True)
def ex_iter2(): def ex_iter2():
for ex in ds_stream2: for ex in ds_stream2:
yield ex yield ex