diff --git a/ds_config_zero3.json b/ds_config_zero3.json index 3765c93..fcb784c 100644 --- a/ds_config_zero3.json +++ b/ds_config_zero3.json @@ -1,6 +1,7 @@ { - "train_micro_batch_size_per_gpu": "auto", - "gradient_accumulation_steps": "auto", + "train_micro_batch_size_per_gpu": 1, + "gradient_accumulation_steps": 1, + "zero_optimization": { "stage": 3, "overlap_comm": true, @@ -9,14 +10,19 @@ "stage3_prefetch_bucket_size": 100000000, "stage3_param_persistence_threshold": 1000000, "offload_optimizer": { "device": "cpu", "pin_memory": true }, - "offload_param": { "device": "none" } + "offload_param": { "device": "none" } }, - "fp16": { "enabled": false }, + "bf16": { "enabled": true }, + "fp16": { "enabled": false }, + "optimizer": { "type": "CPUAdam", "params": { "lr": 2e-5, "betas": [0.9, 0.999], "eps": 1e-8, "weight_decay": 0.1 } }, + "gradient_clipping": 1.0, - "zero_force_ds_cpu_optimizer": true + "zero_force_ds_cpu_optimizer": true, + + "wall_clock_breakdown": false }