.

2025-08-28 22:54:46 +08:00 · 2025-08-28 22:54:46 +08:00 · 1e989784d3
parent 4ad2be2d34
commit 1e989784d3
1 changed files with 1 additions and 1 deletions
--- a/train_sft_ds.py
+++ b/train_sft_ds.py
@ -487,7 +487,7 @@ def main():
        torch_dtype=dtype,
        low_cpu_mem_usage=True,
        trust_remote_code=True,
-        attn_implementation="flash_attention_2"
+        attn_implementation="sdpa"
    )

    print(f"GC enabled? {getattr(model, 'is_gradient_checkpointing', False)}", flush=True)