From 7dc37efc3faf4a36c20c3db4b3130885505181c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= <zhifu.gzf@alibaba-inc.com>
Date: Tue, 6 Aug 2024 01:54:15 +0800
Subject: [PATCH] deepspeed

---
 funasr/train_utils/trainer_ds.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.py
index 66ae7ed62..9ef9dc9e4 100644
--- a/funasr/train_utils/trainer_ds.py
+++ b/funasr/train_utils/trainer_ds.py
@@ -593,6 +593,8 @@ class Trainer:
         time_beg = time.perf_counter()
         time5 = time_beg
         for batch_idx, batch in enumerate(dataloader_train):
+            if batch_idx == 0 and (self.use_ddp or self.use_fsdp or self.use_deepspeed):
+                dist.barrier()
             self.batch_total += 1
             self.step_in_epoch += 1
             loss_dict = {