[BugFix] fix double shutdown of comm group when rank0 clears weights slower than other ranks (PaddlePaddle#5715)

Author: liyonghua0910

fastdeploy/rl/dynamic_weight_manager.py

@@ -272,10 +272,14 @@ def check_model_weights_status(model_weights_status, model_runner, pid, block):
                 logger.info("infer engine stopped! start to load new checkpoint...")
                 model_runner.clear_requests()
                 model_runner.update_parameters(pid)
+                while model_weights_status.value[0] != ModelWeightsStatus.NORMAL:
+                    time.sleep(0.01)
                 logger.info("finished loading new checkpoint")
             elif model_weights_status.value[0] == ModelWeightsStatus.CLEARING:
                 logger.info("infer engine stopped! start to clear checkpoint...")
                 model_runner.clear_requests()
                 model_runner.clear_parameters(pid)
+                while model_weights_status.value[0] != ModelWeightsStatus.CLEARED:
+                    time.sleep(0.01)
                 logger.info("finished clearing checkpoint")
             time.sleep(0.01)