Merge pull request #2191 from FedML-AI/raphael/hotfix-jobrunner

[Deploy] Hot Fix: Job runner context lost after logout.

Author: Raphael-Jin

python/fedml/computing/scheduler/model_scheduler/master_job_runner_manager.py

@@ -42,11 +42,18 @@ def send_deployment_stages(
                 message_center=message_center
             )
 
-    def send_deployment_delete_request_to_edges(self, end_point_id, payload, model_msg_object, message_center=None):
+    def send_deployment_delete_request_to_edges(self, end_point_id, payload, model_msg_object, message_center=None,
+                                                args=None):
         run_id_str = str(end_point_id)
         if self.job_runners.get(run_id_str, None) is not None:
             self.job_runners[run_id_str].send_deployment_delete_request_to_edges(
                 payload, model_msg_object, message_center=message_center)
+        else:
+            # Hotfix: re-instantiate the job runner
+            # TODO(Alay, Raphael): Try to dig into whether re-instantiate the job runner is necessary
+            self.job_runners[run_id_str] = self._generate_job_runner_instance(args)
+            self.job_runners[run_id_str].send_deployment_delete_request_to_edges(
+                payload, model_msg_object, message_center=message_center)
 
     def stop_device_inference_monitor(self, run_id, end_point_name, model_id, model_name, model_version):
         run_id_str = str(run_id)

python/fedml/computing/scheduler/model_scheduler/master_protocol_manager.py

@@ -132,7 +132,7 @@ def callback_delete_deployment(self, topic, payload):
 
         # Send delete deployment request to the edge devices
         FedMLDeployJobRunnerManager.get_instance().send_deployment_delete_request_to_edges(
-            model_msg_object.run_id, payload, model_msg_object, message_center=self.message_center)
+            model_msg_object.run_id, payload, model_msg_object, message_center=self.message_center, args=self.args)
 
         # Stop processes on master
         FedMLDeployJobRunnerManager.get_instance().stop_job_runner(model_msg_object.run_id)