Merge pull request #2105 from FedML-AI/alexleung/dev_branch_latest

Alexleung/dev branch latest

Author: fedml-alex

python/examples/launch/serve_mnist/fedml_model_config.yaml

@@ -1,6 +1,8 @@
 workspace: "./"
 entry_point: "mnist_serve_main.py"
 
+auto_detect_public_ip: true
+
 data_cache_dir: ""
 bootstrap: ""
 

python/fedml/computing/scheduler/comm_utils/job_monitor.py

@@ -223,7 +223,7 @@ def monitor_replicas_number():
             curr_version = fedml.get_env_version()
             num_replica_url_path = "fedmlModelServer/api/v1/endpoint/replica-info"
             mlops_prefix = fedml._get_backend_service()
-            url = f"{mlops_prefix}{num_replica_url_path}"
+            url = f"{mlops_prefix}/{num_replica_url_path}"
 
             cached_token = FedMLModelCache.get_instance().get_end_point_token_with_eid(endpoint_id)
             if cached_token is None:

python/fedml/computing/scheduler/model_scheduler/device_model_inference.py

@@ -26,35 +26,35 @@
     pass
 
 
-# class Settings(BaseSettings):
-#     redis_addr: str
-#     redis_port: str
-#     redis_password: str
-#     end_point_name: str
-#     model_name: str
-#     model_version: str
-#     model_infer_url: str
-#     version: str
-#     use_mqtt_inference: bool
-#     use_worker_gateway: bool
-#     ext_info: str
+class Settings(BaseSettings):
+    redis_addr: str
+    redis_port: str
+    redis_password: str
+    end_point_name: str
+    model_name: str
+    model_version: str
+    model_infer_url: str
+    version: str
+    use_mqtt_inference: bool
+    use_worker_gateway: bool
+    ext_info: str
+
+
+settings = Settings()
+
+# class settings:
+#     redis_addr = "127.0.0.1"
+#     redis_port = 6379
+#     redis_password = "fedml_default"
+#     end_point_name = ""
+#     model_name = ""
+#     model_version = ""
+#     model_infer_url = "127.0.0.1"
+#     version = "dev"
+#     use_mqtt_inference = False
+#     use_worker_gateway = False
+#     ext_info = "2b34303961245c4f175f2236282d7a272c040b0904747579087f6a760112030109010c215d54505707140005190a051c347f365c4a430c020a7d39120e26032a78730f797f7c031f0901657e75"
 #
-#
-# settings = Settings()
-
-class settings:
-    redis_addr = "127.0.0.1"
-    redis_port = 6379
-    redis_password = "fedml_default"
-    end_point_name = ""
-    model_name = ""
-    model_version = ""
-    model_infer_url = "127.0.0.1"
-    version = "dev"
-    use_mqtt_inference = False
-    use_worker_gateway = False
-    ext_info = "2b34303961245c4f175f2236282d7a272c040b0904747579087f6a760112030109010c215d54505707140005190a051c347f365c4a430c020a7d39120e26032a78730f797f7c031f0901657e75"
-
 
 api = FastAPI()
 

python/fedml/computing/scheduler/model_scheduler/master_job_runner.py

@@ -115,8 +115,7 @@ def run_impl(
 
         # start unified inference server
         self.start_device_inference_gateway(
-            run_id, end_point_name, model_id, model_name, model_version,
-            agent_config=self.agent_config, inference_port=inference_port)
+            inference_port=inference_port, agent_config=self.agent_config)
 
         # start inference monitor server
         self.stop_device_inference_monitor(
@@ -464,20 +463,16 @@ def process_deployment_result_message(self, topic=None, payload=None):
 
     @staticmethod
     def start_device_inference_gateway(
-            run_id, end_point_name, model_id,
-            model_name, model_version, inference_port=ServerConstants.MODEL_INFERENCE_DEFAULT_PORT,
+            inference_port=ServerConstants.MODEL_INFERENCE_DEFAULT_PORT,
             agent_config=None, redis_addr="localhost", redis_port=6379, redis_password="fedml_default"
     ):
         # start unified inference server
-        running_model_name = ServerConstants.get_running_model_name(end_point_name,
-                                                                    model_name, model_version, run_id, model_id)
         python_program = get_python_program()
         master_port = os.getenv("FEDML_MASTER_PORT", None)
         if master_port is not None:
             inference_port = int(master_port)
         if not ServerConstants.is_running_on_k8s():
-            logging.info(f"start the model inference gateway, end point {run_id}, "
-                         f"model name {model_name} at port {inference_port}...")
+            logging.info(f"start the model inference gateway...")
             use_mqtt_inference = os.getenv("FEDML_USE_MQTT_INFERENCE", "False")
             use_mqtt_inference = True if use_mqtt_inference.lower() == 'true' else False
             use_worker_gateway = os.getenv("FEDML_USE_WORKER_GATEWAY", "False")
@@ -501,8 +496,8 @@ def start_device_inference_gateway(
                     "USE_MQTT_INFERENCE={} USE_WORKER_GATEWAY={} EXT_INFO={} "
                     "{} -m uvicorn {} --host 0.0.0.0 --port {} --reload --reload-delay 3 --reload-dir {} "
                     "--log-level critical".format(
-                        redis_addr, redis_port, redis_password, end_point_name,
-                        model_name, model_version, "", fedml.get_env_version(), use_mqtt_inference,
+                        redis_addr, str(redis_port), redis_password, "",
+                        "", "", "", fedml.get_env_version(), use_mqtt_inference,
                         use_worker_gateway, ext_info, python_program, inference_gw_cmd, str(inference_port),
                         fedml_base_dir),
                     should_capture_stdout=False, should_capture_stderr=False)
@@ -545,6 +540,14 @@ def stop_device_inference_monitor(run_id, end_point_name, model_id, model_name,
     def recover_inference_and_monitor():
         # noinspection PyBroadException
         try:
+            agent_config = dict()
+            try:
+                agent_config["mqtt_config"], _, _, _ = MLOpsConfigs.fetch_all_configs()
+            except Exception as e:
+                pass
+
+            FedMLDeployMasterJobRunner.start_device_inference_gateway(agent_config=agent_config)
+
             history_jobs = FedMLServerDataInterface.get_instance().get_history_jobs()
             for job in history_jobs.job_list:
                 if job.running_json is None:
@@ -563,16 +566,6 @@ def recover_inference_and_monitor():
                 if not is_activated:
                     continue
 
-                agent_config = dict()
-                try:
-                    agent_config["mqtt_config"], _, _, _ = MLOpsConfigs.fetch_all_configs()
-                except Exception as e:
-                    pass
-
-                FedMLDeployMasterJobRunner.start_device_inference_gateway(
-                    run_id, end_point_name, model_id, model_name, model_version, inference_port=inference_port,
-                    agent_config=agent_config)
-
                 FedMLDeployMasterJobRunner.stop_device_inference_monitor(
                     run_id, end_point_name, model_id, model_name, model_version)
                 FedMLDeployMasterJobRunner.start_device_inference_monitor(