fix(evaluators): update route api (#3977)

Author: nina-kollman

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluator.py

@@ -54,6 +54,7 @@ def __init__(self, async_http_client: httpx.AsyncClient):
 
     @staticmethod
     def _build_evaluator_request(
+        evaluator_slug: str,
         task_id: str,
         experiment_id: str,
         experiment_run_id: str,
@@ -69,21 +70,25 @@ def _build_evaluator_request(
             input_schema_mapping=schema_mapping,
             evaluator_version=evaluator_version,
             evaluator_config=evaluator_config,
+            evaluator_slug=evaluator_slug,
             task_id=task_id,
             experiment_id=experiment_id,
             experiment_run_id=experiment_run_id,
         )
 
-    async def _execute_evaluator_request(
+    async def _execute_experiment_evaluator_request(
         self,
         evaluator_slug: str,
+        experiment_slug: str,
+        experiment_run_id: str,
+        task_id: str,
         request: ExecuteEvaluatorRequest,
         timeout_in_sec: int = 120,
     ) -> ExecuteEvaluatorResponse:
         """Execute evaluator request and return response"""
         body = request.model_dump()
         client = self._async_http_client
-        full_url = f"/v2/evaluators/slug/{evaluator_slug}/execute"
+        full_url = f"/v2/experiments/{experiment_slug}/runs/{experiment_run_id}/tasks/{task_id}"
         response = await client.post(
             full_url, json=body, timeout=httpx.Timeout(timeout_in_sec)
         )
@@ -99,6 +104,7 @@ async def _execute_evaluator_request(
     async def run_experiment_evaluator(
         self,
         evaluator_slug: str,
+        experiment_slug: str,
         task_id: str,
         experiment_id: str,
         experiment_run_id: str,
@@ -126,11 +132,11 @@ async def run_experiment_evaluator(
         _validate_evaluator_input(evaluator_slug, input, evaluator_config)
 
         request = self._build_evaluator_request(
-            task_id, experiment_id, experiment_run_id, input, evaluator_version, evaluator_config
+            evaluator_slug, task_id, experiment_id, experiment_run_id, input, evaluator_version, evaluator_config
         )
 
-        execute_response = await self._execute_evaluator_request(
-            evaluator_slug, request, timeout_in_sec
+        execute_response = await self._execute_experiment_evaluator_request(
+            evaluator_slug, experiment_slug, experiment_run_id, task_id, request, timeout_in_sec
         )
 
         sse_client = SSEClient(shared_client=self._async_http_client)
@@ -145,6 +151,7 @@ async def run_experiment_evaluator(
     async def trigger_experiment_evaluator(
         self,
         evaluator_slug: str,
+        experiment_slug: str,
         task_id: str,
         experiment_id: str,
         experiment_run_id: str,
@@ -170,11 +177,11 @@ async def trigger_experiment_evaluator(
         _validate_evaluator_input(evaluator_slug, input, evaluator_config)
 
         request = self._build_evaluator_request(
-            task_id, experiment_id, experiment_run_id, input, evaluator_version, evaluator_config
+            evaluator_slug, task_id, experiment_id, experiment_run_id, input, evaluator_version, evaluator_config
         )
 
-        execute_response = await self._execute_evaluator_request(
-            evaluator_slug, request, 120
+        execute_response = await self._execute_experiment_evaluator_request(
+            evaluator_slug, experiment_slug, experiment_run_id, task_id, request, 120
         )
 
         # Return execution_id without waiting for SSE result

packages/traceloop-sdk/traceloop/sdk/evaluator/model.py

@@ -19,6 +19,7 @@ class ExecuteEvaluatorRequest(BaseModel):
     input_schema_mapping: InputSchemaMapping
     evaluator_version: Optional[str] = None
     evaluator_config: Optional[Dict[str, Any]] = None
+    evaluator_slug: str
     task_id: str
     experiment_id: str
     experiment_run_id: str

packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py

@@ -190,6 +190,7 @@ async def run_single_row(row: Optional[Dict[str, Any]]) -> TaskResponse:
                                 eval_result = (
                                     await self._evaluator.run_experiment_evaluator(
                                         evaluator_slug=evaluator_slug,
+                                        experiment_slug=experiment_slug,
                                         evaluator_version=evaluator_version,
                                         evaluator_config=evaluator_config,
                                         task_id=task_id,
@@ -203,6 +204,7 @@ async def run_single_row(row: Optional[Dict[str, Any]]) -> TaskResponse:
                             else:
                                 await self._evaluator.trigger_experiment_evaluator(
                                     evaluator_slug=evaluator_slug,
+                                    experiment_slug=experiment_slug,
                                     evaluator_version=evaluator_version,
                                     evaluator_config=evaluator_config,
                                     task_id=task_id,
@@ -436,7 +438,7 @@ def _create_task(
             output=task_output,
         )
         response = self._http_client.post(
-            f"/experiments/{experiment_slug}/runs/{experiment_run_id}/task",
+            f"/experiments/{experiment_slug}/runs/{experiment_run_id}/tasks",
             body.model_dump(mode="json"),
         )
         if response is None:

packages/traceloop-sdk/traceloop/sdk/guardrails/guardrails.py

@@ -201,6 +201,7 @@ async def execute_evaluator(
             # Use dummy IDs for guardrails (they don't need experiment tracking)
             result = await self._evaluator.run_experiment_evaluator(
                 evaluator_slug=slug,
+                experiment_slug="guardrail",
                 task_id="guardrail",
                 experiment_id="guardrail",
                 experiment_run_id="guardrail",