changes v2

Author: Rafi-Microsoft

src/ContentProcessor/src/libs/pipeline/handlers/evaluate_handler.py

@@ -3,8 +3,6 @@
 
 import json
 
-from openai.types.chat.parsed_chat_completion import ParsedChatCompletion
-
 from libs.application.application_context import AppContext
 from libs.azure_helper.model.content_understanding import AnalyzedResult
 from libs.pipeline.entities.pipeline_file import ArtifactType, PipelineLogEntry
@@ -44,19 +42,17 @@ async def execute(self, context: MessageContext) -> StepResult:
             **json.loads(output_file_json_string_from_extract)
         )
 
-        # Get the result from Map step handler - OpenAI
+        # Get the result from Map step handler - Azure AI Foundry
         output_file_json_string_from_map = self.download_output_file_to_json_string(
             processed_by="map",
             artifact_type=ArtifactType.SchemaMappedData,
         )
 
-        # Deserialize the result to ParsedChatCompletion (Azure OpenAI)
-        gpt_result = ParsedChatCompletion(
-            **json.loads(output_file_json_string_from_map)
-        )
+        # Deserialize the result from Azure AI Foundry SDK response
+        gpt_result = json.loads(output_file_json_string_from_map)
 
-        # Mapped Result by GPT
-        parsed_message_from_gpt = gpt_result.choices[0].message.parsed
+        # Mapped Result from Azure AI Foundry
+        parsed_message_from_gpt = gpt_result["choices"][0]["message"]["parsed"]
 
         # Convert the parsed message to a dictionary
         gpt_evaluate_confidence_dict = parsed_message_from_gpt
@@ -69,7 +65,7 @@ async def execute(self, context: MessageContext) -> StepResult:
 
         # Evaluate Confidence Score - GPT
         gpt_confidence_score = gpt_confidence(
-            gpt_evaluate_confidence_dict, gpt_result.choices[0]
+            gpt_evaluate_confidence_dict, gpt_result["choices"][0]
         )
 
         # Merge the confidence scores - Content Understanding and GPT results.
@@ -89,8 +85,8 @@ async def execute(self, context: MessageContext) -> StepResult:
             extracted_result=gpt_evaluate_confidence_dict,
             confidence=merged_confidence_score,
             comparison_result=result_data,
-            prompt_tokens=gpt_result.usage.prompt_tokens,
-            completion_tokens=gpt_result.usage.completion_tokens,
+            prompt_tokens=gpt_result["usage"]["prompt_tokens"],
+            completion_tokens=gpt_result["usage"]["completion_tokens"],
             execution_time=0,
         )
 

src/ContentProcessor/src/libs/pipeline/handlers/logics/evaluate_handler/openai_confidence_evaluator.py

@@ -4,20 +4,19 @@
 import math
 
 import tiktoken
-from openai.types.chat.chat_completion import Choice
 
 from libs.pipeline.handlers.logics.evaluate_handler.confidence import (
     get_confidence_values,
 )
 
 
-def evaluate_confidence(extract_result: dict, choice: Choice, model: str = "gpt-4o"):
+def evaluate_confidence(extract_result: dict, choice: dict, model: str = "gpt-4o"):
     """
-    Evaluate confidence for each field value in the extracted result based on the logprobs of the response from Azure OpenAI.
+    Evaluate confidence for each field value in the extracted result based on the logprobs of the response from Azure AI Foundry.
 
     Args:
         extract_result: The extraction result.
-        choice: The choice object from the OpenAI response.
+        choice: The choice dictionary from the Azure AI Foundry response.
         model: The model used for the response.
 
     Returns:
@@ -30,16 +29,16 @@ def evaluate_confidence(extract_result: dict, choice: Choice, model: str = "gpt-
     encoding = tiktoken.encoding_for_model(model)
 
     # To perform the confidence evaluation, we need the original text from the response, not just the object result.
-    generated_text = choice.message.content
+    generated_text = choice["message"]["content"]
 
-    if choice.logprobs is None:
+    if choice.get("logprobs") is None:
         confidence["_overall"] = 0.0
         return confidence
 
-    logprobs = choice.logprobs.content
+    logprobs = choice["logprobs"]["content"]
 
-    tokens = [token_logprob.token for token_logprob in logprobs]
-    token_logprobs = [token_logprob.logprob for token_logprob in logprobs]
+    tokens = [token_logprob["token"] for token_logprob in logprobs]
+    token_logprobs = [token_logprob["logprob"] for token_logprob in logprobs]
 
     # Encode the entire generated text to map tokens to character positions
     token_offsets = []