Update to Azure OpenAI version 2024-02-01 (#565)

Author: cecheta

.devcontainer/devcontainer.json

@@ -35,7 +35,10 @@
 			"settings": {
 				"azureFunctions.projectSubpath": "code/backend/batch",
                 "python.defaultInterpreterPath": "/usr/local/bin/python",
-				"python.pythonPath": "/usr/local/bin/python"
+				"python.pythonPath": "/usr/local/bin/python",
+				"files.insertFinalNewline": true,
+				"files.trimFinalNewlines": true,
+				"files.trimTrailingWhitespace": true
             }
 		}
 	}

.env.sample

@@ -27,7 +27,7 @@ AZURE_OPENAI_TOP_P=1.0
 AZURE_OPENAI_MAX_TOKENS=1000
 AZURE_OPENAI_STOP_SEQUENCE=
 AZURE_OPENAI_SYSTEM_MESSAGE=You are an AI assistant that helps people find information.
-AZURE_OPENAI_API_VERSION=2023-12-01-preview
+AZURE_OPENAI_API_VERSION=2024-02-01
 AZURE_OPENAI_STREAM=True
 # Backend for processing the documents and application logging in the app
 AzureWebJobsStorage=
@@ -44,8 +44,8 @@ AZURE_FORM_RECOGNIZER_KEY=
 # Azure AI Content Safety for filtering out the inappropriate questions or answers
 AZURE_CONTENT_SAFETY_ENDPOINT=
 AZURE_CONTENT_SAFETY_KEY=
-# Orchestration strategy. Use Azure OpenAI Functions (openai_functions) or LangChain (langchain) for messages orchestration. If you are using a new model version 0613 select "openai_functions" (or "langchain"), if you are using a 0314 model version select "langchain"
-ORCHESTRATION_STRATEGY=openai_functions
+# Orchestration strategy. Use Azure OpenAI Functions (openai_function) or LangChain (langchain) for messages orchestration. If you are using a new model version 0613 select "openai_function" (or "langchain"), if you are using a 0314 model version select "langchain"
+ORCHESTRATION_STRATEGY=openai_function
 #Speech-to-text feature
 AZURE_SPEECH_SERVICE_KEY=
 AZURE_SPEECH_SERVICE_REGION=
@@ -54,4 +54,4 @@ AZURE_SPEECH_SERVICE_REGION=
 # When USE_KEY_VAULT=true, please make sure to set AZURE_KEY_VAULT_ENDPOINT
 AZURE_AUTH_TYPE=keys
 USE_KEY_VAULT=true
-AZURE_KEY_VAULT_ENDPOINT=
+AZURE_KEY_VAULT_ENDPOINT=

code/app.py

@@ -47,77 +47,70 @@ def prepare_body_headers_with_data(request):
 
     body = {
         "messages": request_messages,
-        "temperature": env_helper.AZURE_OPENAI_TEMPERATURE,
-        "max_tokens": env_helper.AZURE_OPENAI_MAX_TOKENS,
-        "top_p": env_helper.AZURE_OPENAI_TOP_P,
+        "temperature": float(env_helper.AZURE_OPENAI_TEMPERATURE),
+        "max_tokens": int(env_helper.AZURE_OPENAI_MAX_TOKENS),
+        "top_p": float(env_helper.AZURE_OPENAI_TOP_P),
         "stop": (
             env_helper.AZURE_OPENAI_STOP_SEQUENCE.split("|")
             if env_helper.AZURE_OPENAI_STOP_SEQUENCE
             else None
         ),
         "stream": env_helper.SHOULD_STREAM,
-        "dataSources": [
+        "data_sources": [
             {
-                "type": "AzureCognitiveSearch",
+                "type": "azure_search",
                 "parameters": {
+                    # authentication is set below
                     "endpoint": env_helper.AZURE_SEARCH_SERVICE,
-                    "key": env_helper.AZURE_SEARCH_KEY,
-                    "indexName": env_helper.AZURE_SEARCH_INDEX,
-                    "fieldsMapping": {
-                        "contentField": (
+                    "index_name": env_helper.AZURE_SEARCH_INDEX,
+                    "fields_mapping": {
+                        "content_fields": (
                             env_helper.AZURE_SEARCH_CONTENT_COLUMNS.split("|")
                             if env_helper.AZURE_SEARCH_CONTENT_COLUMNS
                             else []
                         ),
-                        "titleField": (
-                            env_helper.AZURE_SEARCH_TITLE_COLUMN
-                            if env_helper.AZURE_SEARCH_TITLE_COLUMN
-                            else None
-                        ),
-                        "urlField": (
-                            env_helper.AZURE_SEARCH_URL_COLUMN
-                            if env_helper.AZURE_SEARCH_URL_COLUMN
-                            else None
-                        ),
-                        "filepathField": (
-                            env_helper.AZURE_SEARCH_FILENAME_COLUMN
-                            if env_helper.AZURE_SEARCH_FILENAME_COLUMN
-                            else None
+                        "title_field": env_helper.AZURE_SEARCH_TITLE_COLUMN or None,
+                        "url_field": env_helper.AZURE_SEARCH_URL_COLUMN or None,
+                        "filepath_field": (
+                            env_helper.AZURE_SEARCH_FILENAME_COLUMN or None
                         ),
                     },
-                    "inScope": env_helper.AZURE_SEARCH_ENABLE_IN_DOMAIN,
-                    "topNDocuments": env_helper.AZURE_SEARCH_TOP_K,
-                    "queryType": (
+                    "in_scope": env_helper.AZURE_SEARCH_ENABLE_IN_DOMAIN,
+                    "top_n_documents": env_helper.AZURE_SEARCH_TOP_K,
+                    "query_type": (
                         "semantic"
                         if env_helper.AZURE_SEARCH_USE_SEMANTIC_SEARCH
                         else "simple"
                     ),
-                    "semanticConfiguration": (
+                    "semantic_configuration": (
                         env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG
                         if env_helper.AZURE_SEARCH_USE_SEMANTIC_SEARCH
                         and env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG
                         else ""
                     ),
-                    "roleInformation": env_helper.AZURE_OPENAI_SYSTEM_MESSAGE,
+                    "role_information": env_helper.AZURE_OPENAI_SYSTEM_MESSAGE,
                 },
             }
         ],
     }
 
-    chatgpt_url = f"{env_helper.AZURE_OPENAI_ENDPOINT}openai/deployments/{env_helper.AZURE_OPENAI_MODEL}"
-    if env_helper.is_chat_model():
-        chatgpt_url += "/chat/completions?api-version=2023-12-01-preview"
-    else:
-        chatgpt_url += "/completions?api-version=2023-12-01-preview"
-
     headers = {
         "Content-Type": "application/json",
-        "api-key": env_helper.AZURE_OPENAI_API_KEY,
-        "chatgpt_url": chatgpt_url,
-        "chatgpt_key": env_helper.AZURE_OPENAI_API_KEY,
         "x-ms-useragent": "GitHubSampleWebApp/PublicAPI/1.0.0",
     }
 
+    if env_helper.AZURE_AUTH_TYPE == "rbac":
+        body["data_sources"][0]["parameters"]["authentication"] = {
+            "type": "system_assigned_managed_identity"
+        }
+        headers["Authorization"] = f"Bearer {env_helper.AZURE_TOKEN_PROVIDER()}"
+    else:
+        body["data_sources"][0]["parameters"]["authentication"] = {
+            "type": "api_key",
+            "key": env_helper.AZURE_SEARCH_KEY,
+        }
+        headers["api-key"] = env_helper.AZURE_OPENAI_API_KEY
+
     return body, headers
 
 
@@ -128,37 +121,54 @@ def stream_with_data(body, headers, endpoint):
         "model": "",
         "created": 0,
         "object": "",
-        "choices": [{"messages": []}],
+        "choices": [
+            {
+                "messages": [
+                    {
+                        "content": "",
+                        "end_turn": False,
+                        "role": "tool",
+                    },
+                    {
+                        "content": "",
+                        "end_turn": False,
+                        "role": "assistant",
+                    },
+                ]
+            }
+        ],
     }
     try:
         with s.post(endpoint, json=body, headers=headers, stream=True) as r:
             for line in r.iter_lines(chunk_size=10):
                 if line:
-                    lineJson = json.loads(line.lstrip(b"data:").decode("utf-8"))
+                    lineJson = json.loads(line.lstrip(b"data: ").decode("utf-8"))
                     if "error" in lineJson:
                         yield json.dumps(lineJson, ensure_ascii=False) + "\n"
+                        return
+
+                    if lineJson["choices"][0]["end_turn"]:
+                        response["choices"][0]["messages"][1]["end_turn"] = True
+                        yield json.dumps(response, ensure_ascii=False) + "\n"
+                        return
+
                     response["id"] = lineJson["id"]
                     response["model"] = lineJson["model"]
                     response["created"] = lineJson["created"]
                     response["object"] = lineJson["object"]
 
-                    role = lineJson["choices"][0]["messages"][0]["delta"].get("role")
-                    if role == "tool":
-                        response["choices"][0]["messages"].append(
-                            lineJson["choices"][0]["messages"][0]["delta"]
-                        )
-                    elif role == "assistant":
-                        response["choices"][0]["messages"].append(
-                            {"role": "assistant", "content": ""}
+                    delta = lineJson["choices"][0]["delta"]
+                    role = delta.get("role")
+
+                    if role == "assistant":
+                        response["choices"][0]["messages"][0]["content"] = json.dumps(
+                            delta["context"],
+                            ensure_ascii=False,
                         )
                     else:
-                        deltaText = lineJson["choices"][0]["messages"][0]["delta"][
+                        response["choices"][0]["messages"][1]["content"] += delta[
                             "content"
                         ]
-                        if deltaText != "[DONE]":
-                            response["choices"][0]["messages"][1][
-                                "content"
-                            ] += deltaText
 
                     yield json.dumps(response, ensure_ascii=False) + "\n"
     except Exception as e:
@@ -167,39 +177,68 @@ def stream_with_data(body, headers, endpoint):
 
 def conversation_with_data(request):
     body, headers = prepare_body_headers_with_data(request)
-    endpoint = f"{env_helper.AZURE_OPENAI_ENDPOINT}openai/deployments/{env_helper.AZURE_OPENAI_MODEL}/extensions/chat/completions?api-version={env_helper.AZURE_OPENAI_API_VERSION}"
+    endpoint = f"{env_helper.AZURE_OPENAI_ENDPOINT}openai/deployments/{env_helper.AZURE_OPENAI_MODEL}/chat/completions?api-version={env_helper.AZURE_OPENAI_API_VERSION}"
 
     if not env_helper.SHOULD_STREAM:
         r = requests.post(endpoint, headers=headers, json=body)
         status_code = r.status_code
         r = r.json()
 
-        return Response(json.dumps(r, ensure_ascii=False), status=status_code)
+        response = {
+            "id": r["id"],
+            "model": r["model"],
+            "created": r["created"],
+            "object": r["object"],
+            "choices": [
+                {
+                    "messages": [
+                        {
+                            "content": json.dumps(
+                                r["choices"][0]["message"]["context"],
+                                ensure_ascii=False,
+                            ),
+                            "end_turn": False,
+                            "role": "tool",
+                        },
+                        {
+                            "content": r["choices"][0]["message"]["content"],
+                            "end_turn": True,
+                            "role": "assistant",
+                        },
+                    ]
+                }
+            ],
+        }
+
+        return Response(json.dumps(response, ensure_ascii=False), status=status_code)
     else:
-        if request.method == "POST":
-            return Response(
-                stream_with_data(body, headers, endpoint),
-                mimetype="application/json-lines",
-            )
-        else:
-            return Response(None, mimetype="application/json-lines")
+        return Response(
+            stream_with_data(body, headers, endpoint),
+            mimetype="application/json-lines",
+        )
 
 
 def stream_without_data(response):
     responseText = ""
     for line in response:
-        deltaText = line["choices"][0]["delta"].get("content")
-        if deltaText and deltaText != "[DONE]":
-            responseText += deltaText
+        if not line.choices:
+            continue
+
+        deltaText = line.choices[0].delta.content
+
+        if deltaText is None:
+            return
+
+        responseText += deltaText
 
         response_obj = {
-            "id": line["id"],
-            "model": line["model"],
-            "created": line["created"],
-            "object": line["object"],
+            "id": line.id,
+            "model": line.model,
+            "created": line.created,
+            "object": line.object,
             "choices": [{"messages": [{"role": "assistant", "content": responseText}]}],
         }
-        yield json.dumps(response_obj).replace("\n", "\\n") + "\n"
+        yield json.dumps(response_obj, ensure_ascii=False) + "\n"
 
 
 def conversation_without_data(request):
@@ -239,7 +278,7 @@ def conversation_without_data(request):
 
     if not env_helper.SHOULD_STREAM:
         response_obj = {
-            "id": response,
+            "id": response.id,
             "model": response.model,
             "created": response.created,
             "object": response.object,
@@ -257,15 +296,12 @@ def conversation_without_data(request):
 
         return jsonify(response_obj), 200
     else:
-        if request.method == "POST":
-            return Response(
-                stream_without_data(response), mimetype="application/json-lines"
-            )
-        else:
-            return Response(None, mimetype="application/json-lines")
+        return Response(
+            stream_without_data(response), mimetype="application/json-lines"
+        )
 
 
-@app.route("/api/conversation/azure_byod", methods=["GET", "POST"])
+@app.route("/api/conversation/azure_byod", methods=["POST"])
 def conversation_azure_byod():
     try:
         if env_helper.should_use_data():
@@ -297,7 +333,7 @@ def get_orchestrator_config():
     return ConfigHelper.get_active_config_or_default().orchestrator
 
 
-@app.route("/api/conversation/custom", methods=["GET", "POST"])
+@app.route("/api/conversation/custom", methods=["POST"])
 def conversation_custom():
     message_orchestrator = get_message_orchestrator()
 

code/backend/batch/utilities/helpers/EnvHelper.py

@@ -57,16 +57,16 @@ def __init__(self, **kwargs) -> None:
         self.AZURE_OPENAI_MODEL_NAME = os.getenv(
             "AZURE_OPENAI_MODEL_NAME", "gpt-35-turbo"
         )
-        self.AZURE_OPENAI_TEMPERATURE = os.getenv("AZURE_OPENAI_TEMPERATURE", 0)
-        self.AZURE_OPENAI_TOP_P = os.getenv("AZURE_OPENAI_TOP_P", 1.0)
-        self.AZURE_OPENAI_MAX_TOKENS = os.getenv("AZURE_OPENAI_MAX_TOKENS", 1000)
+        self.AZURE_OPENAI_TEMPERATURE = os.getenv("AZURE_OPENAI_TEMPERATURE", "0")
+        self.AZURE_OPENAI_TOP_P = os.getenv("AZURE_OPENAI_TOP_P", "1.0")
+        self.AZURE_OPENAI_MAX_TOKENS = os.getenv("AZURE_OPENAI_MAX_TOKENS", "1000")
         self.AZURE_OPENAI_STOP_SEQUENCE = os.getenv("AZURE_OPENAI_STOP_SEQUENCE", "")
         self.AZURE_OPENAI_SYSTEM_MESSAGE = os.getenv(
             "AZURE_OPENAI_SYSTEM_MESSAGE",
             "You are an AI assistant that helps people find information.",
         )
         self.AZURE_OPENAI_API_VERSION = os.getenv(
-            "AZURE_OPENAI_API_VERSION", "2023-12-01-preview"
+            "AZURE_OPENAI_API_VERSION", "2024-02-01"
         )
         self.AZURE_OPENAI_STREAM = os.getenv("AZURE_OPENAI_STREAM", "true")
         self.AZURE_OPENAI_EMBEDDING_MODEL = os.getenv(
@@ -160,7 +160,7 @@ def should_use_data(self) -> bool:
         if (
             self.AZURE_SEARCH_SERVICE
             and self.AZURE_SEARCH_INDEX
-            and self.AZURE_SEARCH_KEY
+            and (self.AZURE_SEARCH_KEY or self.AZURE_AUTH_TYPE == "rbac")
         ):
             return True
         return False

code/tests/functional/backend_api/conftest.py

@@ -52,7 +52,7 @@ def app_port() -> int:
 
 
 @pytest.fixture(scope="session")
-def app_url(app_port: int) -> int:
+def app_url(app_port: int) -> str:
     return f"http://localhost:{app_port}"
 
 

code/tests/functional/backend_api/test_conversation_custom.py

@@ -76,7 +76,7 @@ def test_post_makes_correct_call_to_openai_embeddings(
                 "Authorization": f"Bearer {app_config.get('AZURE_OPENAI_API_KEY')}",
                 "Api-Key": app_config.get("AZURE_OPENAI_API_KEY"),
             },
-            query_string="api-version=2023-12-01-preview",
+            query_string="api-version=2024-02-01",
             times=2,
         ),
     )
@@ -193,7 +193,7 @@ def test_post_makes_correct_call_to_openai_chat_completions(
                 "Authorization": f"Bearer {app_config.get('AZURE_OPENAI_API_KEY')}",
                 "Api-Key": app_config.get("AZURE_OPENAI_API_KEY"),
             },
-            query_string="api-version=2023-12-01-preview",
+            query_string="api-version=2024-02-01",
             times=1,
         ),
     )