Merge pull request #218 from johnoliver/json-schema

Add json_schema support

Author: johnoliver

CHANGELOG.md

@@ -1,3 +1,18 @@
+# 1.3.0
+
+- Added support for Json Schema to Open AI Chat Completions
+- Upgraded to openai sdk 1.0.0-beta.11
+- Added convenience method `FunctionInvocation.withResultTypeAutoConversion` which sets the return type and registers a
+  type converter based on Jackson for the return type.
+
+### Bug Fixes
+
+- Fixed type converters not being passed on to be used in tool invocations
+
+### Breaking Changes
+
+- To support the new Json Schema feature, ResponseFormat has changed from an enum to a class.
+
 # 1.2.2
 
 - Fix bug in `FunctionInvocation` not using per-invocation type conversion when calling `withResultType`.

aiservices/google/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.microsoft.semantic-kernel</groupId>
         <artifactId>semantickernel-parent</artifactId>
-        <version>1.2.3-SNAPSHOT</version>
+        <version>1.3.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

aiservices/huggingface/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.microsoft.semantic-kernel</groupId>
         <artifactId>semantickernel-parent</artifactId>
-        <version>1.2.3-SNAPSHOT</version>
+        <version>1.3.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java

@@ -2,6 +2,7 @@
 package com.microsoft.semantickernel.aiservices.huggingface.services;
 
 import com.microsoft.semantickernel.orchestration.PromptExecutionSettings;
+import com.microsoft.semantickernel.orchestration.responseformat.ResponseFormat;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -38,7 +39,7 @@ public HuggingFacePromptExecutionSettings(PromptExecutionSettings copy) {
             copy.getUser(),
             copy.getStopSequences(),
             copy.getTokenSelectionBiases(),
-            copy.getResponseFormat() == null ? null : copy.getResponseFormat().toString());
+            copy.getResponseFormat() == null ? null : copy.getResponseFormat());
         this.topK = null;
         this.repetitionPenalty = null;
         this.maxTime = null;
@@ -78,7 +79,7 @@ public HuggingFacePromptExecutionSettings(
         String user,
         @Nullable List<String> stopSequences,
         @Nullable Map<Integer, Integer> tokenSelectionBiases,
-        @Nullable String responseFormat,
+        @Nullable ResponseFormat responseFormat,
         @Nullable Integer topK,
         @Nullable Double repetitionPenalty,
         @Nullable Double maxTime,
@@ -119,7 +120,7 @@ public static HuggingFacePromptExecutionSettings fromExecutionSettings(
             promptExecutionSettings.getStopSequences(),
             promptExecutionSettings.getTokenSelectionBiases(),
             promptExecutionSettings.getResponseFormat() != null
-                ? promptExecutionSettings.getResponseFormat().toString()
+                ? promptExecutionSettings.getResponseFormat()
                 : null,
             null,
             null,

aiservices/openai/pom.xml

@@ -1,12 +1,13 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
         <groupId>com.microsoft.semantic-kernel</groupId>
         <artifactId>semantickernel-parent</artifactId>
-        <version>1.2.3-SNAPSHOT</version>
+        <version>1.3.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
@@ -55,6 +56,17 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.github.victools</groupId>
+            <artifactId>jsonschema-generator</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.github.victools</groupId>
+            <artifactId>jsonschema-module-jackson</artifactId>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 
     <build>

aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java

@@ -7,10 +7,13 @@
 import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall;
 import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition;
 import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat;
+import com.azure.ai.openai.models.ChatCompletionsNamedToolSelection;
 import com.azure.ai.openai.models.ChatCompletionsOptions;
 import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat;
 import com.azure.ai.openai.models.ChatCompletionsToolCall;
 import com.azure.ai.openai.models.ChatCompletionsToolDefinition;
+import com.azure.ai.openai.models.ChatCompletionsToolSelection;
+import com.azure.ai.openai.models.ChatCompletionsToolSelectionPreset;
 import com.azure.ai.openai.models.ChatMessageImageContentItem;
 import com.azure.ai.openai.models.ChatMessageImageDetailLevel;
 import com.azure.ai.openai.models.ChatMessageImageUrl;
@@ -23,13 +26,15 @@
 import com.azure.ai.openai.models.ChatResponseMessage;
 import com.azure.ai.openai.models.CompletionsUsage;
 import com.azure.ai.openai.models.FunctionCall;
-import com.azure.core.util.BinaryData;
+import com.azure.json.JsonOptions;
+import com.azure.json.implementation.DefaultJsonReader;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ContainerNode;
 import com.microsoft.semantickernel.Kernel;
 import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
+import com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.ChatCompletionsJsonSchemaResponseFormat;
 import com.microsoft.semantickernel.aiservices.openai.implementation.OpenAIRequestSettings;
 import com.microsoft.semantickernel.contextvariables.ContextVariable;
 import com.microsoft.semantickernel.contextvariables.ContextVariableTypes;
@@ -50,6 +55,8 @@
 import com.microsoft.semantickernel.orchestration.InvocationReturnMode;
 import com.microsoft.semantickernel.orchestration.PromptExecutionSettings;
 import com.microsoft.semantickernel.orchestration.ToolCallBehavior;
+import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema;
+import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat;
 import com.microsoft.semantickernel.semanticfunctions.KernelFunction;
 import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments;
 import com.microsoft.semantickernel.services.chatcompletion.AuthorRole;
@@ -61,6 +68,7 @@
 import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageImageContent;
 import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder;
 import io.opentelemetry.api.trace.Span;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -618,6 +626,8 @@ private Mono<FunctionResult<String>> invokeFunctionTool(
             return function
                 .invokeAsync(kernel)
                 .withArguments(arguments)
+                .withTypes(invocationContext.getContextVariableTypes())
+                .withTypes(contextVariableTypes)
                 .withResultType(contextVariableTypes.getVariableTypeForClass(String.class));
         } catch (JsonProcessingException e) {
             return Mono.error(new SKException("Failed to parse tool arguments", e));
@@ -900,13 +910,21 @@ private static ChatCompletionsOptions getCompletionsOptions(
             .setLogitBias(logit);
 
         if (promptExecutionSettings.getResponseFormat() != null) {
-            switch (promptExecutionSettings.getResponseFormat()) {
+            switch (promptExecutionSettings.getResponseFormat().getType()) {
+                case JSON_SCHEMA:
+                    JsonResponseSchema schema = ((JsonSchemaResponseFormat) promptExecutionSettings
+                        .getResponseFormat())
+                        .getJsonSchema();
+
+                    options.setResponseFormat(new ChatCompletionsJsonSchemaResponseFormat(schema));
+                    break;
                 case JSON_OBJECT:
                     options.setResponseFormat(new ChatCompletionsJsonResponseFormat());
                     break;
                 case TEXT:
                     options.setResponseFormat(new ChatCompletionsTextResponseFormat());
                     break;
+
                 default:
                     throw new SKException(
                         "Unknown response format: " + promptExecutionSettings.getResponseFormat());
@@ -957,9 +975,17 @@ private static void configureToolCallBehaviorOptions(
             try {
                 String json = String.format(
                     "{\"type\":\"function\",\"function\":{\"name\":\"%s\"}}", toolChoiceName);
-                options.setToolChoice(BinaryData.fromObject(new ObjectMapper().readTree(json)));
+
+                options.setToolChoice(
+                    new ChatCompletionsToolSelection(
+                        ChatCompletionsNamedToolSelection.fromJson(
+                            DefaultJsonReader.fromString(
+                                json,
+                                new JsonOptions()))));
             } catch (JsonProcessingException e) {
                 throw SKException.build("Failed to parse tool choice", e);
+            } catch (IOException e) {
+                throw new SKException(e);
             }
             return;
         }
@@ -985,7 +1011,8 @@ private static void configureToolCallBehaviorOptions(
         }
 
         options.setTools(toolDefinitions);
-        options.setToolChoice(BinaryData.fromString("auto"));
+        options.setToolChoice(
+            new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO));
     }
 
     private static boolean hasToolCallBeenExecuted(List<ChatRequestMessage> chatRequestMessages,

aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+package com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat;
+
+import com.azure.ai.openai.models.ChatCompletionsResponseFormat;
+import com.azure.json.JsonWriter;
+import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema;
+import java.io.IOException;
+
+public class ChatCompletionsJsonSchemaResponseFormat extends ChatCompletionsResponseFormat {
+
+    private final JsonResponseSchema schema;
+    private String type = "json_schema";
+
+    public ChatCompletionsJsonSchemaResponseFormat(JsonResponseSchema schema) {
+        this.schema = schema;
+    }
+
+    public String getType() {
+        return this.type;
+    }
+
+    public JsonWriter toJson(JsonWriter jsonWriter) throws IOException {
+        jsonWriter.writeStartObject();
+        jsonWriter.writeStringField("type", this.type);
+        jsonWriter.writeStartObject("json_schema");
+
+        jsonWriter.writeBooleanField("strict", this.schema.isStrict());
+        jsonWriter.writeStringField("name", this.schema.getName());
+
+        jsonWriter.writeRawField("schema", this.schema.getSchema());
+        jsonWriter.writeEndObject();
+        return jsonWriter.writeEndObject();
+    }
+
+}

aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java

@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft. All rights reserved.
+package com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ContainerNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.github.victools.jsonschema.generator.OptionPreset;
+import com.github.victools.jsonschema.generator.SchemaGenerator;
+import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder;
+import com.github.victools.jsonschema.generator.SchemaVersion;
+import com.github.victools.jsonschema.module.jackson.JacksonModule;
+import com.microsoft.semantickernel.orchestration.responseformat.ResponseSchemaGenerator;
+
+public class JacksonResponseFormatGenerator implements ResponseSchemaGenerator {
+
+    private final SchemaGenerator generator;
+
+    public JacksonResponseFormatGenerator() {
+        JacksonModule module = new JacksonModule();
+        SchemaGeneratorConfigBuilder builder = new SchemaGeneratorConfigBuilder(
+            SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON)
+            .with(module);
+
+        builder
+            .forFields()
+            .withRequiredCheck(fieldScope -> {
+                return true;
+            });
+
+        generator = new SchemaGenerator(builder.build());
+    }
+
+    public JacksonResponseFormatGenerator(SchemaGenerator generator) {
+        this.generator = generator;
+    }
+
+    @Override
+    public String generateSchema(Class<?> clazz) {
+        ObjectNode schema = generator.generateSchema(clazz);
+
+        sanitize(schema);
+
+        return schema.toPrettyString();
+    }
+
+    private static void sanitize(ContainerNode schema) {
+        if (schema instanceof ObjectNode) {
+            ((ObjectNode) schema).remove("$schema");
+
+            if (schema.has("type") && schema.get("type").asText().equals("object")) {
+                ((ObjectNode) schema).put("additionalProperties", false);
+            }
+
+            for (JsonNode node : (ObjectNode) schema) {
+                if (node instanceof ContainerNode) {
+                    sanitize((ContainerNode) node);
+                }
+            }
+        } else if (schema instanceof ArrayNode) {
+            for (JsonNode node : (ArrayNode) schema) {
+                if (node instanceof ContainerNode) {
+                    sanitize((ContainerNode) node);
+                }
+            }
+        }
+    }
+}

aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Bar.java

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+package com.microsoft.semantickernel.aiservices.openai.chatcompletion;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Bar {
+
+    private final String bar;
+
+    public Bar(
+        @JsonProperty("bar") String bar) {
+        this.bar = bar;
+    }
+
+    public String getBar() {
+        return bar;
+    }
+
+}

aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Baz.java

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+package com.microsoft.semantickernel.aiservices.openai.chatcompletion;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Baz {
+
+    @JsonProperty("bar")
+    private final Bar bar;
+
+    @JsonCreator
+    public Baz(
+        @JsonProperty("bar") Bar bar) {
+        this.bar = bar;
+    }
+
+    @JsonProperty("bar")
+    public Bar getBar() {
+        return bar;
+    }
+}