Skip to content

Commit 0650bd2

Browse files
author
Milder Hernandez Cagua
committed
Merge 'main' into 'postgres-vector-index'
2 parents 5d9448a + 2f7d845 commit 0650bd2

73 files changed

Lines changed: 2070 additions & 683 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.microsoft.semantickernel.exceptions.AIException;
2222
import com.microsoft.semantickernel.exceptions.SKCheckedException;
2323
import com.microsoft.semantickernel.exceptions.SKException;
24+
import com.microsoft.semantickernel.localization.SemanticKernelResources;
2425
import com.microsoft.semantickernel.orchestration.FunctionResult;
2526
import com.microsoft.semantickernel.orchestration.FunctionResultMetadata;
2627
import com.microsoft.semantickernel.orchestration.InvocationContext;
@@ -97,7 +98,8 @@ private Mono<List<ChatMessageContent<?>>> internalChatMessageContentsAsync(
9798
try {
9899
GenerativeModel model = getGenerativeModel(kernel, invocationContext);
99100
return MonoConverter.fromApiFuture(model.generateContentAsync(contents))
100-
.doOnError(e -> LOGGER.error("Error generating chat completion", e))
101+
.doOnError(e -> LOGGER.error(
102+
SemanticKernelResources.getString("error.generating.chat.completion"), e))
101103
.flatMap(result -> {
102104
// Get ChatMessageContent from the response
103105
GeminiChatMessageContent<?> response = getGeminiChatMessageContentFromResponse(
@@ -253,7 +255,8 @@ private GenerativeModel getGenerativeModel(@Nullable Kernel kernel,
253255

254256
if (settings.getResultsPerPrompt() < 1
255257
|| settings.getResultsPerPrompt() > MAX_RESULTS_PER_PROMPT) {
256-
throw SKCheckedException.build("Error building generative model.",
258+
throw SKCheckedException.build(
259+
SemanticKernelResources.getString("error.building.generative.model"),
257260
new AIException(AIException.ErrorCodes.INVALID_REQUEST,
258261
String.format(
259262
"Results per prompt must be in range between 1 and %d, inclusive.",

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.microsoft.semantickernel.tests.connectors.memory;
22

3+
import com.fasterxml.jackson.annotation.JsonCreator;
4+
import com.fasterxml.jackson.annotation.JsonProperty;
35
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute;
46
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute;
57
import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute;
@@ -13,8 +15,10 @@ public class Hotel {
1315
private final String name;
1416
@VectorStoreRecordDataAttribute
1517
private final int code;
18+
@JsonProperty("summary")
1619
@VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding")
1720
private final String description;
21+
@JsonProperty("summaryEmbedding")
1822
@VectorStoreRecordVectorAttribute(dimensions = 3)
1923
private final List<Float> descriptionEmbedding;
2024
@VectorStoreRecordVectorAttribute(dimensions = 3, indexKind = "hnsw", distanceFunction = "cosine")
@@ -26,7 +30,14 @@ public Hotel() {
2630
this(null, null, 0, null, null, 0.0);
2731
}
2832

29-
public Hotel(String id, String name, int code, String description, List<Float> descriptionEmbedding, double rating) {
33+
@JsonCreator
34+
public Hotel(
35+
@JsonProperty("id") String id,
36+
@JsonProperty("name") String name,
37+
@JsonProperty("code") int code,
38+
@JsonProperty("summary") String description,
39+
@JsonProperty("summaryVector") List<Float> descriptionEmbedding,
40+
@JsonProperty("rating") double rating) {
3041
this.id = id;
3142
this.name = name;
3243
this.code = code;

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore;
44
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions;
55
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider;
6+
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions;
67
import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider;
78
import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider;
89
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
@@ -16,7 +17,6 @@
1617
import org.testcontainers.junit.jupiter.Testcontainers;
1718
import org.testcontainers.utility.DockerImageName;
1819

19-
import javax.annotation.Nonnull;
2020
import javax.sql.DataSource;
2121
import java.util.Arrays;
2222
import java.util.List;
@@ -89,7 +89,10 @@ public void getCollectionNamesAsync(QueryProvider provider) {
8989
List<String> collectionNames = Arrays.asList("collection1", "collection2", "collection3");
9090

9191
for (String collectionName : collectionNames) {
92-
vectorStore.getCollection(collectionName, Hotel.class, null).createCollectionAsync().block();
92+
vectorStore.getCollection(collectionName,
93+
JDBCVectorStoreRecordCollectionOptions.<Hotel>builder()
94+
.withRecordClass(Hotel.class)
95+
.build()).createCollectionAsync().block();
9396
}
9497

9598
List<String> retrievedCollectionNames = vectorStore.getCollectionNamesAsync().block();

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
package com.microsoft.semantickernel.tests.connectors.memory.redis;
2+
3+
import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection;
4+
import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions;
5+
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField;
6+
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition;
7+
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField;
8+
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField;
9+
import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField;
10+
import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions;
11+
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
12+
import com.redis.testcontainers.RedisContainer;
13+
import org.junit.jupiter.api.BeforeAll;
14+
import org.junit.jupiter.api.MethodOrderer;
15+
import org.junit.jupiter.api.Order;
16+
import org.junit.jupiter.api.Test;
17+
import org.junit.jupiter.api.TestMethodOrder;
18+
import org.junit.jupiter.params.ParameterizedTest;
19+
import org.junit.jupiter.params.provider.EnumSource;
20+
import org.testcontainers.junit.jupiter.Container;
21+
import org.testcontainers.junit.jupiter.Testcontainers;
22+
import redis.clients.jedis.JedisPooled;
23+
24+
import javax.annotation.Nonnull;
25+
import java.util.ArrayList;
26+
import java.util.Arrays;
27+
import java.util.HashMap;
28+
import java.util.List;
29+
import java.util.Map;
30+
31+
import static org.junit.jupiter.api.Assertions.assertEquals;
32+
import static org.junit.jupiter.api.Assertions.assertNotNull;
33+
import static org.junit.jupiter.api.Assertions.assertNull;
34+
35+
@Testcontainers
36+
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
37+
public class RedisHashSetVectorStoreRecordCollectionTest {
38+
39+
@Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest");
40+
41+
private static final Map<RecordCollectionOptions, RedisHashSetVectorStoreRecordCollectionOptions<Hotel>> optionsMap = new HashMap<>();
42+
43+
public enum RecordCollectionOptions {
44+
DEFAULT, WITH_CUSTOM_DEFINITION
45+
}
46+
47+
@BeforeAll
48+
static void setup() {
49+
optionsMap.put(RecordCollectionOptions.DEFAULT, RedisHashSetVectorStoreRecordCollectionOptions.<Hotel>builder()
50+
.withRecordClass(Hotel.class)
51+
.build());
52+
53+
List<VectorStoreRecordField> fields = new ArrayList<>();
54+
fields.add(VectorStoreRecordKeyField.builder()
55+
.withName("id")
56+
.withFieldType(String.class)
57+
.build());
58+
fields.add(VectorStoreRecordDataField.builder()
59+
.withName("name")
60+
.withFieldType(String.class)
61+
.build());
62+
fields.add(VectorStoreRecordDataField.builder()
63+
.withName("code")
64+
.withFieldType(Integer.class)
65+
.build());
66+
fields.add(VectorStoreRecordDataField.builder()
67+
.withName("description")
68+
.withStorageName("summary")
69+
.withFieldType(String.class)
70+
.withHasEmbedding(true)
71+
.withEmbeddingFieldName("descriptionEmbedding")
72+
.build());
73+
fields.add(VectorStoreRecordVectorField.builder()
74+
.withName("descriptionEmbedding")
75+
.withStorageName("summaryEmbedding")
76+
.withFieldType(List.class)
77+
.withDimensions(768)
78+
.build());
79+
fields.add(VectorStoreRecordDataField.builder()
80+
.withName("rating")
81+
.withFieldType(Double.class)
82+
.build());
83+
VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields);
84+
85+
optionsMap.put(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, RedisHashSetVectorStoreRecordCollectionOptions.<Hotel>builder()
86+
.withRecordClass(Hotel.class)
87+
.withRecordDefinition(recordDefinition)
88+
.build());
89+
}
90+
91+
private RedisHashSetVectorStoreRecordCollection<Hotel> buildrecordCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions<Hotel> options, @Nonnull String collectionName) {
92+
return new RedisHashSetVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisHashSetVectorStoreRecordCollectionOptions.<Hotel>builder()
93+
.withRecordClass(options.getRecordClass())
94+
.withVectorStoreRecordMapper(options.getVectorStoreRecordMapper())
95+
.withRecordDefinition(options.getRecordDefinition())
96+
.withPrefixCollectionName(options.isPrefixCollectionName())
97+
.build());
98+
}
99+
100+
private List<Hotel> getHotels() {
101+
return List.of(
102+
new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0),
103+
new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0),
104+
new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0),
105+
new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0),
106+
new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0)
107+
);
108+
}
109+
110+
@Order(1)
111+
@ParameterizedTest
112+
@EnumSource(RecordCollectionOptions.class)
113+
public void buildrecordCollection(RecordCollectionOptions options) {
114+
assertNotNull(buildrecordCollection(optionsMap.get(options), options.name()));
115+
}
116+
117+
@Order(2)
118+
@ParameterizedTest
119+
@EnumSource(RecordCollectionOptions.class)
120+
public void createCollectionAsync(RecordCollectionOptions options) {
121+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
122+
123+
assertEquals(false, recordCollection.collectionExistsAsync().block());
124+
recordCollection.createCollectionAsync().block();
125+
assertEquals(true, recordCollection.collectionExistsAsync().block());
126+
}
127+
128+
@Test
129+
public void deleteCollectionAsync() {
130+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync");
131+
132+
assertEquals(false, recordCollection.collectionExistsAsync().block());
133+
recordCollection.createCollectionAsync().block();
134+
recordCollection.deleteCollectionAsync().block();
135+
assertEquals(false, recordCollection.collectionExistsAsync().block());
136+
}
137+
138+
@ParameterizedTest
139+
@EnumSource(RecordCollectionOptions.class)
140+
public void upsertAndGetRecordAsync(RecordCollectionOptions options) {
141+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
142+
143+
List<Hotel> hotels = getHotels();
144+
for (Hotel hotel : hotels) {
145+
recordCollection.upsertAsync(hotel, null).block();
146+
}
147+
148+
for (Hotel hotel : hotels) {
149+
Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block();
150+
assertNotNull(retrievedHotel);
151+
assertEquals(hotel.getId(), retrievedHotel.getId());
152+
}
153+
}
154+
155+
@ParameterizedTest
156+
@EnumSource(RecordCollectionOptions.class)
157+
public void getBatchAsync(RecordCollectionOptions options) {
158+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
159+
160+
List<Hotel> hotels = getHotels();
161+
for (Hotel hotel : hotels) {
162+
recordCollection.upsertAsync(hotel, null).block();
163+
}
164+
165+
List<String> ids = new ArrayList<>();
166+
hotels.forEach(hotel -> ids.add(hotel.getId()));
167+
168+
List<Hotel> retrievedHotels = recordCollection.getBatchAsync(ids, null).block();
169+
170+
assertNotNull(retrievedHotels);
171+
assertEquals(hotels.size(), retrievedHotels.size());
172+
for (int i = 0; i < hotels.size(); i++) {
173+
assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId());
174+
}
175+
}
176+
177+
@ParameterizedTest
178+
@EnumSource(RecordCollectionOptions.class)
179+
public void upsertBatchAsync(RecordCollectionOptions options) {
180+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
181+
182+
List<Hotel> hotels = getHotels();
183+
List<String> keys = recordCollection.upsertBatchAsync(hotels, null).block();
184+
assertNotNull(keys);
185+
186+
List<Hotel> retrievedHotels = (List<Hotel>) recordCollection.getBatchAsync(keys, null).block();
187+
188+
assertNotNull(retrievedHotels);
189+
assertEquals(hotels.size(), retrievedHotels.size());
190+
for (int i = 0; i < hotels.size(); i++) {
191+
assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId());
192+
}
193+
}
194+
195+
@ParameterizedTest
196+
@EnumSource(RecordCollectionOptions.class)
197+
public void deleteAsync(RecordCollectionOptions options) {
198+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
199+
200+
List<Hotel> hotels = getHotels();
201+
recordCollection.upsertBatchAsync(hotels, null).block();
202+
203+
for (Hotel hotel : hotels) {
204+
recordCollection.deleteAsync(hotel.getId(), null).block();
205+
Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block();
206+
assertNull(retrievedHotel);
207+
}
208+
}
209+
210+
@ParameterizedTest
211+
@EnumSource(RecordCollectionOptions.class)
212+
public void deleteBatchAsync(RecordCollectionOptions options) {
213+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
214+
215+
List<Hotel> hotels = getHotels();
216+
recordCollection.upsertBatchAsync(hotels, null).block();
217+
218+
List<String> ids = new ArrayList<>();
219+
hotels.forEach(hotel -> ids.add(hotel.getId()));
220+
221+
recordCollection.deleteBatchAsync(ids, null).block();
222+
223+
for (String id : ids) {
224+
Hotel retrievedHotel = recordCollection.getAsync(id, null).block();
225+
assertNull(retrievedHotel);
226+
}
227+
}
228+
229+
@ParameterizedTest
230+
@EnumSource(RecordCollectionOptions.class)
231+
public void getAsyncWithVectors(RecordCollectionOptions options) {
232+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
233+
234+
List<Hotel> hotels = getHotels();
235+
recordCollection.upsertBatchAsync(hotels, null).block();
236+
237+
for (Hotel hotel : hotels) {
238+
Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block();
239+
assertNotNull(retrievedHotel);
240+
assertNotNull(retrievedHotel.getDescriptionEmbedding());
241+
assertEquals(hotel.getId(), retrievedHotel.getId());
242+
assertEquals(hotel.getDescription(), retrievedHotel.getDescription());
243+
}
244+
}
245+
246+
@ParameterizedTest
247+
@EnumSource(RecordCollectionOptions.class)
248+
public void getBatchAsyncWithVectors(RecordCollectionOptions options) {
249+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
250+
251+
List<Hotel> hotels = getHotels();
252+
recordCollection.upsertBatchAsync(hotels, null).block();
253+
254+
List<String> ids = new ArrayList<>();
255+
hotels.forEach(hotel -> ids.add(hotel.getId()));
256+
257+
List<Hotel> retrievedHotels = recordCollection.getBatchAsync(ids, null).block();
258+
259+
assertNotNull(retrievedHotels);
260+
assertEquals(hotels.size(), retrievedHotels.size());
261+
for (int i = 0; i < hotels.size(); i++) {
262+
assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId());
263+
assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription());
264+
assertNotNull(retrievedHotels.get(i).getDescriptionEmbedding());
265+
}
266+
}
267+
268+
@ParameterizedTest
269+
@EnumSource(RecordCollectionOptions.class)
270+
public void getAsyncWithNoVectors(RecordCollectionOptions options) {
271+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
272+
273+
List<Hotel> hotels = getHotels();
274+
recordCollection.upsertBatchAsync(hotels, null).block();
275+
276+
GetRecordOptions getRecordOptions = GetRecordOptions.builder().includeVectors(false).build();
277+
for (Hotel hotel : hotels) {
278+
Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), getRecordOptions).block();
279+
assertNotNull(retrievedHotel);
280+
assertNull(retrievedHotel.getDescriptionEmbedding());
281+
assertEquals(hotel.getId(), retrievedHotel.getId());
282+
assertEquals(hotel.getDescription(), retrievedHotel.getDescription());
283+
}
284+
}
285+
286+
@ParameterizedTest
287+
@EnumSource(RecordCollectionOptions.class)
288+
public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) {
289+
RedisHashSetVectorStoreRecordCollection<Hotel> recordCollection = buildrecordCollection(optionsMap.get(options), options.name());
290+
291+
List<Hotel> hotels = getHotels();
292+
recordCollection.upsertBatchAsync(hotels, null).block();
293+
294+
GetRecordOptions getRecordOptions = GetRecordOptions.builder().includeVectors(false).build();
295+
List<String> ids = new ArrayList<>();
296+
hotels.forEach(hotel -> ids.add(hotel.getId()));
297+
298+
List<Hotel> retrievedHotels = recordCollection.getBatchAsync(ids, getRecordOptions).block();
299+
300+
assertNotNull(retrievedHotels);
301+
assertEquals(hotels.size(), retrievedHotels.size());
302+
for (int i = 0; i < hotels.size(); i++) {
303+
assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId());
304+
assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription());
305+
assertNull(retrievedHotels.get(i).getDescriptionEmbedding());
306+
}
307+
}
308+
}

0 commit comments

Comments
 (0)