Skip to content

Commit 89bc5ea

Browse files
fmeheustfmeheust
committed
Merge remote-tracking branch 'origin/add-oracle-store-tests' into add-oracle-store
2 parents 0bf9199 + 434b9e6 commit 89bc5ea

10 files changed

Lines changed: 970 additions & 108 deletions

File tree

‎data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ protected String getKeyFromRecord(Record data) {
195195
Field keyField = data.getClass()
196196
.getDeclaredField(recordDefinition.getKeyField().getName());
197197
keyField.setAccessible(true);
198-
return (String) keyField.get(data);
198+
return keyField.get(data).toString();
199199
} catch (NoSuchFieldException | IllegalAccessException e) {
200200
throw new SKException("Failed to get key from record", e);
201201
}

‎data/semantickernel-data-oracle/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@
5151
<artifactId>jackson-core</artifactId>
5252
<scope>compile</scope>
5353
</dependency>
54+
<dependency>
55+
<groupId>com.fasterxml.jackson.datatype</groupId>
56+
<artifactId>jackson-datatype-jsr310</artifactId>
57+
<version>2.18.0</version>
58+
</dependency>
5459
<dependency>
5560
<groupId>com.github.jknack</groupId>
5661
<artifactId>handlebars</artifactId>

‎data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping;
44
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField;
5+
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField;
56
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField;
67
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField;
78
import oracle.jdbc.OracleTypes;
@@ -255,9 +256,16 @@ public static String getTypeForVectorField(VectorStoreRecordVectorField field) {
255256
* @param field the vector field definition.
256257
* @return the JDBC oracle type.
257258
*/
258-
public static int getOracleTypeForField(VectorStoreRecordVectorField field) {
259+
public static int getOracleTypeForVectorField(VectorStoreRecordVectorField field) {
259260
if (field.getFieldSubType() == null) {
260-
return mapOracleTypeToVector.get(field.getFieldType()).intValue();
261+
Integer oracleType = mapOracleTypeToVector.get(field.getFieldType());
262+
if (oracleType != null) {
263+
return oracleType.intValue();
264+
} else {
265+
// field was declared as list with no subtype, assume FLOAT
266+
return OracleTypes.VECTOR_FLOAT32;
267+
}
268+
261269
} else {
262270
switch (field.getFieldSubType().getName()) {
263271
case "java.lang.Double":
@@ -272,6 +280,10 @@ public static int getOracleTypeForField(VectorStoreRecordVectorField field) {
272280
}
273281
}
274282

283+
public static boolean isUUID (VectorStoreRecordField field) {
284+
return (field.getFieldType().getName() == "java.util.UUID");
285+
}
286+
275287
/**
276288
* Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name.
277289
* @param effectiveStorageName the field name.

‎data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java‎

Lines changed: 60 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@
33
import com.fasterxml.jackson.core.JsonProcessingException;
44
import com.fasterxml.jackson.databind.JsonNode;
55
import com.fasterxml.jackson.databind.ObjectMapper;
6+
import com.fasterxml.jackson.databind.SerializationFeature;
67
import com.fasterxml.jackson.databind.node.ArrayNode;
8+
import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
9+
import com.fasterxml.jackson.databind.util.StdDateFormat;
10+
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
711
import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause;
812
import com.microsoft.semantickernel.data.filter.EqualToFilterClause;
913
import com.microsoft.semantickernel.data.jdbc.*;
@@ -21,20 +25,32 @@
2125
import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions;
2226
import com.microsoft.semantickernel.exceptions.SKException;
2327
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
28+
import oracle.jdbc.OraclePreparedStatement;
2429
import oracle.jdbc.OracleStatement;
2530
import oracle.jdbc.OracleTypes;
31+
import oracle.sql.TIMESTAMPLTZ;
32+
import oracle.sql.TIMESTAMPTZ;
2633

2734
import javax.annotation.Nonnull;
2835
import javax.annotation.concurrent.GuardedBy;
2936
import javax.sql.DataSource;
37+
import java.nio.ByteBuffer;
3038
import java.sql.Connection;
3139
import java.sql.PreparedStatement;
3240
import java.sql.ResultSet;
41+
import java.sql.SQLType;
3342
import java.sql.Statement;
3443
import java.sql.SQLException;
44+
import java.sql.Timestamp;
45+
import java.text.SimpleDateFormat;
46+
import java.time.LocalDateTime;
47+
import java.time.OffsetDateTime;
48+
import java.time.format.DateTimeFormatter;
49+
import java.time.format.DateTimeFormatterBuilder;
3550
import java.util.ArrayList;
3651
import java.util.Collections;
3752
import java.util.List;
53+
import java.util.UUID;
3854
import java.util.logging.Logger;
3955
import java.util.stream.Collectors;
4056
import java.util.stream.StreamSupport;
@@ -90,6 +106,7 @@ private OracleVectorStoreQueryProvider(
90106
OracleVectorStoreFieldHelper.getSupportedVectorTypes());
91107
this.collectionsTable = collectionsTable;
92108
this.objectMapper = objectMapper;
109+
this.objectMapper.registerModule(new JavaTimeModule());
93110
}
94111

95112
@Override
@@ -120,15 +137,15 @@ public void createCollection(String collectionName,
120137
try (Statement statement = connection.createStatement()) {
121138
// Create table
122139
System.out.println(createStorageTable);
123-
statement.execute(createStorageTable);
140+
statement.addBatch(createStorageTable);
124141

125142
// Index filterable columns
126143
for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) {
127144
if (dataField.isFilterable()) {
128145
String dataFieldIndex = OracleVectorStoreFieldHelper.createIndexForDataField(
129146
getCollectionTableName(collectionName), dataField, supportedDataTypes);
130147
System.out.println(dataFieldIndex);
131-
statement.execute(dataFieldIndex);
148+
statement.addBatch(dataFieldIndex);
132149
}
133150
}
134151

@@ -138,10 +155,10 @@ public void createCollection(String collectionName,
138155
vectorField, getCollectionTableName(collectionName));
139156
if (createVectorIndex != null) {
140157
System.out.println(createVectorIndex);
141-
statement.execute(createVectorIndex);
158+
statement.addBatch(createVectorIndex);
142159
}
143160
}
144-
//statement.executeBatch();
161+
statement.executeBatch();
145162

146163
try (PreparedStatement insert = connection.prepareStatement(
147164
insertCollectionQuery)) {
@@ -220,6 +237,10 @@ public void upsertRecords(String collectionName, List<?> records, VectorStoreRec
220237

221238
private void setUpsertStatementValues(PreparedStatement statement, Object record,
222239
List<VectorStoreRecordField> fields) {
240+
objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
241+
String datePattern = "yyyy-MM-dd HH:mm:ss.SSSZ";
242+
SimpleDateFormat df = new SimpleDateFormat(datePattern);
243+
objectMapper.setDateFormat(df);
223244
JsonNode jsonNode = objectMapper.valueToTree(record);
224245

225246
for (int i = 0; i < fields.size(); ++i) {
@@ -230,27 +251,55 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record
230251
if (field instanceof VectorStoreRecordVectorField) {
231252
// Convert the vector field to a string
232253
if (!field.getFieldType().equals(String.class)) {
233-
double[] values = valueNode.isNull()
254+
double[] values = (valueNode == null || valueNode.isNull())
234255
? null
235256
: StreamSupport.stream((
236257
(ArrayNode)valueNode).spliterator(), false)
237258
.mapToDouble(d -> d.asDouble()).toArray();
238259
statement.setObject(i + 1, values,
239-
OracleVectorStoreFieldHelper.getOracleTypeForField((VectorStoreRecordVectorField)field));
260+
OracleVectorStoreFieldHelper.getOracleTypeForVectorField((VectorStoreRecordVectorField)field));
240261
System.out.println("Set values: " + values);
241262
continue;
242263
}
243264
} else if (field instanceof VectorStoreRecordDataField) {
244265
// Convert List field to a string
245266
if (field.getFieldType().equals(List.class)) {
246267
statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode));
247-
System.out.println("Set values: " + objectMapper.writeValueAsString(valueNode));
268+
System.out.println(
269+
"Set values: " + objectMapper.writeValueAsString(valueNode));
270+
continue;
271+
}
272+
if (OracleVectorStoreFieldHelper.isUUID(field)) {
273+
if (valueNode == null || valueNode.isNull()) {
274+
statement.setNull(i + 1, OracleTypes.RAW);
275+
} else {
276+
UUID uuid = UUID.fromString(valueNode.textValue());
277+
ByteBuffer bb = ByteBuffer.allocate(16);
278+
bb.putLong(uuid.getMostSignificantBits());
279+
bb.putLong(uuid.getLeastSignificantBits());
280+
statement.setBytes(i + 1, bb.array());
281+
System.out.println("Set values: " + objectMapper.convertValue(valueNode,
282+
field.getFieldType()));
283+
}
284+
continue;
285+
}
286+
if (field.getFieldType().equals(OffsetDateTime.class)) {
287+
if (valueNode == null || valueNode.isNull()) {
288+
statement.setNull(i + 1, OracleTypes.TIMESTAMPTZ);
289+
} else {
290+
OffsetDateTime offsetDateTime = OffsetDateTime.parse(
291+
valueNode.asText());
292+
((OraclePreparedStatement) statement).setTIMESTAMPTZ(i + 1,
293+
TIMESTAMPTZ.of(offsetDateTime));
294+
System.out.println("Set values: " + objectMapper.convertValue(valueNode,
295+
field.getFieldType()));
296+
}
248297
continue;
249298
}
250299
}
251300

252301
statement.setObject(i + 1,
253-
objectMapper.convertValue(valueNode, field.getFieldType()));
302+
objectMapper.convertValue(valueNode,field.getFieldType()));
254303
System.out.println("Set values: " + objectMapper.convertValue(valueNode, field.getFieldType()));
255304
} catch (SQLException | JsonProcessingException e) {
256305
throw new RuntimeException(e);
@@ -317,7 +366,7 @@ public <Record> VectorSearchResults<Record> search(String collectionName, List<F
317366
defineDataColumnType(columnIndex++, oracleStatement, field.getFieldType());
318367
else
319368
oracleStatement.defineColumnType(columnIndex++,
320-
OracleVectorStoreFieldHelper.getOracleTypeForField((VectorStoreRecordVectorField) field),
369+
OracleVectorStoreFieldHelper.getOracleTypeForVectorField((VectorStoreRecordVectorField) field),
321370
Integer.MAX_VALUE);
322371
}
323372
oracleStatement.setLobPrefetchSize(Integer.MAX_VALUE); // Workaround for Oracle JDBC bug 37030121
@@ -380,6 +429,8 @@ private void defineDataColumnType(int columnIndex, OracleStatement statement, Cl
380429
statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE);
381430
break;
382431
case OracleDataTypesMapping.UUID:
432+
statement.defineColumnType(columnIndex, OracleTypes.RAW);
433+
break;
383434
case OracleDataTypesMapping.BYTE_ARRAY:
384435
statement.defineColumnType(columnIndex, OracleTypes.RAW);
385436
default:

‎data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
// Copyright (c) Microsoft. All rights reserved.
22
package com.microsoft.semantickernel.data.jdbc.oracle;
33

4-
import com.fasterxml.jackson.core.JsonProcessingException;
54
import com.fasterxml.jackson.databind.JsonNode;
65
import com.fasterxml.jackson.databind.ObjectMapper;
76
import com.fasterxml.jackson.databind.node.ObjectNode;
87
import com.microsoft.semantickernel.builders.SemanticKernelBuilder;
98
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper;
10-
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField;
119
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition;
1210
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField;
1311
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField;
@@ -16,13 +14,12 @@
1614
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
1715
import oracle.jdbc.OracleResultSet;
1816
import oracle.jdbc.provider.oson.OsonModule;
19-
import oracle.sql.json.OracleJsonArray;
20-
import oracle.sql.json.OracleJsonObject;
17+
import oracle.sql.TIMESTAMPTZ;
18+
import java.nio.ByteBuffer;
2119
import java.sql.ResultSet;
2220
import java.sql.SQLException;
23-
import java.time.OffsetDateTime;
24-
import java.util.List;
2521
import java.util.Map;
22+
import java.util.UUID;
2623
import java.util.function.BiFunction;
2724

2825
/**
@@ -175,21 +172,37 @@ public OracleVectorStoreRecordMapper<Record> build() {
175172
value = resultSet.getBoolean(field.getEffectiveStorageName());
176173
break;
177174
case OracleDataTypesMapping.OFFSET_DATE_TIME:
178-
value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName())
179-
.offsetDateTimeValue();
175+
TIMESTAMPTZ timestamptz = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName());
176+
value = timestamptz != null ? timestamptz.offsetDateTimeValue() : null;
180177
break;
181178
case OracleDataTypesMapping.BYTE_ARRAY:
182179
value = resultSet.getBytes(field.getEffectiveStorageName());
183180
break;
184-
// fallthrough
185181
case OracleDataTypesMapping.UUID:
182+
byte[] bytes = resultSet.getBytes(field.getEffectiveStorageName());
183+
if (bytes != null) {
184+
ByteBuffer bb = ByteBuffer.wrap(bytes);
185+
long firstLong = bb.getLong();
186+
long secondLong = bb.getLong();
187+
value = new UUID(firstLong, secondLong);
188+
} else {
189+
value = null;
190+
}
191+
break;
186192
case OracleDataTypesMapping.JSON:
187193
value = resultSet.getObject(field.getEffectiveStorageName(), fieldType);
188194
break;
189195
default:
190196
value = resultSet.getString(field.getEffectiveStorageName());
191197
}
198+
// Result set getter method sometimes returns a default value when NULL,
199+
// set value to null in that case.
200+
if (resultSet.wasNull()) {
201+
value = null;
202+
}
203+
192204
JsonNode genericNode = objectMapper.valueToTree(value);
205+
193206
objectNode.set(field.getEffectiveStorageName(), genericNode);
194207
}
195208
if (options != null && options.isIncludeVectors()) {

0 commit comments

Comments
 (0)