Skip to content

Commit ba1aa96

Browse files
psilberkpsilberk
committed
Upsert and Main sample
1 parent bc2c366 commit ba1aa96

6 files changed

Lines changed: 171 additions & 179 deletions

File tree

‎data/semantickernel-data-jdbc/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,5 +63,10 @@
6363
<artifactId>sqlite-jdbc</artifactId>
6464
<version>3.47.0.0</version>
6565
</dependency>
66+
<dependency>
67+
<groupId>com.oracle.database.jdbc</groupId>
68+
<artifactId>ojdbc11</artifactId>
69+
<version>23.7.0.25.01</version>
70+
</dependency>
6671
</dependencies>
6772
</project>

‎data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java‎

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.microsoft.semantickernel.data.jdbc.oracle;
22

3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
35
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider;
6+
import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider;
47
import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition;
58
import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions;
69
import com.microsoft.semantickernel.exceptions.SKException;
@@ -18,9 +21,14 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider
1821
// This could be removed if super.collectionTable made protected
1922
private final String collectionsTable;
2023

21-
private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables) {
24+
// This could be common to all query providers
25+
private final ObjectMapper objectMapper;
26+
27+
private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables,
28+
ObjectMapper objectMapper) {
2229
super(dataSource, collectionsTable, prefixForCollectionTables);
2330
this.collectionsTable = collectionsTable;
31+
this.objectMapper = objectMapper;
2432
}
2533

2634
@Override
@@ -37,13 +45,50 @@ public void prepareVectorStore() {
3745
}
3846
}
3947

48+
@Override
49+
public void createCollection(String collectionName,
50+
VectorStoreRecordDefinition recordDefinition) {
51+
// TODO Override implementation. Eg: mapping TEXT to VARCHAR
52+
super.createCollection(collectionName, recordDefinition);
53+
}
54+
4055
@Override
4156
public void upsertRecords(String collectionName, List<?> records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) {
4257

43-
// Using hsqldb impl
58+
// TODO look for public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) {
59+
60+
// TODO Make this a MERGE query
61+
62+
// String upsertStatemente = formatQuery("""
63+
// MERGE INTO %s EXIST_REC USING (SELECT ? AS ID) NEW_REC ON (EXIST_REC.%s = NEW_REC.ID)
64+
// WHEN MATACHED THEN UPDATE SET EXISTING REC
65+
// """,
66+
// getCollectionTableName(collectionName),
67+
// recordDefinition.getKeyField().getName(),
68+
// getQueryColumnsFromFields(fields),
69+
// getWildcardString(fields.size()),
70+
// onDuplicateKeyUpdate);super.upsertRecords(collectionName, records, recordDefinition, options);
4471

72+
String query = formatQuery("INSERT INTO %s (%s, %s, %s) values (?, ?, ?)",
73+
getCollectionTableName(collectionName),
74+
recordDefinition.getAllFields().get(0).getStorageName(),
75+
recordDefinition.getAllFields().get(1).getStorageName(),
76+
recordDefinition.getAllFields().get(2).getStorageName());
4577

46-
super.upsertRecords(collectionName, records, recordDefinition, options);
78+
try (Connection connection = dataSource.getConnection();
79+
PreparedStatement statement = connection.prepareStatement(query)) {
80+
for (Object record : records) {
81+
JsonNode jsonNode = objectMapper.valueToTree(record);
82+
for (int i = 0; i < 3; i++) {
83+
statement.setObject(i + 1, jsonNode
84+
.get(recordDefinition.getAllFields().get(i).getStorageName()).asText());
85+
}
86+
statement.addBatch();
87+
}
88+
statement.executeBatch();
89+
} catch (SQLException e) {
90+
throw new SKException("Failed to upsert records", e);
91+
}
4792
}
4893

4994
public static Builder builder() {
@@ -56,6 +101,7 @@ public static class Builder
56101
private DataSource dataSource;
57102
private String collectionsTable = DEFAULT_COLLECTIONS_TABLE;
58103
private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES;
104+
private ObjectMapper objectMapper = new ObjectMapper();
59105

60106
@SuppressFBWarnings("EI_EXPOSE_REP2")
61107
public Builder withDataSource(DataSource dataSource) {
@@ -83,9 +129,16 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) {
83129
return this;
84130
}
85131

132+
public Builder withObjectMapper(
133+
ObjectMapper objectMapper) {
134+
this.objectMapper = objectMapper;
135+
return this;
136+
}
137+
86138
@Override
87139
public OracleVectorStoreQueryProvider build() {
88-
return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, prefixForCollectionTables);
140+
return new OracleVectorStoreQueryProvider(dataSource, collectionsTable,
141+
prefixForCollectionTables, objectMapper);
89142
}
90143
}
91144
}

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
<module>semantickernel-bom</module>
7272
<module>semantickernel-api</module>
7373
<module>semantickernel-experimental</module>
74+
<module>samples</module>
7475
<module>aiservices/openai</module>
7576
<module>aiservices/google</module>
7677
<module>aiservices/huggingface</module>

‎samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,17 @@
131131
<groupId>com.github.victools</groupId>
132132
<artifactId>jsonschema-module-jackson</artifactId>
133133
</dependency>
134+
<dependency>
135+
<groupId>com.microsoft.semantic-kernel</groupId>
136+
<artifactId>semantickernel-data-jdbc</artifactId>
137+
<version>1.4.4-SNAPSHOT</version>
138+
</dependency>
139+
<dependency>
140+
<groupId>com.microsoft.semantic-kernel</groupId>
141+
<artifactId>semantickernel-learn-resources</artifactId>
142+
<version>1.4.4-SNAPSHOT</version>
143+
<scope>compile</scope>
144+
</dependency>
134145
</dependencies>
135146

136147
<profiles>

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java‎

Lines changed: 25 additions & 175 deletions
Original file line numberDiff line numberDiff line change
@@ -1,210 +1,60 @@
11
// Copyright (c) Microsoft. All rights reserved.
22
package com.microsoft.semantickernel.samples.syntaxexamples.memory;
33

4-
import com.azure.ai.openai.OpenAIAsyncClient;
5-
import com.azure.ai.openai.OpenAIClientBuilder;
6-
import com.azure.core.credential.AzureKeyCredential;
7-
import com.azure.core.credential.KeyCredential;
8-
import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService;
94
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore;
105
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions;
6+
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection;
117
import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions;
12-
import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider;
13-
import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults;
8+
import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider;
149
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection;
15-
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData;
16-
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey;
17-
import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector;
18-
import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction;
19-
20-
import java.nio.charset.StandardCharsets;
10+
import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel;
2111
import java.sql.SQLException;
22-
import java.util.Arrays;
23-
import java.util.Base64;
2412
import java.util.Collections;
25-
import java.util.List;
26-
import java.util.Map;
27-
import java.util.stream.Collectors;
28-
29-
import org.postgresql.ds.PGSimpleDataSource;
30-
import reactor.core.publisher.Flux;
31-
import reactor.core.publisher.Mono;
13+
import oracle.jdbc.datasource.impl.OracleDataSource;
3214

3315
public class VectorStoreWithOracle {
3416

35-
static class GitHubFile {
36-
@VectorStoreRecordKey
37-
private final String id;
38-
@VectorStoreRecordData
39-
private final String description;
40-
@VectorStoreRecordData
41-
private final String link;
42-
@VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE)
43-
private final List<Float> embedding;
44-
45-
public GitHubFile() {
46-
this(null, null, null, Collections.emptyList());
47-
}
48-
49-
public GitHubFile(
50-
String id,
51-
String description,
52-
String link,
53-
List<Float> embedding) {
54-
this.id = id;
55-
this.description = description;
56-
this.link = link;
57-
this.embedding = embedding;
58-
}
59-
60-
public String getId() {
61-
return id;
62-
}
63-
64-
public String getDescription() {
65-
return description;
66-
}
67-
68-
public String getLink() {
69-
return link;
70-
}
71-
72-
public List<Float> getEmbedding() {
73-
return embedding;
74-
}
75-
76-
static String encodeId(String realId) {
77-
byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8));
78-
return new String(bytes, StandardCharsets.UTF_8);
79-
}
80-
}
81-
82-
// Run a PostgreSQL server with:
83-
// docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17
84-
8517
public static void main(String[] args) throws SQLException {
8618
System.out.println("==============================================================");
8719
System.out.println("============== Oracle Vector Store Example ===================");
8820
System.out.println("==============================================================");
8921

90-
OpenAIAsyncClient client;
91-
92-
if (AZURE_CLIENT_KEY != null) {
93-
client = new OpenAIClientBuilder()
94-
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
95-
.endpoint(CLIENT_ENDPOINT)
96-
.buildAsyncClient();
97-
98-
} else {
99-
client = new OpenAIClientBuilder()
100-
.credential(new KeyCredential(CLIENT_KEY))
101-
.buildAsyncClient();
102-
}
103-
104-
// Create an OpenAI text embedding generation service
105-
var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder()
106-
.withOpenAIAsyncClient(client)
107-
.withModelId(MODEL_ID)
108-
.withDimensions(EMBEDDING_DIMENSIONS)
109-
.build();
110-
111-
storeAndSearch(embeddingGeneration);
112-
}
113-
114-
public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) {
11522
// Configure the data source
116-
PGSimpleDataSource dataSource = new PGSimpleDataSource();
117-
dataSource.setUrl("jdbc:postgresql://localhost:5432/sk");
118-
dataSource.setUser("postgres");
119-
dataSource.setPassword("root");
23+
OracleDataSource dataSource = new OracleDataSource();
24+
dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1");
25+
dataSource.setUser("scott");
26+
dataSource.setPassword("tiger");
12027

12128
// Build a query provider
122-
// Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider
123-
var queryProvider = PostgreSQLVectorStoreQueryProvider.builder()
29+
OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder()
12430
.withDataSource(dataSource)
12531
.build();
12632

12733
// Build a vector store
128-
var jdbcVectorStore = JDBCVectorStore.builder()
34+
JDBCVectorStore vectorStore = JDBCVectorStore.builder()
12935
.withDataSource(dataSource)
13036
.withOptions(JDBCVectorStoreOptions.builder()
13137
.withQueryProvider(queryProvider)
13238
.build())
13339
.build();
13440

135-
// Set up the record collection to use
136-
String collectionName = "skgithubfiles";
137-
var collection = jdbcVectorStore.getCollection(collectionName,
138-
JDBCVectorStoreRecordCollectionOptions.<GitHubFile>builder()
139-
.withRecordClass(GitHubFile.class)
140-
.build());
141-
142-
// Create collection if it does not exist and store data
143-
collection
144-
.createCollectionIfNotExistsAsync()
145-
.then(storeData(collection, embeddingGeneration, sampleData()))
41+
// Get a collection from the vector store
42+
VectorStoreRecordCollection<String, Hotel> collection =
43+
vectorStore.getCollection("skhotels",
44+
JDBCVectorStoreRecordCollectionOptions.<Hotel>builder()
45+
.withRecordClass(Hotel.class)
46+
.build());
47+
48+
// Create the collection if it doesn't exist yet.
49+
collection.createCollectionAsync().block();
50+
51+
collection.upsertAsync(new Hotel("1",
52+
"HotelOne",
53+
"Desc for HotelOne",
54+
Collections.emptyList(), Collections.emptyList()),
55+
null)
14656
.block();
14757

148-
// Search for results
149-
var results = search("How to get started", collection, embeddingGeneration).block();
150-
151-
if (results == null || results.getTotalCount() == 0) {
152-
System.out.println("No search results found.");
153-
return;
154-
}
155-
var searchResult = results.getResults().get(0);
156-
System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n",
157-
searchResult.getScore(), searchResult.getRecord().link,
158-
searchResult.getRecord().description);
15958
}
16059

161-
private static Mono<VectorSearchResults<GitHubFile>> search(
162-
String searchText,
163-
VectorStoreRecordCollection<String, GitHubFile> recordCollection,
164-
OpenAITextEmbeddingGenerationService embeddingGeneration) {
165-
// Generate embeddings for the search text and search for the closest records
166-
return embeddingGeneration.generateEmbeddingAsync(searchText)
167-
.flatMap(r -> recordCollection.searchAsync(r.getVector(), null));
168-
}
169-
170-
private static Mono<List<String>> storeData(
171-
VectorStoreRecordCollection<String, GitHubFile> recordStore,
172-
OpenAITextEmbeddingGenerationService embeddingGeneration,
173-
Map<String, String> data) {
174-
175-
return Flux.fromIterable(data.entrySet())
176-
.flatMap(entry -> {
177-
System.out.println("Save '" + entry.getKey() + "' to memory.");
178-
179-
// Generate embeddings for the data and store it
180-
return embeddingGeneration
181-
.generateEmbeddingsAsync(Collections.singletonList(entry.getValue()))
182-
.flatMap(embeddings -> {
183-
GitHubFile gitHubFile = new GitHubFile(
184-
GitHubFile.encodeId(entry.getKey()),
185-
entry.getValue(),
186-
entry.getKey(),
187-
embeddings.get(0).getVector());
188-
return recordStore.upsertAsync(gitHubFile, null);
189-
});
190-
})
191-
.collectList();
192-
}
193-
194-
private static Map<String, String> sampleData() {
195-
return Arrays.stream(new String[][] {
196-
{ "https://github.com/microsoft/semantic-kernel/blob/main/README.md",
197-
"README: Installation, getting started with Semantic Kernel, and how to contribute" },
198-
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb",
199-
"Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" },
200-
{ "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT",
201-
"Sample demonstrating how to create a chat skill interfacing with ChatGPT" },
202-
{ "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs",
203-
"C# class that defines a volatile embedding store" },
204-
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md",
205-
"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" },
206-
{ "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md",
207-
"README: README associated with a sample chat summary react-based webapp" },
208-
}).collect(Collectors.toMap(element -> element[0], element -> element[1]));
209-
}
21060
}

0 commit comments

Comments
 (0)