Merge branch 'main' into my-feature-branch

Author: eric-epsilla

.github/workflows/pypi-publish.yml

@@ -11,7 +11,7 @@ name: Upload Python Package
 on:
   push:
     branches: [ "main" ]
-    paths: ['pyepsilla/vectordb/version.py']
+    paths: ['pyepsilla/vectordb/version.py', 'pyproject.toml']
 
 jobs:
   deploy:

README.md

@@ -1,41 +1,54 @@
-# Epsilla Python SDK
+<p align="center">
+    <img width="275" alt="Epsilla Logo" src="https://epsilla-misc.s3.amazonaws.com/epsilla-horizontal.png">
+</p>
 
-Welcome to Python SDK for Epsilla Vector Database! 
-https://pypi.org/project/pyepsilla/#history
+<p align="center">
+    <b>Python Client for <a href="https://github.com/epsilla-cloud/vectordb">Epsilla</a> Vector Database</b>
+</p>
 
-## Installation
-```shell
-pip3 install pyepsilla
-```
-or
+<hr />
+
+Welcome to Python SDK for Epsilla Vector Database!
+- <a href="https://epsilla-inc.gitbook.io/epsilladb/vector-database/connect-to-a-database">QuickStart</a>
+- <a href="https://pypi.org/project/pyepsilla/#history">Release History</a>
+
+## Install pyepsilla
 ```shell
 pip3 install --upgrade pyepsilla
 ```
 
-## Documentation
+## Connect to Epsilla Vector Database
 
-### 1.1 Run epsilla vectordb on localhost
+#### Run epsilla vectordb on localhost
 ```shell
 docker pull epsilla/vectordb
 docker run -d -p 8888:8888 epsilla/vectordb
 ```
 
-### 1.2 Use pyepsilla to connect to and interact with local vector database
+#### When Port 8888 conflicted with Jupyter Notebook
+If you are using Jupyter Notebook on localhost, the port 8888 maybe conflict!
+
+So you can change the vectordb port to another number, such as 18888
+```
+docker run -d -p 18888:8888 epsilla/vectordb
+```
+
+#### Use pyepsilla to connect to and interact with local vector database
 
 ```python
 from pyepsilla import vectordb
 
-## connect to vectordb
+## 1.Connect to vectordb
 client = vectordb.Client(
   host='localhost',
   port='8888'
 )
 
-## load and use a database
+## 2.Load and use a database
 client.load_db(db_name="MyDB", db_path="/tmp/epsilla")
 client.use_db(db_name="MyDB")
 
-## create a table in the current database
+## 3.Create a table in the current database
 client.create_table(
   table_name="MyTable",
   table_fields=[
@@ -45,7 +58,7 @@ client.create_table(
   ]
 )
 
-## insert records
+## 4.Insert records
 client.insert(
   table_name="MyTable",
   records=[
@@ -57,7 +70,7 @@ client.insert(
   ]
 )
 
-## search with specific response field
+## 5.Search with specific response field
 status_code, response = client.query(
   table_name="MyTable",
   query_field="Embedding",
@@ -67,7 +80,7 @@ status_code, response = client.query(
 )
 print(response)
 
-## search without specific response field, then it will return all fields
+## 6.Search without specific response field, then it will return all fields
 status_code, response = client.query(
   table_name="MyTable",
   query_field="Embedding",
@@ -76,46 +89,117 @@ status_code, response = client.query(
 )
 print(response)
 
-
-
-## delete records by primary_keys (and filter)
-# status_code, response =  client.delete(table_name="MyTable", ids=[3])
+## 7.Delete records by primary_keys (and filter)
 status_code, response =  client.delete(table_name="MyTable", primary_keys=[3, 4])
-# status_code, response =  client.delete(table_name="MyTable", filter="Doc <> 'San Francisco'")
+status_code, response =  client.delete(table_name="MyTable", filter="Doc <> 'San Francisco'")
 print(response)
 
 
-## drop a table
-#client.drop_table("MyTable")
+## 8.Drop a table
+client.drop_table("MyTable")
 
-## unload a database from memory
-#client.unload_db("MyDB")
+## 9.Unload a database from memory
+client.unload_db("MyDB")
 ```
 
 
+## Connect to Epsilla Cloud
 
-### 2 Run epsilla vectordb on epsilla cloud
+#### Register and create vectordb on Epsilla Cloud
+https://cloud.epsilla.com
 
+#### Use Epsilla Cloud module to connect with the vectordb
+Please get the project_id, db_id, epsilla_api_key from Epsilla Cloud at first
 ```python3
-
 from pyepsilla import cloud
 
-# Connect to Epsilla Cloud
-client = cloud.Client(project_id="32ef3a3f-****-****-****-************", api_key="epsilla*****")
+epsilla_api_key = os.getenv("EPSILLA_API_KEY", "Your-Epsilla-API-Key")
+project_id = os.getenv("EPSILLA_PROJECT_ID", "Your-Project-ID")
+db_id = os.getenv("EPSILLA_DB_ID", "Your-DB-ID")
+
+
+# 1.Connect to Epsilla Cloud
+client = cloud.Client(project_id="*****-****-****-****-************", api_key="eps_**********")
+
+# 2.Connect to Vectordb
+db_client = cloud_client.vectordb(db_id)
+
+# 3.Create a table with schema
+status_code, response = db.create_table(
+    table_name="MyTable",
+    table_fields=[
+        {"name": "ID", "dataType": "INT", "primaryKey": True},
+        {"name": "Doc", "dataType": "STRING"},
+        {"name": "Embedding", "dataType": "VECTOR_FLOAT", "dimensions": 4},
+    ],
+)
+print(status_code, response)
+
+# 4.Insert new vector records into table
+status_code, response = db.insert(
+    table_name="MyTable",
+    records=[
+        {"ID": 1, "Doc": "Berlin", "Embedding": [0.05, 0.61, 0.76, 0.74]},
+        {"ID": 2, "Doc": "London", "Embedding": [0.19, 0.81, 0.75, 0.11]},
+        {"ID": 3, "Doc": "Moscow", "Embedding": [0.36, 0.55, 0.47, 0.94]},
+        {"ID": 4, "Doc": "San Francisco", "Embedding": [0.18, 0.01, 0.85, 0.80]},
+        {"ID": 5, "Doc": "Shanghai", "Embedding": [0.24, 0.18, 0.22, 0.44]},
+    ],
+)
+print(status_code, response)
+
+
+# 5.Query Vectors with specific response field, otherwise it will return all fields
+status_code, response = db.query(
+    table_name="MyTable",
+    query_field="Embedding",
+    query_vector=[0.35, 0.55, 0.47, 0.94],
+    response_fields=["Doc"],
+    limit=2,
+)
+print(status_code, response)
+
+
+# 6.Delete specific records from table
+status_code, response = db.delete(table_name="MyTable", primary_keys=[4, 5])
+status_code, response = db.delete(table_name="MyTable", filter="Doc <> 'San Francisco'")
+print(status_code, response)
+
+# 7.Drop table
+status_code, response = db.drop_table(table_name="MyTable")
+print(status_code, response)
 
-# Connect to Vectordb
-db = client.vectordb(db_id="df7431d0-****-****-****-************")
 
 ```
-Please check https://github.com/epsilla-cloud/epsilla-python-client/blob/main/examples/hello_epsilla_cloud.py for detail.
 
 
+## Connect to Epsilla RAG
+Please get the project_id, epsilla_api_key, ragapp_id, converstation_id(optional) from Epsilla Cloud at first
+The resp will contains answer as well as contexts, like {"answer": "****", "contexts": ['context1','context2', ...]}
+
+```python3
+from pyepsilla import cloud
+
+# 1.Connect to Epsilla RAG
+client = cloud.RAG(
+    project_id="ce07c6fc-****-****-b7bd-b7819f22bcff",
+    api_key="eps_**********",
+    ragapp_id="153a5a49-****-****-b2b8-496451eda8b5",
+    conversation_id="6fa22a6a-****-****-b1c3-5c795d0f45ef",
+)
+
+# 2.Start a new conversation with RAG
+client.start_new_conversation()
+resp = client.query("What's RAG?")
+
+print("[INFO] response is", resp)
+```
 
 
 ## Contributing
-Bug reports and pull requests are welcome on GitHub at https://github.com/epsilla-cloud/epsilla-python-client/
+Bug reports and pull requests are welcome on GitHub at [here](https://github.com/epsilla-cloud/epsilla-python-client)
 
-If you have any question or problem, please join our discord https://discord.com/invite/cDaY2CxZc5
+If you have any question or problem, please join our [discord](https://discord.com/invite/cDaY2CxZc5)
 
-## We love your <a href="https://forms.gle/z73ra1sGBxH9wiUR8">Feedback</a>!
+We love your <a href="https://forms.gle/z73ra1sGBxH9wiUR8">Feedback</a>!
 

examples/Question_Answering_Pipeline_with_LangChain_and_Epsilla.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+
+# Question Answering Pipeline with LangChain and Epsilla
+# Step1. Install the required packages
+"""
+pip install langchain
+pip install openai
+pip install tiktoken
+pip install pyepsilla
+pip install -U langchain-community
+pip install -U langchain-openai
+"""
+
+
+# Step2. Configure the OpenAI API Key
+import os
+os.environ["OPENAI_API_KEY"] = "Your-OpenAI-API-Key"
+
+
+# Step3. Load the documents
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+
+loader = WebBaseLoader("https://raw.githubusercontent.com/hwchase17/chat-your-data/master/state_of_the_union.txt")
+documents = loader.load()
+documents = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0).split_documents(documents)
+embeddings = OpenAIEmbeddings()
+
+
+# Step4. Load the vector store
+from langchain_community.vectorstores import Epsilla
+from pyepsilla import vectordb
+
+db_client = vectordb.Client(protocol="https", host="demo.epsilla.com", port="443")
+
+status_code, response = db_client.load_db("MyDB", "/data/MyDB")
+print(status_code, response)
+
+vector_store = Epsilla.from_documents(
+    documents,
+    embeddings,
+    db_client,
+    db_path="/data/MyDB",
+    db_name="MyDB",
+    collection_name="MyCollection",
+)
+
+
+
+
+# Step4. Create the QA for Retrieval
+from langchain.chains import RetrievalQA
+from langchain_openai import OpenAI
+
+qa = RetrievalQA.from_chain_type(
+    llm=OpenAI(), chain_type="stuff", retriever=vector_store.as_retriever()
+)
+query = "What did the president say about Ketanji Brown Jackson"
+resp = qa.invoke(query)
+print("resp:", resp)

examples/Question_Answering_Pipeline_with_LangChain_and_EpsillaCloud.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+
+# Question Answering Pipeline with LangChain and Epsilla
+# Step1. Install the required packages
+"""
+pip install langchain
+pip install openai
+pip install tiktoken
+pip install pyepsilla
+pip install -U langchain-openai
+pip install -U langchain-community
+"""
+
+
+# Step2. Configure the OpenAI API Key
+import os
+
+os.environ["OPENAI_API_KEY"] = "Your-OpenAI-API-Key"
+epsilla_api_key = os.getenv("EPSILLA_API_KEY", "Your-Epsilla-API-Key")
+project_id = os.getenv("EPSILLA_PROJECT_ID", "Your-Project-ID")
+db_id = os.getenv("EPSILLA_DB_ID", "Your-DB-ID")
+db_sharding_id = os.getenv("EPSILLA_DB_SHARDING_ID", 0)
+
+
+# Step3. Load the documents
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+
+loader = WebBaseLoader("https://raw.githubusercontent.com/hwchase17/chat-your-data/master/state_of_the_union.txt")
+documents = loader.load()
+documents = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0).split_documents(documents)
+embeddings = OpenAIEmbeddings()
+
+
+# Step4. Load the vector store
+from langchain_community.vectorstores import Epsilla
+from pyepsilla import cloud
+
+db_name = f"db_{db_id.replace('-', '_')}"
+db_path = f"/data/{project_id}/{db_name}/s{db_sharding_id}"
+table_name = "MyCollection"
+
+# Connect to Epsilla Cloud
+cloud_client = cloud.Client(
+    project_id=project_id,
+    api_key=epsilla_api_key,
+)
+
+# Connect to Vectordb
+db_client = cloud_client.vectordb(db_id)
+
+vector_store = Epsilla.from_documents(
+    documents,
+    embeddings,
+    db_client,
+    db_path=db_path,
+    db_name=db_name,
+    collection_name=table_name,
+)
+
+# query = "What did the president say about Ketanji Brown Jackson"
+# docs = vector_store.similarity_search(query)
+# print(docs[0].page_content)
+
+
+
+# Step4. Create the QA for Retrieval
+from langchain.chains import RetrievalQA
+from langchain_openai import OpenAI
+
+qa = RetrievalQA.from_chain_type(
+    llm=OpenAI(), chain_type="stuff", retriever=vector_store.as_retriever()
+)
+query = "What did the president say about Ketanji Brown Jackson"
+resp = qa.invoke(query)
+print("resp:", resp)

examples/hello_epsilla.py

@@ -16,7 +16,7 @@
 # client = vectordb.Client(protocol='https', host='demo.epsilla.com', port='443')
 
 # Load DB with path
-## pay attention to change db_path to persistent volume for production environment
+# pay attention to change db_path to persistent volume for production environment
 status_code, response = client.load_db(db_name="MyDB", db_path="/data/epsilla_demo")
 print(response)