Merge pull request #36 from epsilla-cloud/dev

fix the issue of limit of pyepsilla

Author: eric-epsilla

examples/hello_epsilla.py

@@ -10,7 +10,7 @@
 from pyepsilla import vectordb
 
 # Connect to Epsilla VectorDB
-client = vectordb.Client(protocol='http',  host='127.0.0.1',        port='8888')
+client = vectordb.Client(protocol="http", host="127.0.0.1", port="8888")
 
 # You can also use Epsilla Cloud
 # client = vectordb.Client(protocol='https', host='demo.epsilla.com', port='443')
@@ -25,12 +25,12 @@
 
 # Create a table with schema in current DB
 status_code, response = client.create_table(
-  table_name="MyTable",
-  table_fields=[
-    {"name": "ID", "dataType": "INT", "primaryKey": True},
-    {"name": "Doc", "dataType": "STRING"},
-    {"name": "Embedding", "dataType": "VECTOR_FLOAT", "dimensions": 4}
-  ]
+    table_name="MyTable",
+    table_fields=[
+        {"name": "ID", "dataType": "INT", "primaryKey": True},
+        {"name": "Doc", "dataType": "STRING"},
+        {"name": "Embedding", "dataType": "VECTOR_FLOAT", "dimensions": 4},
+    ],
 )
 print(response)
 
@@ -40,43 +40,45 @@
 
 # Insert new vector records into table
 status_code, response = client.insert(
-  table_name="MyTable",
-  records=[
-    {"ID": 1, "Doc": "Berlin", "Embedding": [0.05, 0.61, 0.76, 0.74]},
-    {"ID": 2, "Doc": "London", "Embedding": [0.19, 0.81, 0.75, 0.11]},
-    {"ID": 3, "Doc": "Moscow", "Embedding": [0.36, 0.55, 0.47, 0.94]},
-    {"ID": 4, "Doc": "San Francisco", "Embedding": [0.18, 0.01, 0.85, 0.80]},
-    {"ID": 5, "Doc": "Shanghai", "Embedding": [0.24, 0.18, 0.22, 0.44]}
-  ]
+    table_name="MyTable",
+    records=[
+        {"ID": 1, "Doc": "Berlin", "Embedding": [0.05, 0.61, 0.76, 0.74]},
+        {"ID": 2, "Doc": "London", "Embedding": [0.19, 0.81, 0.75, 0.11]},
+        {"ID": 3, "Doc": "Moscow", "Embedding": [0.36, 0.55, 0.47, 0.94]},
+        {"ID": 4, "Doc": "San Francisco", "Embedding": [0.18, 0.01, 0.85, 0.80]},
+        {"ID": 5, "Doc": "Shanghai", "Embedding": [0.24, 0.18, 0.22, 0.44]},
+    ],
 )
 print(response)
 
 # Query Vectors with specific response field
 status_code, response = client.query(
-  table_name="MyTable",
-  query_field="Embedding",
-  query_vector=[0.35, 0.55, 0.47, 0.94],
-  response_fields = ["Doc"],
-  limit=2
+    table_name="MyTable",
+    query_field="Embedding",
+    query_vector=[0.35, 0.55, 0.47, 0.94],
+    response_fields=["Doc"],
+    limit=2,
 )
 
 # Query Vectors without specific response field, then it will return all fields
 status_code, response = client.query(
-  table_name="MyTable",
-  query_field="Embedding",
-  query_vector=[0.35, 0.55, 0.47, 0.94],
-  limit=2
+    table_name="MyTable",
+    query_field="Embedding",
+    query_vector=[0.35, 0.55, 0.47, 0.94],
+    limit=2,
 )
 print(response)
 
+# Get Vectors
+status_code, response = client.get(table_name="MyTable", limit=2)
+print(response)
 
 # status_code, response =  client.delete(table_name="MyTable", ids=[3])
-status_code, response =  client.delete(table_name="MyTable", primary_keys=[3, 4])
+status_code, response = client.delete(table_name="MyTable", primary_keys=[3, 4])
 # status_code, response =  client.delete(table_name="MyTable", filter="Doc <> 'San Francisco'")
 print(response)
 
 
-
 # Drop table
 # status_code, response = client.drop_table("MyTable")
 # print(response)

pyepsilla/cloud/client.py

@@ -1,38 +1,52 @@
 #!/usr/bin/env python
 # -*- coding:utf-8 -*-
-import json, datetime, socket, requests, json, pprint
-from typing import Union, Optional
+from __future__ import annotations
+
+import datetime
+import json
+import pprint
+import socket
+from typing import Optional, Union
+
+import requests
 import sentry_sdk
+
 requests.packages.urllib3.disable_warnings()
 
+
 class Client(object):
     def __init__(self, project_id: str, api_key: str):
         self._project_id = project_id
         self._apikey = api_key
-        self._baseurl = "https://dispatch.epsilla.com/api/v2/project/{}".format(self._project_id)
+        self._baseurl = "https://dispatch.epsilla.com/api/v2/project/{}".format(
+            self._project_id
+        )
         self._timeout = 10
-        self._header = {'Content-type': 'application/json', "Connection": "close", 'X-API-Key': api_key}
-
+        self._header = {
+            "Content-type": "application/json",
+            "Connection": "close",
+            "X-API-Key": api_key,
+        }
 
     def validate(self):
-        res = requests.get(url=self._baseurl, data=None, headers=self._header, verify=False)
+        res = requests.get(
+            url=self._baseurl, data=None, headers=self._header, verify=False
+        )
         data = res.json()
         res.close()
         return data
 
-
     def get_db_list(self):
         db_list = []
         req_url = "{}/vectordb/list".format(self._baseurl)
         res = requests.get(url=req_url, data=None, headers=self._header, verify=False)
         status_code = res.status_code
         body = res.json()
         if status_code == 200 and body["statusCode"] == 200:
-            db_list = [ db_id for db_id in res.json()["result"] ]
+            db_list = [db_id for db_id in res.json()["result"]]
         res.close()
         return db_list
 
-
     def get_db_info(self, db_id: str):
         req_url = "{}/vectordb/{}".format(self._baseurl, db_id)
         res = requests.get(url=req_url, data=None, headers=self._header, verify=False)
@@ -41,7 +55,6 @@ def get_db_info(self, db_id: str):
         res.close()
         return status_code, body
 
-
     def vectordb(self, db_id: str):
         ## validate project_id and api_key
         res = self.validate()
@@ -54,33 +67,37 @@ def vectordb(self, db_id: str):
         ## validate db_id
         if not db_id in self.get_db_list():
             raise Exception("Invalid db_id")
-        
+
         ## fetch db public endpoint
         status_code, resp = self.get_db_info(db_id=db_id)
         if resp["statusCode"] == 200:
-            return Vectordb(self._project_id, db_id, self._apikey, resp["result"]["public_endpoint"])
+            return Vectordb(
+                self._project_id, db_id, self._apikey, resp["result"]["public_endpoint"]
+            )
         else:
             print(resp)
             raise Exception("Failed to get db info")
-    
+
     ## TODO
-    def create_db(self, db_id: str, db_type: str, db_name: str, db_description: str = ""):
+    def create_db(
+        self, db_id: str, db_type: str, db_name: str, db_description: str = ""
+    ):
         pass
 
     def delete_db(self, db_id: str):
         pass
 
 
-
 class Vectordb(Client):
     def __init__(self, project_id: str, db_id: str, api_key: str, public_endpoint: str):
         self._project_id = project_id
         self._db_id = db_id
         self._api_key = api_key
         self._public_endpoint = public_endpoint
-        self._baseurl = "https://{}/api/v2/project/{}/vectordb/{}".format(self._public_endpoint, self._project_id, self._db_id)
-        self._header = {'Content-type': 'application/json', 'X-API-Key': self._api_key}
-
+        self._baseurl = "https://{}/api/v2/project/{}/vectordb/{}".format(
+            self._public_endpoint, self._project_id, self._db_id
+        )
+        self._header = {"Content-type": "application/json", "X-API-Key": self._api_key}
 
     ## TODO
     ## create table
@@ -91,22 +108,31 @@ def create_table(self, table_name: str = "MyTable", table_fields: list[str] = No
     def drop_table(self, table_name: str):
         pass
 
-
     ## insert data into table
     def insert(self, table_name: str, records: list[dict]):
         req_url = "{}/data/insert".format(self._baseurl)
         req_data = {"table": table_name, "data": records}
-        res = requests.post(url=req_url, data=json.dumps(req_data), headers=self._header, verify=False)
+        res = requests.post(
+            url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
+        )
         status_code = res.status_code
         body = res.json()
         res.close()
         return status_code, body
 
-
     ## query data from table
-    def query(self, table_name: str, query_field: str = None, query_vector: Union[list,dict] = None, response_fields: Optional[list] = None, limit: int = 2, filter: Optional[str] = None, with_distance: Optional[bool] = False):
+    def query(
+        self,
+        table_name: str,
+        query_field: str = None,
+        query_vector: Union[list, dict] = None,
+        response_fields: Optional[list] = None,
+        limit: int = 2,
+        filter: Optional[str] = None,
+        with_distance: Optional[bool] = False,
+    ):
         req_url = "{}/data/query".format(self._baseurl)
-        req_data = { "table": table_name }
+        req_data = {"table": table_name}
         if query_field != None:
             req_data["queryField"] = query_field
         if query_vector != None:
@@ -120,51 +146,74 @@ def query(self, table_name: str, query_field: str = None, query_vector: Union[li
         if with_distance != None:
             req_data["withDistance"] = with_distance
 
-        res = requests.post(url=req_url, data=json.dumps(req_data), headers=self._header, verify=False)
+        res = requests.post(
+            url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
+        )
         status_code = res.status_code
         body = res.json()
         res.close()
         return status_code, body
 
-
     ## delete data from table
-    def delete(self, table_name: str, primary_keys: Optional[list[Union[str,int]]] = None, ids: Optional[list[Union[str,int]]] = None, filter: Optional[str] = None):
+    def delete(
+        self,
+        table_name: str,
+        primary_keys: Optional[list[Union[str, int]]] = None,
+        ids: Optional[list[Union[str, int]]] = None,
+        filter: Optional[str] = None,
+    ):
         """Epsilla supports delete records by primary keys as default for now."""
         if filter == None:
             if primary_keys == None and ids == None:
-                raise Exception("[ERROR] Please provide at least one of primary keys(ids) and filter to delete record(s).")
+                raise Exception(
+                    "[ERROR] Please provide at least one of primary keys(ids) and filter to delete record(s)."
+                )
         if primary_keys == None and ids != None:
             primary_keys = ids
         if primary_keys != None and ids != None:
             try:
                 sentry_sdk.sdk("Duplicate Keys with both primary keys and ids", "info")
             except Exception as e:
                 pass
-            print("[WARN] Both primary_keys and ids are prvoided, will use primary keys by default!")
+            print(
+                "[WARN] Both primary_keys and ids are prvoided, will use primary keys by default!"
+            )
 
         req_url = "{}/data/delete".format(self._baseurl)
-        req_data = { "table": table_name }
+        req_data = {"table": table_name}
         if primary_keys != None:
             req_data["primaryKeys"] = primary_keys
         if filter != None:
             req_data["filter"] = filter
 
-        res = requests.post(url=req_url, data=json.dumps(req_data), headers=self._header, verify=False)
+        res = requests.post(
+            url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
+        )
         status_code = res.status_code
         body = res.json()
         res.close()
         return status_code, body
 
-
     ## get data from table
-    def get(self, table_name: str, response_fields: Optional[list] = None, primary_keys: Optional[list[Union[str,int]]] = None, ids: Optional[list[Union[str,int]]] = None, filter: Optional[str] = None, skip: Optional[int] = None, limit: Optional[int] = None):
+    def get(
+        self,
+        table_name: str,
+        response_fields: Optional[list] = None,
+        primary_keys: Optional[list[Union[str, int]]] = None,
+        ids: Optional[list[Union[str, int]]] = None,
+        filter: Optional[str] = None,
+        skip: Optional[int] = None,
+        limit: Optional[int] = None,
+    ):
         """Epsilla supports get records by primary keys as default for now."""
         if primary_keys != None and ids != None:
             try:
                 sentry_sdk.sdk("Duplicate Keys with both primary_keys and ids", "info")
             except Exception as e:
                 pass
-            print("[WARN]Both primary_keys and ids are prvoided, will use primary keys by default!")
+            print(
+                "[WARN]Both primary_keys and ids are prvoided, will use primary keys by default!"
+            )
         if primary_keys == None and ids != None:
             primary_keys = ids
 
@@ -181,12 +230,10 @@ def get(self, table_name: str, response_fields: Optional[list] = None, primary_k
             req_data["limit"] = limit
 
         req_url = "{}/data/get".format(self._baseurl)
-        res = requests.post(url=req_url, data=json.dumps(req_data), headers=self._header, verify=False)
+        res = requests.post(
+            url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
+        )
         status_code = res.status_code
         body = res.json()
         res.close()
         return status_code, body
-    
-
-
-

pyepsilla/enterprise/client.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # -*- coding:utf-8 -*-
+from __future__ import annotations
+
 import datetime
 import json
 import pprint
@@ -221,7 +223,7 @@ def query(
         self,
         table_name: str,
         query_field: str = None,
-        query_vector: Union[list,dict] = None,
+        query_vector: Union[list, dict] = None,
         response_fields: Optional[list] = None,
         limit: int = 2,
         filter: Optional[str] = None,