support create/drop table for cloud

Author: eric-epsilla

examples/hello_epsilla_cloud.py

@@ -6,19 +6,30 @@
 # 2. create a table with schema in db
 # 3. get the api key with project id, run this program
 
+import sys
 
 from pyepsilla import cloud
 
 # Connect to Epsilla Cloud
 client = cloud.Client(
-    project_id="32ef3a3f-fcb0-4c4b-98bb-fca01bca0d0a", api_key="epsilla"
+    project_id="7a68814c-f839-4a67-9ec6-93c027c865e6",
+    api_key="epsilla-cloud-api-key",
 )
 
 # Connect to Vectordb
-db = client.vectordb(db_id="df7431d0-806b-4654-8b45-4bdb20038e26")
+db = client.vectordb(db_id="6accafb1-476d-43b0-aa64-12ebfbf7442d")
 
 
-# Create a table with schema on Epsilla Cloud Console
+# Create a table with schema
+status_code, response = db.create_table(
+    table_name="MyTable",
+    table_fields=[
+        {"name": "ID", "dataType": "INT", "primaryKey": True},
+        {"name": "Doc", "dataType": "STRING"},
+        {"name": "Embedding", "dataType": "VECTOR_FLOAT", "dimensions": 4},
+    ],
+)
+print(status_code, response)
 
 
 # Insert new vector records into table
@@ -49,3 +60,7 @@
 status_code, response = db.delete(table_name="MyTable", primary_keys=[4, 5])
 status_code, response = db.delete(table_name="MyTable", filter="Doc <> 'San Francisco'")
 print(status_code, response)
+
+# Drop table
+status_code, response = db.drop_table(table_name="MyTable")
+print(status_code, response)

pyepsilla/cloud/client.py

@@ -10,6 +10,7 @@
 
 import requests
 import sentry_sdk
+from pydantic import BaseModel, Field, constr
 
 requests.packages.urllib3.disable_warnings()
 
@@ -18,7 +19,7 @@ class Client(object):
     def __init__(self, project_id: str, api_key: str):
         self._project_id = project_id
         self._apikey = api_key
-        self._baseurl = "https://dispatch.epsilla.com/api/v2/project/{}".format(
+        self._baseurl = "https://dispatch.epsilla.com/api/v3/project/{}".format(
             self._project_id
         )
         self._timeout = 10
@@ -30,7 +31,10 @@ def __init__(self, project_id: str, api_key: str):
 
     def validate(self):
         res = requests.get(
-            url=self._baseurl, data=None, headers=self._header, verify=False
+            url=self._baseurl + "/vectordb/list",
+            data=None,
+            headers=self._header,
+            verify=False,
         )
         data = res.json()
         res.close()
@@ -56,19 +60,20 @@ def get_db_info(self, db_id: str):
         return status_code, body
 
     def vectordb(self, db_id: str):
-        ## validate project_id and api_key
+        # validate project_id and api_key
         res = self.validate()
         if res["statusCode"] != 200:
             if res["statusCode"] == 404:
                 raise Exception("Invalid project_id")
             if res["statusCode"] == 401:
                 raise Exception("Invalid api_key")
 
-        ## validate db_id
-        if not db_id in self.get_db_list():
+        # validate db_id
+        db_list = self.get_db_list()
+        if db_id not in db_list:
             raise Exception("Invalid db_id")
 
-        ## fetch db public endpoint
+        # fetch db public endpoint
         status_code, resp = self.get_db_info(db_id=db_id)
         if resp["statusCode"] == 200:
             return Vectordb(
@@ -78,14 +83,86 @@ def vectordb(self, db_id: str):
             print(resp)
             raise Exception("Failed to get db info")
 
-    ## TODO
+    # Create DB
     def create_db(
-        self, db_id: str, db_type: str, db_name: str, db_description: str = ""
+        self,
+        db_name: str = Field(pattern=r"^[a-zA-Z-0-9]{4,32}$", strict=True),
+        db_id: Optional[str] = None,
+        project_id: Optional[str] = "default",
+        min_replicas: Optional[int] = 0,
+        max_replicas: Optional[int] = 1,
+        sharding_init_number: Optional[int] = 1,
+        sharding_increase_step: Optional[int] = 2,
+        sharding_capacity: Optional[int] = 150000,
+        sharding_increase_threshold: Optional[float] = 0.9,
     ):
-        pass
+        req_url = "{}/vectordb/create".format(self._baseurl)
+        req_data = {
+            "db_name": db_name,
+            "db_uuid": db_id,
+            "project_id": project_id,
+            "min_replicas": min_replicas,
+            "max_replicas": max_replicas,
+            "sharding_init_number": sharding_init_number,
+            "sharding_increase_step": sharding_increase_step,
+            "sharding_capacity": sharding_capacity,
+            "sharding_increase_threshold": sharding_increase_threshold,
+        }
+        resp = requests.post(
+            url=req_url,
+            data=json.dumps(req_data),
+            headers=self._header,
+            verify=False,
+        )
+        status_code = resp.status_code
+        body = resp.json()
+        resp.close()
+        return status_code, body
+
+    # Load DB
+    def load_db(self, db_id: str):
+        req_url = "{}/vectordb/{}/load".format(self._baseurl, db_id)
+        req_data = {}
+        resp = requests.post(
+            url=req_url,
+            data=json.dumps(req_data),
+            headers=self._header,
+            verify=False,
+        )
+        status_code = resp.status_code
+        body = resp.json()
+        resp.close()
+        return status_code, body
+
+    # Unload DB
+    def unload_db(self, db_id: str):
+        req_url = "{}/vectordb/{}/unload".format(self._baseurl, db_id)
+        req_data = {}
+        resp = requests.post(
+            url=req_url,
+            data=json.dumps(req_data),
+            headers=self._header,
+            verify=False,
+        )
+        status_code = resp.status_code
+        body = resp.json()
+        resp.close()
+        return status_code, body
 
-    def delete_db(self, db_id: str):
-        pass
+    # Delete DB
+    def drop_db(self, db_id: str):
+        req_url = "{}/vectordb/{}".format(self._baseurl, db_id)
+        req_data = {}
+        resp = requests.delete(
+            url=req_url,
+            data=json.dumps(req_data),
+            headers=self._header,
+            verify=False,
+        )
+        status_code = resp.status_code
+        body = resp.json()
+        resp.close()
+        return status_code, body
 
 
 class Vectordb(Client):
@@ -94,24 +171,58 @@ def __init__(self, project_id: str, db_id: str, api_key: str, public_endpoint: s
         self._db_id = db_id
         self._api_key = api_key
         self._public_endpoint = public_endpoint
-        self._baseurl = "https://{}/api/v2/project/{}/vectordb/{}".format(
+        self._baseurl = "https://{}/api/v3/project/{}/vectordb/{}".format(
             self._public_endpoint, self._project_id, self._db_id
         )
         self._header = {"Content-type": "application/json", "X-API-Key": self._api_key}
 
-    ## TODO
-    ## create table
-    def create_table(self, table_name: str = "MyTable", table_fields: list[str] = None):
-        pass
+    # List table
+    def list_tables(self):
+        if self._db_id is None:
+            raise Exception("[ERROR] db_id is None!")
+        req_url = "{}/table/list".format(self._baseurl)
+        res = requests.get(url=req_url, headers=self._header, verify=False)
+        status_code = res.status_code
+        body = res.json()
+        res.close()
+        return status_code, body
 
-    ## drop table
+    # Create table
+    def create_table(self, table_name: str, table_fields: list[str] = None):
+        if self._db_id is None:
+            raise Exception("[ERROR] db_id is None!")
+        if table_fields is None:
+            table_fields = []
+        req_url = "{}/table/create".format(self._baseurl)
+        req_data = {"table_name": table_name, "fields": table_fields}
+        res = requests.post(
+            url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
+        )
+        status_code = res.status_code
+        body = res.json()
+        res.close()
+        return status_code, body
+
+    # Drop table
     def drop_table(self, table_name: str):
-        pass
+        if self._db_id is None:
+            raise Exception("[ERROR] db_id is None!")
+        req_url = "{}/table/delete?table_name={}".format(self._baseurl, table_name)
+        req_data = {}
+        res = requests.delete(
+            url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
+        )
+        status_code = res.status_code
+        body = res.json()
+        res.close()
+        return status_code, body
 
-    ## insert data into table
+    # Insert data into table
     def insert(self, table_name: str, records: list[dict]):
         req_url = "{}/data/insert".format(self._baseurl)
         req_data = {"table": table_name, "data": records}
+        print("req_url: ", req_url)
+        print("req_data: ", req_data)
         res = requests.post(
             url=req_url, data=json.dumps(req_data), headers=self._header, verify=False
         )
@@ -120,7 +231,7 @@ def insert(self, table_name: str, records: list[dict]):
         res.close()
         return status_code, body
 
-    ## query data from table
+    # Query data from table
     def query(
         self,
         table_name: str,
@@ -154,7 +265,7 @@ def query(
         res.close()
         return status_code, body
 
-    ## delete data from table
+    # Delete data from table
     def delete(
         self,
         table_name: str,
@@ -163,14 +274,14 @@ def delete(
         filter: Optional[str] = None,
     ):
         """Epsilla supports delete records by primary keys as default for now."""
-        if filter == None:
-            if primary_keys == None and ids == None:
+        if filter is None:
+            if primary_keys is None and ids is None:
                 raise Exception(
                     "[ERROR] Please provide at least one of primary keys(ids) and filter to delete record(s)."
                 )
-        if primary_keys == None and ids != None:
+        if primary_keys is None and ids is not None:
             primary_keys = ids
-        if primary_keys != None and ids != None:
+        if primary_keys is not None and ids is not None:
             try:
                 sentry_sdk.sdk("Duplicate Keys with both primary keys and ids", "info")
             except Exception as e:
@@ -181,9 +292,9 @@ def delete(
 
         req_url = "{}/data/delete".format(self._baseurl)
         req_data = {"table": table_name}
-        if primary_keys != None:
+        if primary_keys is not None:
             req_data["primaryKeys"] = primary_keys
-        if filter != None:
+        if filter is not None:
             req_data["filter"] = filter
 
         res = requests.post(
@@ -194,7 +305,7 @@ def delete(
         res.close()
         return status_code, body
 
-    ## get data from table
+    # Get data from table
     def get(
         self,
         table_name: str,
@@ -206,27 +317,27 @@ def get(
         limit: Optional[int] = None,
     ):
         """Epsilla supports get records by primary keys as default for now."""
-        if primary_keys != None and ids != None:
+        if primary_keys is not None and ids is not None:
             try:
                 sentry_sdk.sdk("Duplicate Keys with both primary_keys and ids", "info")
             except Exception as e:
                 pass
             print(
                 "[WARN]Both primary_keys and ids are prvoided, will use primary keys by default!"
             )
-        if primary_keys == None and ids != None:
+        if primary_keys is None and ids is not None:
             primary_keys = ids
 
         req_data = {"table": table_name}
-        if response_fields != None:
+        if response_fields is not None:
             req_data["response"] = response_fields
-        if primary_keys != None:
+        if primary_keys is not None:
             req_data["primaryKeys"] = primary_keys
-        if filter != None:
+        if filter is not None:
             req_data["filter"] = filter
-        if skip != None:
+        if skip is not None:
             req_data["skip"] = skip
-        if limit != None:
+        if limit is not None:
             req_data["limit"] = limit
 
         req_url = "{}/data/get".format(self._baseurl)