Improve documentation, add type hints, start to break py2

Author: jhodges10

Makefile

@@ -31,5 +31,8 @@ run-benchmark:
 format:
 	black frameioclient
 
+view-docs:
+	cd docs && pip install -r requirements.txt && make dev
+
 publish-docs:
-	cd docs && pip install -r requirements.txt && make jekyll && make publish
+	cd docs && pip install -r requirements.txt && make jekyll && make publish

docs/Makefile

@@ -29,4 +29,8 @@ rst:
 	sphinx-build -b rst . dist/rst
 
 html:
-	sphinx-build -b html . dist/html
+	sphinx-build -b html . dist/html
+
+dev:
+	sphinx-autobuild -b html . _build/html
+

docs/classes/search.rst

@@ -1,2 +1,5 @@
 Search
 ===================
+
+.. autoclass:: frameioclient.Search
+    :members:

docs/conf.py

@@ -19,7 +19,7 @@
 # -- Project information -----------------------------------------------------
 
 project = PACKAGE_TITLE
-copyright = 'MIT License 2021, Frame.io'
+copyright = 'MIT License 2022, Frame.io'
 author = AUTHOR_NAME
 
 # The full version, including alpha/beta/rc tags
@@ -36,6 +36,7 @@
     'sphinx.ext.napoleon',
     'sphinxcontrib.restbuilder',
     'sphinx_jekyll_builder',
+    'sphinx_autodoc_typehints'
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/requirements.txt

@@ -8,4 +8,6 @@ xxhash
 furo
 analytics-python
 token-bucket
-speedtest-cli
+speedtest-cli
+sphinx-autobuild
+sphinx-autodoc-typehints

examples/projects/download_project.py

@@ -1,4 +1,4 @@
-from frameioclient.lib.utils import Utils
+from frameioclient.lib.utils import FormatTypes, Utils
 import os
 from pathlib import Path
 
@@ -34,7 +34,7 @@ def demo_project_download(project_id):
 	# pdb.set_trace()
 
 	print(f"Found {item_count} items")
-	print(f"Took {elapsed} second to download {Utils.format_bytes(folder_size, type='size')} for project: {client.projects.get(project_id)['name']}")
+	print(f"Took {elapsed} second to download {Utils.format_value(folder_size, type=FormatTypes.SIZE)} for project: {client.projects.get(project_id)['name']}")
 	print("\n")
 
 if __name__ == "__main__":

frameioclient/lib/transfer.py

@@ -3,27 +3,21 @@
 import os
 import time
 from pprint import pprint
-from typing import Dict, List
 from random import randint
+from typing import Dict, List
 
 import requests
 
-from .exceptions import (
-    AssetChecksumMismatch,
-    AssetChecksumNotPresent,
-    DownloadException,
-)
+from .exceptions import (AssetChecksumMismatch, AssetChecksumNotPresent,
+                         DownloadException)
 from .logger import SDKLogger
-from .utils import Utils
+from .utils import FormatTypes, Utils
 
 logger = SDKLogger("downloads")
 
 from .bandwidth import DiskBandwidth, NetworkBandwidth
-from .exceptions import (
-    AssetNotFullyUploaded,
-    DownloadException,
-    WatermarkIDDownloadException,
-)
+from .exceptions import (AssetNotFullyUploaded, DownloadException,
+                         WatermarkIDDownloadException)
 from .transport import HTTPClient
 
 
@@ -282,7 +276,7 @@ def _download_whole(self, url: str):
         print(
             "Beginning download -- {} -- {}".format(
                 self.asset["name"],
-                Utils.format_bytes(self.downloader.filesize, type="size"),
+                Utils.format_value(self.downloader.filesize, type=FormatTypes.SIZE),
             )
         )
 
@@ -301,12 +295,12 @@ def _download_whole(self, url: str):
                 raise e
 
         download_time = time.time() - start_time
-        download_speed = Utils.format_bytes(
+        download_speed = Utils.format_value(
             math.ceil(self.downloader.filesize / (download_time))
         )
         print(
             "Downloaded {} at {}".format(
-                Utils.format_bytes(self.downloader.filesize, type="size"),
+                Utils.format_value(self.downloader.filesize, type=FormatTypes.SIZE),
                 download_speed,
             )
         )
@@ -386,7 +380,7 @@ def multi_thread_download(self):
         print(
             "Multi-part download -- {} -- {}".format(
                 self.downloader.asset["name"],
-                Utils.format_bytes(self.downloader.filesize, type="size"),
+                Utils.format_value(self.downloader.filesize, type=FormatTypes.SIZE),
             )
         )
 
@@ -435,7 +429,7 @@ def multi_thread_download(self):
         # Log completion event
         SDKLogger("downloads").info(
             "Downloaded {} at {}".format(
-                Utils.format_bytes(self.downloader.filesize, type="size"),
+                Utils.format_value(self.downloader.filesize, type=FormatTypes.SIZE),
                 download_speed,
             )
         )

frameioclient/lib/transport.py

@@ -1,6 +1,7 @@
 import concurrent.futures
 import threading
 import time
+from typing import Dict, Optional
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -13,17 +14,27 @@
 from .version import ClientVersion
 
 
+class HTTPMethods:
+    GET = 'get'
+    POST = 'post'
+    PUT = 'put'
+    DELETE = 'delete'
+    PATCH = 'patch'
+    HEAD = 'head'
+
 class HTTPClient(object):
     """HTTP Client base that automatically handles the following:
-        - Shared thread/session object
-        - Client version headers
-        - Automated retries
+    - Shared thread/session object
+    - Client version headers
+    - Automated retries
 
-    Args:
-        threads (int): Number of threads to use concurrently.
     """
 
-    def __init__(self, threads=default_thread_count):
+    def __init__(self, threads: Optional[int] = default_thread_count):
+        """
+        :param threads: Number of threads to use concurrently.
+        """
+
         # Setup number of threads to use
         self.threads = threads
 
@@ -72,7 +83,7 @@ class APIClient(HTTPClient, object):
         progress (bool): If True, show status bars in console.
     """
 
-    def __init__(self, token, host, threads, progress):
+    def __init__(self, token: str, host: str, threads: int, progress: bool):
         super().__init__(threads)
         self.host = host
         self.token = token
@@ -82,10 +93,10 @@ def __init__(self, token, host, threads, progress):
         self.session = self._get_session()
         self.auth_header = {"Authorization": "Bearer {}".format(self.token)}
 
-    def _format_api_call(self, endpoint):
+    def _format_api_call(self, endpoint: str):
         return "{}/v2{}".format(self.host, endpoint)
 
-    def _api_call(self, method, endpoint, payload={}, limit=None):
+    def _api_call(self, method, endpoint: str, payload: Dict = {}, limit: Optional[int] = None):
         headers = {**self.shared_headers, **self.auth_header}
 
         r = self.session.request(
@@ -117,7 +128,7 @@ def _api_call(self, method, endpoint, payload={}, limit=None):
 
         return r.raise_for_status()
 
-    def get_specific_page(self, method, endpoint, payload, page):
+    def get_specific_page(self, method: HTTPMethods, endpoint: str, payload: Dict, page: int):
         """
         Gets a specific page for that endpoint, used by Pagination Class
 
@@ -127,11 +138,11 @@ def get_specific_page(self, method, endpoint, payload, page):
             payload (dict): Request payload
             page (int): What page to get
         """
-        if method == "get":
+        if method == HTTPMethods.GET:
             endpoint = "{}?page={}".format(endpoint, page)
             return self._api_call(method, endpoint)
 
-        if method == "post":
+        if method == HTTPMethods.POST:
             payload["page"] = page
         return self._api_call(method, endpoint, payload=payload)
 

frameioclient/lib/upload.py

@@ -1,10 +1,12 @@
-import os
+import concurrent.futures
 import math
-import requests
+import os
 import threading
-import concurrent.futures
+from typing import List
+
+import requests
 
-from .utils import Utils
+from .utils import FormatTypes, Utils
 
 thread_local = threading.local()
 
@@ -18,15 +20,14 @@ def __init__(self, asset=None, file=None):
         self.file_num = 0
         self.futures = []
 
-    def _calculate_chunks(self, total_size, chunk_count):
-        """Calculate chunk size
+    def _calculate_chunks(self, total_size: int, chunk_count: int) -> List[int]:
+        """
+        Calculate chunk size
 
-        Args:
-            total_size (int): Total filesize in bytes
-            chunk_count (int): Total number of URL's we got back from the API
+        :param total_size: Total filesize in bytes
+        :param chunk_count: Total number of URL's we got back from the API
 
-        Returns:
-            chunk_offsets (list): List of chunk offsets
+        :return chunk_offsets: List of chunk offsets
         """
         self.chunk_size = int(math.ceil(total_size / chunk_count))
 
@@ -43,7 +44,7 @@ def _get_session(self):
             thread_local.session = requests.Session()
         return thread_local.session
 
-    def _smart_read_chunk(self, chunk_offset, is_final_chunk):
+    def _smart_read_chunk(self, chunk_offset: int, is_final_chunk: bool) -> bytes:
         with open(os.path.realpath(self.file.name), "rb") as file:
             file.seek(chunk_offset, 0)
             if (
@@ -54,7 +55,7 @@ def _smart_read_chunk(self, chunk_offset, is_final_chunk):
                 data = file.read(self.chunk_size)
             return data
 
-    def _upload_chunk(self, task):
+    def _upload_chunk(self, task) -> int:
         url = task[0]
         chunk_offset = task[1]
         chunk_id = task[2]
@@ -141,7 +142,7 @@ def recursive_upload(self, client, folder, parent_asset_id):
 
             complete_dir_obj = os.path.join(folder, file_p)
             print(
-                "Starting {:02d}/{}, Size: {}, Name: {}".format(self.file_num, self.file_count, Utils.format_bytes(os.path.getsize(complete_dir_obj), type='size'), file_p)
+                "Starting {:02d}/{}, Size: {}, Name: {}".format(self.file_num, self.file_count, Utils.format_value(os.path.getsize(complete_dir_obj), type=FormatTypes.SIZE), file_p)
             )
             client.assets.upload(parent_asset_id, complete_dir_obj)