Merge pull request #35 from Frameio/circleci-project-setup

DEVREL-234, DEVREL-235, DEVREL-250: CircleCI setup, integration testing, improved dev tooling, and CODEOWNER setup

Author: jhodges10

.bumpversion.cfg

@@ -0,0 +1,6 @@
+[bumpversion]
+current_version = 0.7.5
+commit = True
+tag = True
+
+[bumpversion:file:setup.py]

.circleci/config.yml

@@ -0,0 +1,174 @@
+version: 2.1
+
+orbs:
+  python: circleci/python@0.2.1
+  win: circleci/windows@2.2.0
+
+workflows:
+  version: 2
+  build_test_deploy:
+    jobs:
+      - build
+
+      - linux_py_27_17:
+          requires:
+            - build
+
+      - linux_py_27_latest:
+          requires:
+            - build
+
+      - linux_py_37:
+          requires:
+            - build
+
+      - linux_py_latest:
+          requires:
+            - build
+
+      - deploy:
+          requires:
+            - linux_py_27_17
+            - linux_py_27_latest
+            - linux_py_37
+            - linux_py_latest
+
+          filters:
+            branches:
+              only:
+                - master
+            tags:
+              only: /[0-9]+(\.[0-9]+)*/
+
+jobs:
+  build:
+    docker:
+      - image: circleci/python:latest
+    steps:
+      - checkout:
+          name: Checkout Git
+
+      - run:
+          name: Build Package
+          command: |
+            echo -e "Running sdist"
+            python setup.py sdist
+            echo -e "Creating wheel"
+            python setup.py bdist_wheel --universal
+
+      - persist_to_workspace:
+          root: /home/circleci/project/
+          paths:
+            - .
+
+  linux_py_27_17:
+    description: Linux 2.7.17
+    docker:
+      - image: circleci/python:2.7.17
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+  linux_py_27_latest:
+    description: Linux 2.7.18
+    docker:
+      - image: circleci/python:2.7.18
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+  linux_py_37:
+    description: Linux 3.7.7
+    docker:
+      - image: circleci/python:3.7.7
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+  linux_py_latest:
+    description: Linux latest
+    docker:
+      - image: circleci/python:latest
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+
+  deploy:
+    docker:
+    - image: circleci/python:latest
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install dependencies
+          command: |
+            pip install setuptools wheel twine
+
+      - run:
+          name: init .pypirc
+          command: |
+            cd /tmp/artifact
+            echo -e "[pypi]" >> ~/.pypirc
+            echo -e "username = $TWINE_USERNAME" >> ~/.pypirc
+            echo -e "password = $TWINE_PASSWORD" >> ~/.pypirc
+
+      - run:
+          name: Upload to pypi
+          command: |
+            cd /tmp/artifact
+            . venv/bin/activate
+            twine upload dist/
+
+

.github/CODEOWNERS

@@ -0,0 +1,10 @@
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# @global-owner1 and @global-owner2 will be requested for
+# review when someone opens a pull request.
+*       @jhodges10
+
+# Order is important; the last matching pattern takes the most
+# precedence. When someone opens a pull request that only
+# modifies Python files.
+*.py    jmeggesto@frame.io billy@frame.io zach@frame.io jay@frame.io

.github/workflows/pythonpublish.yml

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+install-dev:
+	pip3 install -e .[dev]
+	curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | bash
+
+bump-minor:
+	bump2version minor
+
+bump-major:
+	bump2version major
+
+bump-patch:
+	bump2version patch

Makefile

@@ -32,7 +32,7 @@ _Note: The Frame.io Python client may not work correctly in Python 3.8+_
 
 ## Usage
 
-_Note: A valid token is required to make requests to Frame.io. Please contact platform@frame.io to get setup._
+_Note: A valid token is required to make requests to Frame.io. Go to our [Developer Portal](https://developer.frame.io/) to get a token!_
 
 In addition to the snippets below, examples are included in [/examples](/examples).
 

README.md

@@ -1 +1,2 @@
 from .client import FrameioClient
+from .utils import stream

frameioclient/__init__.py

@@ -1,9 +1,14 @@
+import sys
 import requests
-from .upload import FrameioUploader
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 from .download import FrameioDownloader
 
+if sys.version_info.major >= 3:
+  from .py3_uploader import FrameioUploader
+else:
+  from .py2_uploader import FrameioUploader
+
 class PaginatedResponse(object):
   def __init__(self, results=[], page=0, page_size=0, total=0, total_pages=0):
     super(PaginatedResponse, self).__init__()
@@ -26,12 +31,23 @@ def __init__(self, token, host='https://api.frame.io'):
         status_forcelist=[429],
         method_whitelist=["POST", "OPTIONS", "GET"]
     )
+    self.client_version = self._get_version()
+
+  def _get_version(self):
+    try:
+        from importlib import metadata
+    except ImportError:
+        # Running on pre-3.8 Python; use importlib-metadata package
+        import importlib_metadata as metadata
+
+    return metadata.version('frameioclient')
 
   def _api_call(self, method, endpoint, payload={}):
     url = '{}/v2{}'.format(self.host, endpoint)
 
     headers = {
-      'Authorization': 'Bearer {}'.format(self.token)
+      'Authorization': 'Bearer {}'.format(self.token),
+      'x-frameio-client': 'python/{}'.format(self.client_version)
     }
 
     adapter = HTTPAdapter(max_retries=self.retry_strategy)
@@ -48,13 +64,14 @@ def _api_call(self, method, endpoint, payload={}):
 
     if r.ok:
       if r.headers.get('page-number'):
-        return PaginatedResponse(
-          results=r.json(), 
-          page=r.headers['page-number'], 
-          page_size=r.headers['per-page'],
-          total_pages=r.headers['total-pages'],
-          total=r.headers['total']
-        )
+        if int(r.headers.get('total-pages')) > 1:
+          return PaginatedResponse(
+            results=r.json(), 
+            page=r.headers['page-number'], 
+            page_size=r.headers['per-page'],
+            total_pages=r.headers['total-pages'],
+            total=r.headers['total']
+          )
 
       return r.json()
 
@@ -220,7 +237,7 @@ def update_asset(self, asset_id, **kwargs):
       Example::
         client.update_asset("adeffee123342", name="updated_filename.mp4")
     """
-    endpoint = '/assets/{}/children'.format(asset_id)
+    endpoint = '/assets/{}'.format(asset_id)
     return self._api_call('put', endpoint, kwargs)
 
   def upload(self, asset, file):
@@ -260,7 +277,7 @@ def get_comment(self, comment_id, **kwargs):
     :Args:
       comment_id (string): The comment id.
     """
-    endpoint = '/comments/{id}'.format(comment_id)
+    endpoint = '/comments/{}'.format(comment_id)
     return self._api_call('get', endpoint, **kwargs)
 
   def get_comments(self, asset_id, **kwargs):

frameioclient/client.py

@@ -28,9 +28,14 @@ def upload(self):
     size = int(math.ceil(total_size / len(upload_urls)))
 
     for i, chunk in enumerate(self._read_chunk(self.file, size)):
-      proc = Process(target=self._upload_chunk, args=(upload_urls[i], chunk,))
-      procs.append(proc)
-      proc.start()
+      try:
+        proc = Process(target=self._upload_chunk, args=(upload_urls[i], chunk))
+        procs.append(proc)
+        proc.start()
+      except IndexError:
+        # In python 2.7 the _read_chunk function sometimes keeps going \
+        # past where it should, this prevents that.
+        pass
 
     for proc in procs:
       proc.join()

frameioclient/upload.py frameioclient/py2_uploader.pyframeioclient/upload.py renamed to frameioclient/py2_uploader.py

@@ -0,0 +1,43 @@
+import requests
+import math
+import concurrent.futures
+import threading
+
+thread_local = threading.local()
+
+class FrameioUploader(object):
+  def __init__(self, asset, file):
+    self.asset = asset
+    self.file = file
+
+  def _read_chunk(self, file, size):
+    while True:
+      data = file.read(size)
+      if not data:
+        break
+      yield data
+
+  def _get_session(self):
+    if not hasattr(thread_local, "session"):
+        thread_local.session = requests.Session()
+    return thread_local.session
+
+  def _upload_chunk(self, task):
+    url = task[0]
+    chunk = task[1]
+    session = self._get_session()
+
+    session.put(url, data=chunk, headers={
+      'content-type': self.asset['filetype'],
+      'x-amz-acl': 'private'
+    })
+
+  def upload(self):
+    total_size = self.asset['filesize']
+    upload_urls = self.asset['upload_urls']
+    size = int(math.ceil(total_size / len(upload_urls)))
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+      for i, chunk in enumerate(self._read_chunk(self.file, size)):
+        task = (upload_urls[i], chunk)
+        executor.submit(self._upload_chunk, task)