Separate py2 and py3 uploaders, get tests working!

Finish verification code

Add Python 3.7 test

Increase verobosity in integration test

Fix python 2/3 cmp() function problem

Debug python version comparison

Continue to debug python versions

Finished test!

Add test for latest python version

Add linux 3.6.5

Improve test naming

Make universal package

Modify asset print

Attempting to debug python 2.7

Add more debugging for py 2.7

Improve debug verbosity

Remove debugging for uploader

Seperate py2 and py3 uploaders

Remove version limitations

Remove extraneous print and add cleanup task

Add python version to Frame.io folder name

Try again with new assets

Author: jhodges10

.circleci/config.yml

@@ -9,27 +9,39 @@ workflows:
     jobs:
       - build
 
-      - windows_py_27:
+      - linux_py_27:
           requires:
             - build
 
-      # - mac_py_27:
-      #     requires:
-      #       - build
+      - linux_py_37:
+          requires:
+            - build
+
+      - linux_py_365:
+          requires:
+            - build
+
+      - linux_py_latest:
+          requires:
+            - build
 
       # - windows_py_38:
       #     requires:
       #       - build
+
       # - mac_py_38:
       #     requires:
       #       - build
 
       - deploy:
           requires:
-            - windows_py_27
-            # - mac_py_27
+            - linux_py_27
+            - linux_py_365
+            - linux_py_37
+            - linux_py_latest
             # - windows_py_38
             # - mac_py_38
+
           filters:
             branches:
               only:
@@ -58,7 +70,7 @@ jobs:
             echo -e "Running sdist"
             python setup.py sdist
             echo -e "Creating wheel"
-            python setup.py bdist_wheel
+            python setup.py bdist_wheel --universal
 
       # - store_artifacts:
       #     path: /home/circleci/project/
@@ -69,7 +81,8 @@ jobs:
           paths:
             - .
 
-  windows_py_27:
+  linux_py_27:
+    description: Linux 2.7.17
     docker:
       - image: circleci/python:2.7.17
 
@@ -88,6 +101,69 @@ jobs:
           command: |
             python /tmp/artifact/tests/integration.py
 
+  linux_py_365:
+    description: Linux 3.6.5
+    docker:
+      - image: circleci/python:3.6.5
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+
+  linux_py_37:
+    description: Linux 3.7.7
+    docker:
+      - image: circleci/python:3.7.7
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+
+  linux_py_latest:
+    description: Linux latest
+    docker:
+      - image: circleci/python:latest
+
+    steps:
+      - attach_workspace:
+          at: /tmp/artifact
+          name: Attach build artifact
+
+      - run:
+          name: Install package
+          command: |
+            pip install '/tmp/artifact'
+
+      - run:
+          name: Run integration test
+          command: |
+            python /tmp/artifact/tests/integration.py
+
+
   deploy:
     docker:
     - image: circleci/python:latest

frameioclient/client.py

@@ -1,9 +1,14 @@
+import sys
 import requests
-from .upload import FrameioUploader
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 from .download import FrameioDownloader
 
+if sys.version_info.major >= 3:
+  from .py3_uploader import FrameioUploader
+else:
+  from .py2_uploader import FrameioUploader
+
 class PaginatedResponse(object):
   def __init__(self, results=[], page=0, page_size=0, total=0, total_pages=0):
     super(PaginatedResponse, self).__init__()

frameioclient/upload.py frameioclient/py2_uploader.pyframeioclient/upload.py renamed to frameioclient/py2_uploader.py

@@ -28,9 +28,14 @@ def upload(self):
     size = int(math.ceil(total_size / len(upload_urls)))
 
     for i, chunk in enumerate(self._read_chunk(self.file, size)):
-      proc = Process(target=self._upload_chunk, args=(upload_urls[i], chunk,))
-      procs.append(proc)
-      proc.start()
+      try:
+        proc = Process(target=self._upload_chunk, args=(upload_urls[i], chunk))
+        procs.append(proc)
+        proc.start()
+      except IndexError:
+        # In python 2.7 the _read_chunk function sometimes keeps going \
+        # past where it should, this prevents that.
+        pass
 
     for proc in procs:
       proc.join()

frameioclient/py3_uploader.py

@@ -0,0 +1,47 @@
+import requests
+import math
+import concurrent.futures
+import threading
+
+thread_local = threading.local()
+
+class FrameioUploader(object):
+  def __init__(self, asset, file):
+    self.asset = asset
+    self.file = file
+
+  def _read_chunk(self, file, size):
+    while True:
+      data = file.read(size)
+      if not data:
+        break
+      yield data
+
+  def _get_session(self):
+    if not hasattr(thread_local, "session"):
+        thread_local.session = requests.Session()
+    return thread_local.session
+
+  def _upload_chunk(self, task):
+    url = task[0]
+    chunk = task[1]
+    session = self._get_session()
+
+    session.put(url, data=chunk, headers={
+      'content-type': self.asset['filetype'],
+      'x-amz-acl': 'private'
+    })
+
+  def upload(self):
+    total_size = self.asset['filesize']
+    upload_urls = self.asset['upload_urls']
+    size = int(math.ceil(total_size / len(upload_urls)))
+
+    tasks = []
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+      for i, chunk in enumerate(self._read_chunk(self.file, size)):
+        task = (upload_urls[i], chunk)
+        tasks.append(task)
+
+      executor.map(self._upload_chunk, tasks)

setup.py

@@ -25,7 +25,7 @@ def run(self):
 setuptools.setup(
   name='frameioclient',
   version=version,
-  python_requires='>=2.6, !=3.8.*, <4',
+  python_requires='>=2.6',
   install_requires=[
     'requests',
     'urllib3',