From 356df0562cd529ea9e26caaeee4973e2fa801887 Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 4 Mar 2016 06:20:14 +0000 Subject: [PATCH] [test] Py3 porting --- hubstorage/batchuploader.py | 4 ++-- hubstorage/utils.py | 14 ++++++++------ requirements.txt | 4 +++- tests/test_activity.py | 2 +- tests/test_batchuploader.py | 2 +- tests/test_client.py | 2 +- tests/test_frontier.py | 2 +- tests/test_jobq.py | 2 +- tests/test_jobsmeta.py | 2 +- tests/test_project.py | 4 ++-- tests/test_retry.py | 4 ++-- tests/test_system.py | 2 +- 12 files changed, 24 insertions(+), 20 deletions(-) diff --git a/hubstorage/batchuploader.py b/hubstorage/batchuploader.py index 1337e8d..169bab2 100644 --- a/hubstorage/batchuploader.py +++ b/hubstorage/batchuploader.py @@ -8,7 +8,7 @@ import requests from requests.compat import StringIO from collections import deque -from Queue import Queue +from six.moves.queue import Queue from threading import Thread, Event from .utils import xauth, iterqueue from .serialization import jsonencode @@ -130,7 +130,7 @@ def _checkpoint(self, w): if w.callback is not None: try: w.callback(response) - except Exception, e: + except Exception as e: logger.exception("Callback for %s failed", w.url) def _content_encode(self, qiter, w): diff --git a/hubstorage/utils.py b/hubstorage/utils.py index b34dadb..f8cafb6 100644 --- a/hubstorage/utils.py +++ b/hubstorage/utils.py @@ -1,5 +1,7 @@ import time -from Queue import Empty +import six +from six.moves.queue import Empty +from scrapy.utils.python import to_native_str def urlpathjoin(*parts): @@ -35,10 +37,10 @@ def urlpathjoin(*parts): continue elif isinstance(p, tuple): p = urlpathjoin(*p) - elif isinstance(p, unicode): - p = p.encode('utf8') - elif not isinstance(p, str): - p = str(p) + elif isinstance(p, six.string_types): + p = to_native_str(p) + else: + p = to_native_str(str(p)) url = p if url is None else '{0}/{1}'.format(url.rstrip('/'), p) @@ -81,7 +83,7 @@ class iterqueue(object): it exposes an attribute "count" with the number of messages read - >>> from Queue import Queue + >>> from six.moves.queue import Queue >>> q = Queue() >>> for x in xrange(10): ... q.put(x) diff --git a/requirements.txt b/requirements.txt index 0e8deb5..9b44b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ requests>=1.0 -retrying>=1.3.3 \ No newline at end of file +retrying>=1.3.3 +six +scrapy diff --git a/tests/test_activity.py b/tests/test_activity.py index 0189b3d..a2c3127 100644 --- a/tests/test_activity.py +++ b/tests/test_activity.py @@ -1,7 +1,7 @@ """ Test Activty """ -from hstestcase import HSTestCase +from .hstestcase import HSTestCase class ActivityTest(HSTestCase): diff --git a/tests/test_batchuploader.py b/tests/test_batchuploader.py index 07d7dd3..1e51ae7 100644 --- a/tests/test_batchuploader.py +++ b/tests/test_batchuploader.py @@ -3,8 +3,8 @@ """ import time from collections import defaultdict -from hstestcase import HSTestCase from hubstorage import ValueTooLarge +from .hstestcase import HSTestCase class BatchUploaderTest(HSTestCase): diff --git a/tests/test_client.py b/tests/test_client.py index 52527c7..473104e 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,7 +1,7 @@ """ Test Client """ -from hstestcase import HSTestCase +from .hstestcase import HSTestCase from hubstorage.utils import millitime, apipoll class ClientTest(HSTestCase): diff --git a/tests/test_frontier.py b/tests/test_frontier.py index 50b3a7a..e781377 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -1,7 +1,7 @@ """ Test Frontier """ -from hstestcase import HSTestCase +from .hstestcase import HSTestCase class FrontierTest(HSTestCase): diff --git a/tests/test_jobq.py b/tests/test_jobq.py index 42a1f26..f994635 100644 --- a/tests/test_jobq.py +++ b/tests/test_jobq.py @@ -2,9 +2,9 @@ Test JobQ """ import os, unittest -from hstestcase import HSTestCase from hubstorage.jobq import DuplicateJobError from hubstorage.utils import apipoll +from .hstestcase import HSTestCase EXCLUSIVE = os.environ.get('EXCLUSIVE_STORAGE') diff --git a/tests/test_jobsmeta.py b/tests/test_jobsmeta.py index b2254c4..c2bfbe6 100644 --- a/tests/test_jobsmeta.py +++ b/tests/test_jobsmeta.py @@ -3,7 +3,7 @@ System tests for operations on stored job metadata """ -from hstestcase import HSTestCase +from .hstestcase import HSTestCase class JobsMetadataTest(HSTestCase): diff --git a/tests/test_project.py b/tests/test_project.py index ff39e39..355301b 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -6,8 +6,8 @@ from requests.exceptions import HTTPError from hubstorage import HubstorageClient from hubstorage.utils import millitime -from hstestcase import HSTestCase -from testutil import failing_downloader +from .hstestcase import HSTestCase +from .testutil import failing_downloader class ProjectTest(HSTestCase): diff --git a/tests/test_retry.py b/tests/test_retry.py index 424dcf4..591dbfc 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -1,10 +1,10 @@ """ Test Retry Policy """ -from httplib import BadStatusLine +from six.moves.http_client import BadStatusLine from requests import HTTPError, ConnectionError -from hstestcase import HSTestCase from hubstorage import HubstorageClient +from .hstestcase import HSTestCase import responses import json import re diff --git a/tests/test_system.py b/tests/test_system.py index 4b20d8e..ce82fd5 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -1,8 +1,8 @@ import random from contextlib import closing -from hstestcase import HSTestCase from hubstorage import HubstorageClient from hubstorage.utils import millitime +from .hstestcase import HSTestCase class SystemTest(HSTestCase):