diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..49c52bd --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +.PHONY: release release-minor release-path release-major upload test test-all lint clean + +VERSION?=minor + +release: + bumpversion $(VERSION) + +release-minor: release + +release-patch: + make release VERSION=patch + +release-major: + make release VERSION=major + +upload: clean + python setup.py sdist bdist_wheel upload + +test: + py.test tests/ + +test-all: + tox + +lint: + flake8 hubstorage + +clean: + rm -rf build/ dist/ *.egg-info htmlcov/ + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + diff --git a/tests/hstestcase.py b/tests/hstestcase.py index e096d04..837cfba 100644 --- a/tests/hstestcase.py +++ b/tests/hstestcase.py @@ -1,7 +1,9 @@ import os import unittest import random +import requests from hubstorage import HubstorageClient +from hubstorage.utils import urlpathjoin class HSTestCase(unittest.TestCase): @@ -12,7 +14,7 @@ class HSTestCase(unittest.TestCase): auth = os.getenv('HS_AUTH', 'f' * 32) frontier = 'test' slot = 'site.com' - testbotgroup = 'python-hubstorage-test' + testbotgroups = ['python-hubstorage-test', 'g1'] @classmethod def setUpClass(cls): @@ -63,13 +65,22 @@ def _delete_job(cls, jobkey): @classmethod def _set_testbotgroup(cls): - cls.project.settings.apipost(jl={'botgroups': [cls.testbotgroup]}) + cls.project.settings.apipost(jl={'botgroups': [cls.testbotgroups[0]]}) + # Additional step to populate JobQ's botgroups table + for botgroup in cls.testbotgroups: + url = urlpathjoin(cls.endpoint, 'botgroups', + botgroup, 'max_running') + requests.post(url, auth=cls.project.auth, data='null') cls.project.settings.expire() @classmethod def _unset_testbotgroup(cls): cls.project.settings.apidelete('botgroups') cls.project.settings.expire() + # Additional step to delete botgroups in JobQ + for botgroup in cls.testbotgroups: + url = urlpathjoin(cls.endpoint, 'botgroups', botgroup) + requests.delete(url, auth=cls.project.auth) def start_job(self, **startparams): jobdata = self.project.jobq.start(**startparams) diff --git a/tests/test_jobq.py b/tests/test_jobq.py index 42a1f26..a648d29 100644 --- a/tests/test_jobq.py +++ b/tests/test_jobq.py @@ -142,7 +142,7 @@ def test_summary_jobmeta(self): def test_summary_countstart(self): # push more than 5 jobs into same queue - N = 20 + N = 6 jobq = self.project.jobq for state in ('pending', 'running', 'finished'): for idx in xrange(N): diff --git a/tests/test_retry.py b/tests/test_retry.py index 424dcf4..5a0f22a 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -42,7 +42,7 @@ def test_delete_on_hubstorage_api_does_not_404(self): @responses.activate def test_retrier_does_not_catch_unwanted_exception(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} callback, attempts_count = self.make_request_callback(3, job_metadata, http_error_status=403) @@ -65,7 +65,7 @@ def test_retrier_does_not_catch_unwanted_exception(self): @responses.activate def test_retrier_catches_badstatusline_and_429(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} attempts_count = [0] # use a list for nonlocal mutability used in request_callback @@ -93,7 +93,7 @@ def request_callback(request): @responses.activate def test_api_delete_can_be_set_to_non_idempotent(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} callback_delete, attempts_count_delete = self.make_request_callback(2, job_metadata) @@ -115,7 +115,7 @@ def test_api_delete_can_be_set_to_non_idempotent(self): @responses.activate def test_collection_store_and_delete_are_retried(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1) callback_post, attempts_count_post = self.make_request_callback(2, []) callback_delete, attempts_count_delete = self.make_request_callback(2, []) @@ -136,7 +136,7 @@ def test_collection_store_and_delete_are_retried(self): @responses.activate def test_delete_requests_are_retried(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} callback_getpost, attempts_count_getpost = self.make_request_callback(0, job_metadata) callback_delete, attempts_count_delete = self.make_request_callback(2, job_metadata) @@ -157,7 +157,7 @@ def test_delete_requests_are_retried(self): @responses.activate def test_metadata_save_does_retry(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} callback_get, attempts_count_get = self.make_request_callback(0, job_metadata) callback_post, attempts_count_post = self.make_request_callback(2, job_metadata) @@ -197,7 +197,7 @@ def test_push_job_does_not_retry(self): @responses.activate def test_get_job_does_retry(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} callback, attempts_count = self.make_request_callback(2, job_metadata) @@ -236,7 +236,7 @@ def test_get_job_does_fails_if_no_retries(self): @responses.activate def test_get_job_does_fails_on_too_many_retries(self): # Prepare - client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2) + client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2, max_retry_time=1) job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'} callback, attempts_count = self.make_request_callback(3, job_metadata) diff --git a/tox.ini b/tox.ini index 0428973..ccf5ac5 100644 --- a/tox.ini +++ b/tox.ini @@ -4,12 +4,18 @@ # and then run "tox" from this directory. [tox] -envlist = py27, pypy, py3 +envlist = py27, pypy [testenv] deps = -rrequirements.txt responses==0.5.0 - nose + pytest + pytest-cov -commands = nosetests [] +commands = py.test --cov {posargs} + +addopts = + -v # verbose mode + -r a # show extra test summary info + --doctest-modules # run doctests in all .py modules