From dd5153f1d7995257645d0053f27412003b17d974 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Thu, 13 Sep 2018 16:05:19 -0700 Subject: [PATCH 01/11] Ignore pytype import errors in google/auth. These errors are raised because the packages are not required, i.e. not listed in setup.py. We can't guarantee they'll be installed or that their pyi files exist in github.com/python/typeshed, so silence potential errors instead. This does impact the accuracy of type checking. --- google/auth/_oauth2client.py | 5 ++++- google/auth/app_engine.py | 2 +- google/auth/crypt/_cryptography_rsa.py | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/google/auth/_oauth2client.py b/google/auth/_oauth2client.py index 71fd7bf41..e450a480d 100644 --- a/google/auth/_oauth2client.py +++ b/google/auth/_oauth2client.py @@ -25,19 +25,22 @@ from google.auth import _helpers import google.auth.app_engine +import google.auth.compute_engine import google.oauth2.credentials import google.oauth2.service_account try: + # pytype: disable=import-error import oauth2client.client import oauth2client.contrib.gce import oauth2client.service_account + # pytype: enable=import-error except ImportError as caught_exc: six.raise_from( ImportError('oauth2client is not installed.'), caught_exc) try: - import oauth2client.contrib.appengine + import oauth2client.contrib.appengine # pytype: disable=import-error _HAS_APPENGINE = True except ImportError: _HAS_APPENGINE = False diff --git a/google/auth/app_engine.py b/google/auth/app_engine.py index f47dae126..e8be07ac5 100644 --- a/google/auth/app_engine.py +++ b/google/auth/app_engine.py @@ -29,7 +29,7 @@ from google.auth import crypt try: - from google.appengine.api import app_identity + from google.appengine.api import app_identity # pytype: disable=import-error except ImportError: app_identity = None diff --git a/google/auth/crypt/_cryptography_rsa.py b/google/auth/crypt/_cryptography_rsa.py index 87076b0ab..6beceb170 100644 --- a/google/auth/crypt/_cryptography_rsa.py +++ b/google/auth/crypt/_cryptography_rsa.py @@ -19,12 +19,14 @@ ``rsa`` library. """ +# pytype: disable=import-error import cryptography.exceptions from cryptography.hazmat import backends from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import padding import cryptography.x509 +# pytype: enable=import-error import pkg_resources from google.auth import _helpers From b4d8f9ae7c967551678743d23bf71d70a1889dda Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Thu, 13 Sep 2018 16:07:36 -0700 Subject: [PATCH 02/11] Ignore pytype import errors in transport/. These imports are not listed in setup.py, because they are optional -- it is assumed the user has them installed if needed. Disabling import-error for these imports prevents useless errors from pytype. --- google/auth/transport/grpc.py | 2 +- google/auth/transport/urllib3.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/google/auth/transport/grpc.py b/google/auth/transport/grpc.py index 0d44f6458..09ecec8a4 100644 --- a/google/auth/transport/grpc.py +++ b/google/auth/transport/grpc.py @@ -18,7 +18,7 @@ import six try: - import grpc + import grpc # pytype: disable=import-error except ImportError as caught_exc: # pragma: NO COVER six.raise_from( ImportError( diff --git a/google/auth/transport/urllib3.py b/google/auth/transport/urllib3.py index 37eb31757..dd889781e 100644 --- a/google/auth/transport/urllib3.py +++ b/google/auth/transport/urllib3.py @@ -31,7 +31,7 @@ certifi = None try: - import urllib3 + import urllib3 # pytype: disable=import-error except ImportError as caught_exc: # pragma: NO COVER import six six.raise_from( @@ -42,7 +42,9 @@ caught_exc, ) import six +# pytype: disable=import-error import urllib3.exceptions # pylint: disable=ungrouped-imports +# pytype: enable=import-error from google.auth import exceptions from google.auth import transport From 5ed53aaa748563ce5d7e66231fb5b87891644b23 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Thu, 13 Sep 2018 16:09:31 -0700 Subject: [PATCH 03/11] Ignore various type errors raised by pytype. - `__init__.py`: pytype is not aware of `__path__`. - jwt.py: the pyi file for urllib.unparse is not aware of None. Empty strings are clearer. --- google/__init__.py | 2 +- google/auth/jwt.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google/__init__.py b/google/__init__.py index a35569c36..4c4f161f2 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -19,4 +19,4 @@ pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) + __path__ = pkgutil.extend_path(__path__, __name__) # pytype: disable=name-error diff --git a/google/auth/jwt.py b/google/auth/jwt.py index ef23db233..3e0faf116 100644 --- a/google/auth/jwt.py +++ b/google/auth/jwt.py @@ -46,7 +46,7 @@ import datetime import json -import cachetools +import cachetools # pytype: disable=import-error import six from six.moves import urllib @@ -738,7 +738,7 @@ def before_request(self, request, method, url, headers): parts = urllib.parse.urlsplit(url) # Strip query string and fragment audience = urllib.parse.urlunsplit( - (parts.scheme, parts.netloc, parts.path, None, None)) + (parts.scheme, parts.netloc, parts.path, "", "")) token = self._get_jwt_for_audience(audience) self.apply(headers, token=token) From bbd81c1457df44f2d8d4e144eb1e843f59a67c39 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Fri, 14 Sep 2018 13:57:30 -0700 Subject: [PATCH 04/11] Add pytype disable comments for scripts/ oauth2client isn't listed as a requirement, so users may not have it installed. --- scripts/obtain_user_auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/obtain_user_auth.py b/scripts/obtain_user_auth.py index ae864666a..daf04f02a 100644 --- a/scripts/obtain_user_auth.py +++ b/scripts/obtain_user_auth.py @@ -23,8 +23,10 @@ import json import os +# pytype: disable=import-error from oauth2client import client from oauth2client import tools +# pytype: disable=import-error HERE = os.path.dirname(__file__) CLIENT_SECRETS_PATH = os.path.abspath(os.path.join( From 1b69b1e9f95570bf8f053fef1efaff21679d0ee2 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Wed, 19 Sep 2018 11:31:53 -0700 Subject: [PATCH 05/11] Fix lint errors from pytype directives. --- google/__init__.py | 3 ++- google/auth/app_engine.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/google/__init__.py b/google/__init__.py index 4c4f161f2..bcb36403b 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -19,4 +19,5 @@ pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) # pytype: disable=name-error + __path__ = pkgutil.extend_path(__path__, # pytype: disable=name-error + __name__) diff --git a/google/auth/app_engine.py b/google/auth/app_engine.py index e8be07ac5..91ba8427d 100644 --- a/google/auth/app_engine.py +++ b/google/auth/app_engine.py @@ -28,10 +28,12 @@ from google.auth import credentials from google.auth import crypt +# pytype: disable=import-error try: - from google.appengine.api import app_identity # pytype: disable=import-error + from google.appengine.api import app_identity except ImportError: app_identity = None +# pytype: enable=import-error class Signer(crypt.Signer): From 5293f85e6c8730a1653f04cafa51c3d04f3a2f94 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Thu, 20 Sep 2018 15:07:19 -0700 Subject: [PATCH 06/11] Enable pytype -V3.6. A few notes: - Previous commits fixed type errors detected by pytype. - setup.cfg disables pytype's `pyi-error`. This is necessary due to incomplete type stubs in https://github.com/python/typeshed. - This only enables pytype's Python3.6 checks. Python2.7 is supported by pytype but incomplete type stubs cause spurious type errors. --- .gitignore | 1 + .travis.yml | 1 + scripts/travis.sh | 4 ++++ setup.cfg | 15 +++++++++++++++ 4 files changed, 21 insertions(+) diff --git a/.gitignore b/.gitignore index c6ab9e7e6..88a8b8bc4 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ tests/data/user-key.json # Generated files pylintrc pylintrc.test +pytype_output/ diff --git a/.travis.yml b/.travis.yml index 8c6728826..896c1e176 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ cache: - ${HOME}/.cache install: - pip install --upgrade tox +- pip install --upgrade pytype script: - scripts/travis.sh deploy: diff --git a/scripts/travis.sh b/scripts/travis.sh index 28bf5faab..d2ba60ec7 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -40,3 +40,7 @@ fi # Run tox. echo "Running tox..." tox + +# Run pytype. +echo "Running pytype..." +pytype diff --git a/setup.cfg b/setup.cfg index 2a9acf13d..0f15a3826 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,17 @@ [bdist_wheel] universal = 1 + +[pytype] +# Where to start analysis. +inputs = . +# Some files aren't worth analyzing, namely tests. Hidden directories (e.g. +# .tox) are automatically filtered out. +exclude = tests system_tests +# All pytype output goes here. +output = pytype_output +# Python version (major.minor) of the target code. +python_version = 3.6 +# Paths to source code directories, separated by ':'. +pythonpath = . +# Errors to disable. +disable = pyi-error From 4f11261df14e71849d069c04bb6bbf2b98e75871 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Thu, 27 Sep 2018 12:45:03 -0700 Subject: [PATCH 07/11] Remove pytype directives. Updates to pytype made these directives unnecessary. --- google/__init__.py | 3 +-- google/auth/jwt.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/google/__init__.py b/google/__init__.py index bcb36403b..a35569c36 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -19,5 +19,4 @@ pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil - __path__ = pkgutil.extend_path(__path__, # pytype: disable=name-error - __name__) + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/google/auth/jwt.py b/google/auth/jwt.py index 3e0faf116..3805f3716 100644 --- a/google/auth/jwt.py +++ b/google/auth/jwt.py @@ -46,7 +46,7 @@ import datetime import json -import cachetools # pytype: disable=import-error +import cachetools import six from six.moves import urllib From f2164ce17dba26c2eab1f9357fe92f9ff0476357 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Thu, 27 Sep 2018 14:31:13 -0700 Subject: [PATCH 08/11] Move pytype install command. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 896c1e176..69b76dece 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,7 @@ cache: directories: - ${HOME}/.cache install: -- pip install --upgrade tox -- pip install --upgrade pytype +- pip install --upgrade tox pytype script: - scripts/travis.sh deploy: From 41cf058ba373a721a3ab87e7b4ec08403daad4b9 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Fri, 28 Sep 2018 14:55:38 -0700 Subject: [PATCH 09/11] Add pytype to tox. --- .travis.yml | 4 +++- scripts/travis.sh | 4 ---- tox.ini | 10 +++++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69b76dece..43f8e31e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,11 +22,13 @@ matrix: env: TOXENV=py36-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 - python: 2.7 env: TOXENV=py27-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 + - python: 3.6 + env: TOXENV=pytype cache: directories: - ${HOME}/.cache install: -- pip install --upgrade tox pytype +- pip install --upgrade tox script: - scripts/travis.sh deploy: diff --git a/scripts/travis.sh b/scripts/travis.sh index d2ba60ec7..28bf5faab 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -40,7 +40,3 @@ fi # Run tox. echo "Running tox..." tox - -# Run pytype. -echo "Running pytype..." -pytype diff --git a/tox.ini b/tox.ini index 1c52c4bb0..59fd6aba1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = lint,py27,py34,py35,py36,pypy,cover +envlist = lint,py27,py34,py35,py36,pypy,cover,pytype [testenv] deps = @@ -82,3 +82,11 @@ deps = flake8 flake8-import-order docutils + +[testenv:pytype] +basepython = python3.6 +commands = + pytype +deps = + {[testenv]deps} + pytype From 6c81ed0161f6c5617d09b09ae41a359a7e0b3ec8 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Mon, 1 Oct 2018 11:00:29 -0700 Subject: [PATCH 10/11] Remove pytype directives for tox-installed imports. These imports are handled by setup.py and tox.ini, so they'll be available when pytype is run under tox. --- google/auth/_oauth2client.py | 2 -- google/auth/crypt/_cryptography_rsa.py | 2 -- google/auth/transport/grpc.py | 2 +- google/auth/transport/urllib3.py | 4 +--- scripts/obtain_user_auth.py | 2 -- 5 files changed, 2 insertions(+), 10 deletions(-) diff --git a/google/auth/_oauth2client.py b/google/auth/_oauth2client.py index e450a480d..afe7dc45d 100644 --- a/google/auth/_oauth2client.py +++ b/google/auth/_oauth2client.py @@ -30,11 +30,9 @@ import google.oauth2.service_account try: - # pytype: disable=import-error import oauth2client.client import oauth2client.contrib.gce import oauth2client.service_account - # pytype: enable=import-error except ImportError as caught_exc: six.raise_from( ImportError('oauth2client is not installed.'), caught_exc) diff --git a/google/auth/crypt/_cryptography_rsa.py b/google/auth/crypt/_cryptography_rsa.py index 6beceb170..87076b0ab 100644 --- a/google/auth/crypt/_cryptography_rsa.py +++ b/google/auth/crypt/_cryptography_rsa.py @@ -19,14 +19,12 @@ ``rsa`` library. """ -# pytype: disable=import-error import cryptography.exceptions from cryptography.hazmat import backends from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import padding import cryptography.x509 -# pytype: enable=import-error import pkg_resources from google.auth import _helpers diff --git a/google/auth/transport/grpc.py b/google/auth/transport/grpc.py index 09ecec8a4..5d67eef22 100644 --- a/google/auth/transport/grpc.py +++ b/google/auth/transport/grpc.py @@ -18,7 +18,7 @@ import six try: - import grpc # pytype: disable=import-error + import grpc except ImportError as caught_exc: # pragma: NO COVER six.raise_from( ImportError( diff --git a/google/auth/transport/urllib3.py b/google/auth/transport/urllib3.py index dd889781e..37eb31757 100644 --- a/google/auth/transport/urllib3.py +++ b/google/auth/transport/urllib3.py @@ -31,7 +31,7 @@ certifi = None try: - import urllib3 # pytype: disable=import-error + import urllib3 except ImportError as caught_exc: # pragma: NO COVER import six six.raise_from( @@ -42,9 +42,7 @@ caught_exc, ) import six -# pytype: disable=import-error import urllib3.exceptions # pylint: disable=ungrouped-imports -# pytype: enable=import-error from google.auth import exceptions from google.auth import transport diff --git a/scripts/obtain_user_auth.py b/scripts/obtain_user_auth.py index daf04f02a..ae864666a 100644 --- a/scripts/obtain_user_auth.py +++ b/scripts/obtain_user_auth.py @@ -23,10 +23,8 @@ import json import os -# pytype: disable=import-error from oauth2client import client from oauth2client import tools -# pytype: disable=import-error HERE = os.path.dirname(__file__) CLIENT_SECRETS_PATH = os.path.abspath(os.path.join( From 65836b948557bd507b9f54df7e5f052ededaff23 Mon Sep 17 00:00:00 2001 From: Teddy Sudol Date: Mon, 1 Oct 2018 11:21:16 -0700 Subject: [PATCH 11/11] Fix lint error. --- google/auth/transport/grpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/auth/transport/grpc.py b/google/auth/transport/grpc.py index 5d67eef22..0d44f6458 100644 --- a/google/auth/transport/grpc.py +++ b/google/auth/transport/grpc.py @@ -18,7 +18,7 @@ import six try: - import grpc + import grpc except ImportError as caught_exc: # pragma: NO COVER six.raise_from( ImportError(