Updates to Google BigQuery connector (#9713, #8327)

Chris Grinolds · jreback · commit 2cf4132c6c30 · 2015-05-01T12:44:26.000-04:00
commit 3d6fdc8751134f6e5bac700519358cf9a700aba1 Author: Chris Grinolds <cgrinolds@mac.com> Date: Wed Apr 29 21:43:10 2015 -0700 Added missing imports that caused tests to fail commit 6c345d8dbc0c36bd4369dd220eceb495ab5ff2c6 Author: Chris Grinolds <cgrinolds@zulily.com> Date: Wed Apr 29 16:23:01 2015 -0700 Updated test suite to handle changes to gbq.py commit f43e65f924d07f88bbbe308b9ea396b1f2a720f9 Author: Chris Grinolds <cgrinolds@zulily.com> Date: Wed Apr 29 08:46:00 2015 -0700 Updated BigQuery connector to no longer use deprecated ```oauth2client.tools.run() (#8327) commit ea92e200ff341a3025b06868a565b1eea506c4c2 Author: Chris Grinolds <cgrinolds@zulily.com> Date: Wed Apr 29 08:43:43 2015 -0700 Import BigQuery dependencies on a per-method basis (#9713)
diff --git a/doc/source/whatsnew/v0.16.1.txt b/doc/source/whatsnew/v0.16.1.txt
@@ -297,3 +297,5 @@ Bug Fixes
 - Bug in hiding ticklabels with subplots and shared axes when adding a new plot to an existing grid of axes (:issue:`9158`)
 - Bug in ``transform`` and ``filter`` when grouping on a categorical variable (:issue:`9921`)
 - Bug in ``transform`` when groups are equal in number and dtype to the input index (:issue:`9700`)
+- Google BigQuery connector now imports dependencies on a per-method basis.(:issue:`9713`)
+- Updated BigQuery connector to no longer use deprecated ``oauth2client.tools.run()`` (:issue:`8327`)
diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py
@@ -13,75 +13,22 @@
 from pandas.tools.merge import concat
 from pandas.core.common import PandasError
 
-_IMPORTS = False
-_GOOGLE_API_CLIENT_INSTALLED = False
-_GOOGLE_API_CLIENT_VALID_VERSION = False
-_GOOGLE_FLAGS_INSTALLED = False
-_GOOGLE_FLAGS_VALID_VERSION = False
-_HTTPLIB2_INSTALLED = False
-_SETUPTOOLS_INSTALLED = False
 
-def _importers():
-    # import things we need
-    # but make this done on a first use basis
-
-    global _IMPORTS
-    if  _IMPORTS:
-        return
-
-    _IMPORTS = True
-
-    if not compat.PY3:
-
-        global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \
-               _GOOGLE_FLAGS_INSTALLED, _GOOGLE_FLAGS_VALID_VERSION, \
-               _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED
-
-        try:
-            import pkg_resources
-            _SETUPTOOLS_INSTALLED = True
-        except ImportError:
-            _SETUPTOOLS_INSTALLED = False
-
-        if _SETUPTOOLS_INSTALLED:
-            try:
-                from apiclient.discovery import build
-                from apiclient.http import MediaFileUpload
-                from apiclient.errors import HttpError
-
-                from oauth2client.client import OAuth2WebServerFlow
-                from oauth2client.client import AccessTokenRefreshError
-                from oauth2client.client import flow_from_clientsecrets
-                from oauth2client.file import Storage
-                from oauth2client.tools import run
-                _GOOGLE_API_CLIENT_INSTALLED=True
-                _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
-
-                if LooseVersion(_GOOGLE_API_CLIENT_VERSION) >= '1.2.0':
-                    _GOOGLE_API_CLIENT_VALID_VERSION = True
-
-            except ImportError:
-                _GOOGLE_API_CLIENT_INSTALLED = False
-
-
-            try:
-                import gflags as flags
-                _GOOGLE_FLAGS_INSTALLED = True
-
-                _GOOGLE_FLAGS_VERSION = pkg_resources.get_distribution('python-gflags').version
+def _check_google_client_version():
+    if compat.PY3:
+        raise NotImplementedError("Google's libraries do not support Python 3 yet")
 
-                if LooseVersion(_GOOGLE_FLAGS_VERSION) >= '2.0':
-                    _GOOGLE_FLAGS_VALID_VERSION = True
+    try:
+        import pkg_resources
 
-            except ImportError:
-                _GOOGLE_FLAGS_INSTALLED = False
+    except ImportError:
+        raise ImportError('Could not import pkg_resources (setuptools).')
 
-            try:
-                import httplib2
-                _HTTPLIB2_INSTALLED = True
-            except ImportError:
-                _HTTPLIB2_INSTALLED = False
+    _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
 
+    if LooseVersion(_GOOGLE_API_CLIENT_VERSION) < '1.2.0':
+        raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google "
+                          "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION)
 
 logger = logging.getLogger('pandas.io.gbq')
 logger.setLevel(logging.ERROR)
@@ -142,6 +89,16 @@ def __init__(self, project_id, reauth=False):
         self.service        = self.get_service(self.credentials)
 
     def get_credentials(self):
+        try:
+            from oauth2client.client import OAuth2WebServerFlow
+            from oauth2client.file import Storage
+            from oauth2client.tools import run_flow, argparser
+
+        except ImportError:
+            raise ImportError('Could not import Google API Client.')
+
+        _check_google_client_version()
+
         flow = OAuth2WebServerFlow(client_id='495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd.apps.googleusercontent.com',
                                    client_secret='kOc9wMptUtxkcIFbtZCcrEAc',
                                    scope='https://www.googleapis.com/auth/bigquery',
@@ -151,18 +108,41 @@ def get_credentials(self):
         credentials = storage.get()
 
         if credentials is None or credentials.invalid or self.reauth:
-            credentials = run(flow, storage)
+            credentials = run_flow(flow, storage, argparser.parse_args([]))
 
         return credentials
 
     def get_service(self, credentials):
+        try:
+            import httplib2
+
+        except ImportError:
+            raise ImportError("pandas requires httplib2 for Google BigQuery support")
+
+        try:
+            from apiclient.discovery import build
+        
+        except ImportError:
+            raise ImportError('Could not import Google API Client.')
+
+        _check_google_client_version()
+
         http = httplib2.Http()
         http = credentials.authorize(http)
         bigquery_service = build('bigquery', 'v2', http=http)
 
         return bigquery_service
 
     def run_query(self, query):
+        try:
+            from apiclient.errors import HttpError
+            from oauth2client.client import AccessTokenRefreshError
+
+        except ImportError:
+            raise ImportError('Could not import Google API Client.')
+
+        _check_google_client_version()
+
         job_collection = self.service.jobs()
         job_data = {
             'configuration': {
@@ -313,38 +293,6 @@ def _parse_entry(field_value, field_type):
         return field_value == 'true'
     return field_value
 
-def _test_imports():
-
-    _importers()
-    _GOOGLE_API_CLIENT_INSTALLED
-    _GOOGLE_API_CLIENT_VALID_VERSION
-    _GOOGLE_FLAGS_INSTALLED
-    _GOOGLE_FLAGS_VALID_VERSION
-    _HTTPLIB2_INSTALLED
-    _SETUPTOOLS_INSTALLED
-
-    if compat.PY3:
-        raise NotImplementedError("Google's libraries do not support Python 3 yet")
-
-    if not _SETUPTOOLS_INSTALLED:
-        raise ImportError('Could not import pkg_resources (setuptools).')
-
-    if not _GOOGLE_API_CLIENT_INSTALLED:
-        raise ImportError('Could not import Google API Client.')
-
-    if not _GOOGLE_FLAGS_INSTALLED:
-        raise ImportError('Could not import Google Command Line Flags Module.')
-
-    if not _GOOGLE_API_CLIENT_VALID_VERSION:
-        raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google "
-                          "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION)
-
-    if not _GOOGLE_FLAGS_VALID_VERSION:
-        raise ImportError("pandas requires python-gflags >= 2.0.0 for Google "
-                          "BigQuery support, current version " + _GOOGLE_FLAGS_VERSION)
-
-    if not _HTTPLIB2_INSTALLED:
-        raise ImportError("pandas requires httplib2 for Google BigQuery support")
 
 def read_gbq(query, project_id = None, index_col=None, col_order=None, reauth=False):
     """Load data from Google BigQuery.
@@ -379,7 +327,6 @@ def read_gbq(query, project_id = None, index_col=None, col_order=None, reauth=Fa
 
     """
 
-    _test_imports()
 
     if not project_id:
         raise TypeError("Missing required parameter: project_id")
@@ -450,7 +397,6 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=10000,
         if multiple accounts are used.
 
     """
-    _test_imports()
 
     if not project_id:
         raise TypeError("Missing required parameter: project_id")
diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py
@@ -12,6 +12,9 @@
 
 import numpy as np
 
+from distutils.version import LooseVersion
+from pandas import compat
+
 from pandas import NaT
 from pandas.compat import u
 from pandas.core.frame import DataFrame
@@ -22,16 +25,77 @@
 
 VERSION = platform.python_version()
 
+_IMPORTS = False
+_GOOGLE_API_CLIENT_INSTALLED = False
+_GOOGLE_API_CLIENT_VALID_VERSION = False
+_HTTPLIB2_INSTALLED = False
+_SETUPTOOLS_INSTALLED = False
+
 def missing_bq():
     try:
         subprocess.call('bq')
         return False
     except OSError:
         return True
 
+def _test_imports():
+    if not compat.PY3:
+
+        global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \
+               _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED
+
+        try:
+            import pkg_resources
+            _SETUPTOOLS_INSTALLED = True
+        except ImportError:
+            _SETUPTOOLS_INSTALLED = False
+
+        if _SETUPTOOLS_INSTALLED:
+            try:
+                from apiclient.discovery import build
+                from apiclient.errors import HttpError
+
+                from oauth2client.client import OAuth2WebServerFlow
+                from oauth2client.client import AccessTokenRefreshError
+
+                from oauth2client.file import Storage
+                from oauth2client.tools import run_flow
+                _GOOGLE_API_CLIENT_INSTALLED=True
+                _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
+
+                if LooseVersion(_GOOGLE_API_CLIENT_VERSION) >= '1.2.0':
+                    _GOOGLE_API_CLIENT_VALID_VERSION = True
+
+            except ImportError:
+                _GOOGLE_API_CLIENT_INSTALLED = False
+
+
+            try:
+                import httplib2
+                _HTTPLIB2_INSTALLED = True
+            except ImportError:
+                _HTTPLIB2_INSTALLED = False
+    
+
+    if compat.PY3:
+        raise NotImplementedError("Google's libraries do not support Python 3 yet")
+
+    if not _SETUPTOOLS_INSTALLED:
+        raise ImportError('Could not import pkg_resources (setuptools).')
+
+    if not _GOOGLE_API_CLIENT_INSTALLED:
+        raise ImportError('Could not import Google API Client.')
+
+    if not _GOOGLE_API_CLIENT_VALID_VERSION:
+        raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google "
+                          "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION)
+
+    if not _HTTPLIB2_INSTALLED:
+        raise ImportError("pandas requires httplib2 for Google BigQuery support")
+
 def test_requirements():
     try:
-        gbq._test_imports()
+        _test_imports()
     except (ImportError, NotImplementedError) as import_exception:
         raise nose.SkipTest(import_exception)
 
