diff --git a/.travis.yml b/.travis.yml index c9c8e275..933ff9c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,45 +10,28 @@ env: - PATH="$PYENV_ROOT/bin:$PATH" - PIP_CACHE_DIR="$HOME/.cache/pip" # unify pip cache location for all platforms stages: -# - check -# - test + - test - build -# Travis doesn't support testing python on Mac yet, so we need -# to workaround it by installing it directly with brew. -# -# Note that we specify language as ruby-2.4.3 to help us satisfy -# fpm dependencies, as travis runs el capitan with an old ruby -# version, that causes fpm to crash. -# # Each job will use the previous job's `stage` tag if not specified. jobs: include: - # patch formatting checks -# - name: "Check patch formatting" -# stage: check -# os: linux -# language: python -# python: 3.7 -# before_install: -# install: -# script: ./scripts/ci/check_patch.sh # test jobs -# - name: "3.7 on Windows" -# stage: test -# os: windows # Windows 10.0.17134 N/A Build 17134 -# language: shell # 'language: python' is an error on Travis CI Windows -# - name: "3.5 on Linux" -# language: python -# python: 3.5 -# - name: "3.6 on Linux" -# language: python -# python: 3.6 -# - name: "3.7 on Linux" -# language: python -# python: 3.7 -# - name: "3.7 on Mac" -# os: osx -# osx_image: xcode11.3 + - name: "3.7 on Windows" + stage: test + os: windows + language: shell + - name: "3.5 on Linux" + language: python + python: 3.5 + - name: "3.6 on Linux" + language: python + python: 3.6 + - name: "3.7 on Linux" + language: python + python: 3.7 + - name: "3.7 on Mac" + os: osx + osx_image: xcode11.3 # build jobs - name: "Pypi pkgs" if: tag is present diff --git a/pydrive2/auth.py b/pydrive2/auth.py index c613ef7b..d4948cd3 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -52,7 +52,7 @@ def _decorated(self, *args, **kwargs): self.auth = GoogleAuth() # Re-create access token if it expired. if self.auth.access_token_expired: - if getattr(self, 'auth_method', False) == 'service': + if getattr(self.auth, 'auth_method', False) == 'service': self.auth.ServiceAuth() else: self.auth.LocalWebserverAuth() @@ -93,14 +93,9 @@ def _decorated(self, *args, **kwargs): decoratee(self, *args, **kwargs) self.Authorize() dirty = True - else: - if self.access_token_expired: - if self.credentials.refresh_token is not None: - self.Refresh() - else: - decoratee(self, *args, **kwargs) - self.Authorize() - dirty = True + elif self.access_token_expired: + self.Refresh() + dirty = True if dirty and save_credentials: self.SaveCredentials() return _decorated @@ -472,7 +467,7 @@ def Refresh(self): """ if self.credentials is None: raise RefreshError('No credential to refresh.') - if self.credentials.refresh_token is None: + if self.credentials.refresh_token is None and self.auth_method != 'service': raise RefreshError('No refresh_token found.' 'Please set access_type of OAuth to offline.') if self.http is None: @@ -521,10 +516,11 @@ def Authorize(self): :raises: AuthenticationError """ - if self.http is None: - self.http = httplib2.Http(timeout=self.http_timeout) if self.access_token_expired: raise AuthenticationError('No valid credentials provided to authorize') + + if self.http is None: + self.http = httplib2.Http(timeout=self.http_timeout) self.http = self.credentials.authorize(self.http) self.service = build('drive', 'v2', http=self.http, cache_discovery=False) diff --git a/pydrive2/files.py b/pydrive2/files.py index 06cd4c0a..dff8eb17 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -560,7 +560,7 @@ def _DeletePermission(self, permission_id): if 'permissions' in self and 'permissions' in self.metadata: permissions = self['permissions'] is_not_current_permission = lambda per: per['id'] == permission_id - permissions = filter(is_not_current_permission, permissions) + permissions = list(filter(is_not_current_permission, permissions)) self['permissions'] = permissions self.metadata['permissions'] = permissions return True diff --git a/pydrive2/test/settings/default.yaml b/pydrive2/test/settings/default.yaml index 0697513f..712cdb63 100644 --- a/pydrive2/test/settings/default.yaml +++ b/pydrive2/test/settings/default.yaml @@ -1,9 +1,12 @@ -client_config_backend: 'file' -client_config_file: 'configs/client_secrets.json' +client_config_backend: file +service_config: + client_service_email: your-service-account-email + client_pkcs12_file_path: your-file-path.p12 + client_user_email: save_credentials: True -save_credentials_backend: 'file' -save_credentials_file: 'credentials/1.dat' +save_credentials_backend: file +save_credentials_file: credentials/default.dat oauth_scope: - - 'https://www.googleapis.com/auth/drive' + - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/settings/test1.yaml b/pydrive2/test/settings/test_oauth_default.yaml similarity index 100% rename from pydrive2/test/settings/test1.yaml rename to pydrive2/test/settings/test_oauth_default.yaml diff --git a/pydrive2/test/settings/test_oauth_test_01.yaml b/pydrive2/test/settings/test_oauth_test_01.yaml new file mode 100644 index 00000000..0697513f --- /dev/null +++ b/pydrive2/test/settings/test_oauth_test_01.yaml @@ -0,0 +1,9 @@ +client_config_backend: 'file' +client_config_file: 'configs/client_secrets.json' + +save_credentials: True +save_credentials_backend: 'file' +save_credentials_file: 'credentials/1.dat' + +oauth_scope: + - 'https://www.googleapis.com/auth/drive' diff --git a/pydrive2/test/settings/test2.yaml b/pydrive2/test/settings/test_oauth_test_02.yaml similarity index 100% rename from pydrive2/test/settings/test2.yaml rename to pydrive2/test/settings/test_oauth_test_02.yaml diff --git a/pydrive2/test/settings/test3.yaml b/pydrive2/test/settings/test_oauth_test_03.yaml similarity index 100% rename from pydrive2/test/settings/test3.yaml rename to pydrive2/test/settings/test_oauth_test_03.yaml diff --git a/pydrive2/test/settings/test4.yaml b/pydrive2/test/settings/test_oauth_test_04.yaml similarity index 100% rename from pydrive2/test/settings/test4.yaml rename to pydrive2/test/settings/test_oauth_test_04.yaml diff --git a/pydrive2/test/settings/test5.yaml b/pydrive2/test/settings/test_oauth_test_05.yaml similarity index 100% rename from pydrive2/test/settings/test5.yaml rename to pydrive2/test/settings/test_oauth_test_05.yaml diff --git a/pydrive2/test/settings/test6.yaml b/pydrive2/test/settings/test_oauth_test_06.yaml similarity index 87% rename from pydrive2/test/settings/test6.yaml rename to pydrive2/test/settings/test_oauth_test_06.yaml index 19d55bbb..6d919958 100644 --- a/pydrive2/test/settings/test6.yaml +++ b/pydrive2/test/settings/test_oauth_test_06.yaml @@ -6,7 +6,7 @@ service_config: save_credentials: True save_credentials_backend: 'file' -save_credentials_file: 'credentials/5.dat' +save_credentials_file: 'credentials/6.dat' oauth_scope: - "https://www.googleapis.com/auth/drive" diff --git a/pydrive2/test/test_apiattr.py b/pydrive2/test/test_apiattr.py index a1632f00..5950ada4 100644 --- a/pydrive2/test/test_apiattr.py +++ b/pydrive2/test/test_apiattr.py @@ -1,21 +1,23 @@ import unittest +import pytest from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive +from pydrive2.test.test_util import pydrive_retry, setup_credentials + class ApiAttributeTest(unittest.TestCase): """Test ApiAttr functions. """ - ga = GoogleAuth('settings/test1.yaml') - ga.LocalWebserverAuth() - first_file = 'a.png' - second_file = 'b.png' + + @classmethod + def setup_class(cls): + setup_credentials() def test_UpdateMetadataNotInfinitelyNesting(self): # Verify 'metadata' field present. self.assertTrue(self.file1.metadata is not None) - self.file1.UpdateMetadata() - self.file1.UpdateMetadata() + pydrive_retry(self.file1.UpdateMetadata) # Verify 'metadata' field still present. self.assertTrue(self.file1.metadata is not None) @@ -23,12 +25,14 @@ def test_UpdateMetadataNotInfinitelyNesting(self): self.assertTrue('metadata' not in self.file1.metadata) def setUp(self): - self.drive = GoogleDrive(self.ga) + ga = GoogleAuth('pydrive2/test/settings/default.yaml') + ga.ServiceAuth() + self.drive = GoogleDrive(ga) self.file1 = self.drive.CreateFile() - self.file1.Upload() + pydrive_retry(self.file1.Upload) def tearDown(self): - self.file1.Delete() + pydrive_retry(self.file1.Delete) if __name__ == '__main__': unittest.main() diff --git a/pydrive2/test/test_drive.py b/pydrive2/test/test_drive.py index 918ec41c..adc45abc 100644 --- a/pydrive2/test/test_drive.py +++ b/pydrive2/test/test_drive.py @@ -1,21 +1,28 @@ # -*- coding: utf-8 -*- import unittest +import pytest from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive +from pydrive2.test.test_util import pydrive_retry, setup_credentials class GoogleDriveTest(unittest.TestCase): """Tests basic operations on meta-data information of the linked Google Drive account. """ - ga = GoogleAuth('settings/test1.yaml') - ga.LocalWebserverAuth() + @classmethod + def setup_class(cls): + setup_credentials() + + cls.ga = GoogleAuth('pydrive2/test/settings/default.yaml') + cls.ga.ServiceAuth() + def test_01_About_Request(self): drive = GoogleDrive(self.ga) - about_object = drive.GetAbout() + about_object = pydrive_retry(drive.GetAbout) self.assertTrue(about_object is not None, "About object not loading.") diff --git a/pydrive2/test/test_file.py b/pydrive2/test/test_file.py index 8ca379e5..8a3e722f 100644 --- a/pydrive2/test/test_file.py +++ b/pydrive2/test/test_file.py @@ -2,6 +2,8 @@ import filecmp import os import unittest +import pytest +import sys from io import BytesIO from six.moves import range @@ -11,8 +13,8 @@ from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive from pydrive2.files import ApiRequestError, GoogleDriveFile - -from . import test_util +from pydrive2.test import test_util +from pydrive2.test.test_util import pydrive_retry, setup_credentials, create_file, delete_file class GoogleDriveFileTest(unittest.TestCase): @@ -21,17 +23,33 @@ class GoogleDriveFileTest(unittest.TestCase): Equivalent to Files.insert, Files.update, Files.patch in Google Drive API. """ - ga = GoogleAuth('settings/test1.yaml') - ga.LocalWebserverAuth() - first_file = 'a.png' - second_file = 'b.png' + first_file = 'first_file' + second_file = 'second_file' + + + @classmethod + def setup_class(cls): + setup_credentials() + + create_file(cls.first_file, cls.first_file) + create_file(cls.second_file, cls.second_file) + + cls.ga = GoogleAuth('pydrive2/test/settings/default.yaml') + cls.ga.ServiceAuth() + + + @classmethod + def tearDownClass(cls): + delete_file(cls.first_file) + delete_file(cls.second_file) + def test_01_Files_Insert(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() filename = 'firsttestfile' file1['title'] = filename - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) file2 = drive.CreateFile({'id': file1['id']}) # Download file from id. @@ -39,12 +57,13 @@ def test_01_Files_Insert(self): self.DeleteUploadedFiles(drive, [file1['id']]) + def test_02_Files_Insert_Unicode(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() filename = u'첫번째 파일' file1['title'] = filename - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) file2 = drive.CreateFile({'id': file1['id']}) # Download file from id. @@ -52,6 +71,7 @@ def test_02_Files_Insert_Unicode(self): self.DeleteUploadedFiles(drive, [file1['id']]) + def test_03_Files_Insert_Content_String(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() @@ -59,20 +79,22 @@ def test_03_Files_Insert_Content_String(self): content = 'hello world!' file1['title'] = filename file1.SetContentString(content) - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.GetContentString(), content) - file1.FetchContent() # Force download and double check content + pydrive_retry(file1.FetchContent) # Force download and double check content self.assertEqual(file1.metadata['title'], filename) self.assertEqual(file1.GetContentString(), content) file2 = drive.CreateFile({'id': file1['id']}) # Download file from id. + pydrive_retry(file2.FetchContent) self.assertEqual(file2.GetContentString(), content) self.assertEqual(file2.metadata['title'], filename) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_04_Files_Insert_Content_Unicode_String(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() @@ -80,39 +102,44 @@ def test_04_Files_Insert_Content_Unicode_String(self): content = u'안녕 세상아!' file1['title'] = filename file1.SetContentString(content) - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.GetContentString(), content) self.assertEqual(file1.metadata['title'], filename) - file1.FetchContent() # Force download and double check content. + pydrive_retry(file1.FetchContent) # Force download and double check content. self.assertEqual(file1.GetContentString(), content) file2 = drive.CreateFile({'id': file1['id']}) # Download file from id. + pydrive_retry(file2.FetchContent) self.assertEqual(file2.GetContentString(), content) self.assertEqual(file2.metadata['title'], filename) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_05_Files_Insert_Content_File(self): - self.DeleteOldFile(self.first_file+'1') - self.DeleteOldFile(self.first_file+'2') + delete_file(self.first_file+'1') + delete_file(self.first_file+'2') drive = GoogleDrive(self.ga) file1 = drive.CreateFile() filename = 'filecontent' file1['title'] = filename file1.SetContentFile(self.first_file) - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) - file1.FetchContent() # Force download and double check content. + pydrive_retry(file1.FetchContent) # Force download and double check content. file1.GetContentFile(self.first_file+'1') self.assertEqual(filecmp.cmp(self.first_file, self.first_file+'1'), True) file2 = drive.CreateFile({'id': file1['id']}) # Download file from id. - file2.GetContentFile(self.first_file+'2') + pydrive_retry(lambda: file2.GetContentFile(self.first_file+'2')) self.assertEqual(filecmp.cmp(self.first_file, self.first_file+'2'), True) self.DeleteUploadedFiles(drive, [file1['id']]) + delete_file(self.first_file + '1') + delete_file(self.first_file + '2') + def test_06_Files_Patch(self): drive = GoogleDrive(self.ga) @@ -120,19 +147,20 @@ def test_06_Files_Patch(self): filename = 'prepatchtestfile' newfilename = 'patchtestfile' file1['title'] = filename - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) file1['title'] = newfilename - file1.Upload() # Files.patch + pydrive_retry(file1.Upload) # Files.patch self.assertEqual(file1.metadata['title'], newfilename) file2 = drive.CreateFile({'id': file1['id']}) # Download file from id. - file2.FetchMetadata() + pydrive_retry(file2.FetchMetadata) self.assertEqual(file2.metadata['title'], newfilename) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_07_Files_Patch_Skipping_Content(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() @@ -142,17 +170,18 @@ def test_07_Files_Patch_Skipping_Content(self): file1['title'] = filename file1.SetContentString(content) - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) file1['title'] = newfilename - file1.Upload() # Files.patch + pydrive_retry(file1.Upload) # Files.patch self.assertEqual(file1.metadata['title'], newfilename) self.assertEqual(file1.GetContentString(), content) self.assertEqual(file1.GetContentString(), content) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_08_Files_Update_String(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() @@ -163,25 +192,26 @@ def test_08_Files_Update_String(self): file1['title'] = filename file1.SetContentString(content) - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) self.assertEqual(file1.GetContentString(), content) - file1.FetchContent() # Force download and double check content. + pydrive_retry(file1.FetchContent) # Force download and double check content. self.assertEqual(file1.GetContentString(), content) file1['title'] = newfilename file1.SetContentString(newcontent) - file1.Upload() # Files.update + pydrive_retry(file1.Upload) # Files.update self.assertEqual(file1.metadata['title'], newfilename) self.assertEqual(file1.GetContentString(), newcontent) self.assertEqual(file1.GetContentString(), newcontent) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_09_Files_Update_File(self): - self.DeleteOldFile(self.first_file+'1') - self.DeleteOldFile(self.second_file+'1') + delete_file(self.first_file+'1') + delete_file(self.second_file+'1') drive = GoogleDrive(self.ga) file1 = drive.CreateFile() filename = 'preupdatetestfile' @@ -189,121 +219,130 @@ def test_09_Files_Update_File(self): file1['title'] = filename file1.SetContentFile(self.first_file) - file1.Upload() # Files.insert + pydrive_retry(file1.Upload) # Files.insert self.assertEqual(file1.metadata['title'], filename) - file1.FetchContent() # Force download and double check content. + pydrive_retry(file1.FetchContent) # Force download and double check content. file1.GetContentFile(self.first_file+'1') self.assertEqual(filecmp.cmp(self.first_file, self.first_file+'1'), True) file1['title'] = newfilename file1.SetContentFile(self.second_file) - file1.Upload() # Files.update + pydrive_retry(file1.Upload) # Files.update self.assertEqual(file1.metadata['title'], newfilename) file1.GetContentFile(self.second_file+'1') self.assertEqual(filecmp.cmp(self.second_file, self.second_file+'1'), True) self.DeleteUploadedFiles(drive, [file1['id']]) + delete_file(self.first_file + '1') + delete_file(self.second_file + '1') + # Tests for Trash/UnTrash/Delete. # =============================== + def test_Files_Trash_File(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) self.assertFalse(file1.metadata[u'labels'][u'trashed']) # Download to verify non-trashed state on GDrive. file2 = drive.CreateFile({'id': file1['id']}) - file2.FetchMetadata() + pydrive_retry(file2.FetchMetadata) self.assertFalse(file2.metadata[u'labels'][u'trashed']) - file1.Trash() + pydrive_retry(file1.Trash) self.assertTrue(file1.metadata[u'labels'][u'trashed']) - file2.FetchMetadata() + pydrive_retry(file2.FetchMetadata) self.assertTrue(file2.metadata[u'labels'][u'trashed']) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_Files_Trash_File_Just_ID(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) self.assertFalse(file1.metadata[u'labels'][u'trashed']) # Trash file by ID. file2 = drive.CreateFile({'id': file1['id']}) - file2.Trash() + pydrive_retry(file2.Trash) # Verify trashed by downloading metadata. - file1.FetchMetadata() + pydrive_retry(file1.FetchMetadata) self.assertTrue(file1.metadata[u'labels'][u'trashed']) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_Files_UnTrash_File(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() - file1.Trash() + pydrive_retry(file1.Upload) + pydrive_retry(file1.Trash) self.assertTrue(file1.metadata[u'labels'][u'trashed']) # Verify that file is trashed by downloading metadata. file2 = drive.CreateFile({'id': file1['id']}) - file2.FetchMetadata() + pydrive_retry(file2.FetchMetadata) self.assertTrue(file2.metadata[u'labels'][u'trashed']) # Un-trash the file, and assert local metadata is updated correctly. - file1.UnTrash() + pydrive_retry(file1.UnTrash) self.assertFalse(file1.metadata[u'labels'][u'trashed']) # Re-fetch the metadata, and assert file un-trashed on GDrive. - file2.FetchMetadata() + pydrive_retry(file2.FetchMetadata) self.assertFalse(file2.metadata[u'labels'][u'trashed']) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_Files_UnTrash_File_Just_ID(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() - file1.Trash() + pydrive_retry(file1.Upload) + pydrive_retry(file1.Trash) self.assertTrue(file1.metadata[u'labels'][u'trashed']) file2 = drive.CreateFile({'id': file1['id']}) - file2.UnTrash() # UnTrash without fetching metadata. + pydrive_retry(file2.UnTrash) # UnTrash without fetching metadata. - file1.FetchMetadata() + pydrive_retry(file1.FetchMetadata) self.assertFalse(file1.metadata[u'labels'][u'trashed']) self.DeleteUploadedFiles(drive, [file1['id']]) + def test_Files_Delete_File(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) file2 = drive.CreateFile({'id': file1['id']}) - file1.Delete() + pydrive_retry(file1.Delete) try: - file2.FetchMetadata() + pydrive_retry(file2.FetchMetadata) self.fail("File not deleted correctly.") except ApiRequestError as e: pass + def test_Files_Delete_File_Just_ID(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) file2 = drive.CreateFile({'id': file1['id']}) - file2.Delete() + pydrive_retry(file2.Delete) try: - file1.FetchMetadata() + pydrive_retry(file1.FetchMetadata) self.fail("File not deleted correctly.") except ApiRequestError as e: pass @@ -311,94 +350,100 @@ def test_Files_Delete_File_Just_ID(self): # Tests for Permissions. # ====================== + def test_Files_FetchMetadata_Fields(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) self.assertFalse('permissions' in file1) - file1.FetchMetadata('permissions') + pydrive_retry(lambda: file1.FetchMetadata('permissions')) self.assertTrue('permissions' in file1) - file1.Delete() + pydrive_retry(file1.Delete) + def test_Files_Insert_Permission(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) # Verify only one permission before inserting permission. - permissions = file1.GetPermissions() + permissions = pydrive_retry(file1.GetPermissions) self.assertEqual(len(permissions), 1) self.assertEqual(len(file1['permissions']), 1) # Insert the permission. - permission = file1.InsertPermission({'type': 'anyone', + permission = pydrive_retry(lambda: file1.InsertPermission({'type': 'anyone', 'value': 'anyone', - 'role': 'reader'}) + 'role': 'reader'})) self.assertTrue(permission) self.assertEqual(len(file1["permissions"]), 2) - self.assertEqual(file1["permissions"][1]["type"], "anyone") + self.assertEqual(file1["permissions"][0]["type"], "anyone") - permissions = file1.GetPermissions() + permissions = pydrive_retry(file1.GetPermissions) self.assertEqual(len(file1["permissions"]), 2) - self.assertEqual(file1["permissions"][1]["type"], "anyone") - self.assertEqual(permissions[1]["type"], "anyone") + self.assertEqual(file1["permissions"][0]["type"], "anyone") + self.assertEqual(permissions[0]["type"], "anyone") # Verify remote changes made. file2 = drive.CreateFile({'id': file1['id']}) - permissions = file2.GetPermissions() + permissions = pydrive_retry(file2.GetPermissions) self.assertEqual(len(permissions), 2) - self.assertEqual(permissions[1]["type"], "anyone") + self.assertEqual(permissions[0]["type"], "anyone") + + pydrive_retry(file1.Delete) - file1.Delete() def test_Files_Get_Permissions(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) self.assertFalse('permissions' in file1) - permissions = file1.GetPermissions() + permissions = pydrive_retry(file1.GetPermissions) self.assertTrue(permissions is not None) self.assertTrue('permissions' in file1) - file1.Delete() + pydrive_retry(file1.Delete) + def test_Files_Delete_Permission(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() - file1.InsertPermission({'type': 'anyone', + pydrive_retry(file1.Upload) + pydrive_retry(lambda: file1.InsertPermission({'type': 'anyone', 'value': 'anyone', - 'role': 'reader'}) - permissions = file1.GetPermissions() + 'role': 'reader'})) + permissions = pydrive_retry(file1.GetPermissions) self.assertEqual(len(permissions), 2) self.assertEqual(len(file1['permissions']), 2) - file1.DeletePermission(permissions[1]['id']) + pydrive_retry(lambda: file1.DeletePermission(permissions[0]['id'])) self.assertEqual(len(file1['permissions']), 1) # Verify remote changes made. file2 = drive.CreateFile({'id': file1['id']}) - permissions = file2.GetPermissions() + permissions = pydrive_retry(file2.GetPermissions) self.assertEqual(len(permissions), 1) - file1.Delete() + pydrive_retry(file1.Delete) + def test_Files_Delete_Permission_Invalid(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() - file1.Upload() + pydrive_retry(file1.Upload) try: - file1.DeletePermission('invalid id') + pydrive_retry(lambda: file1.DeletePermission('invalid id')) self.fail("Deleting invalid permission not raising exception.") except ApiRequestError as e: pass - file1.Delete() + pydrive_retry(file1.Delete) + def test_GFile_Conversion_Lossless_String(self): drive = GoogleDrive(self.ga) @@ -407,7 +452,7 @@ def test_GFile_Conversion_Lossless_String(self): # Upload a string, and convert into Google Doc format. test_string = 'Generic, non-exhaustive ASCII test string.' file1.SetContentString(test_string) - file1.Upload({'convert': True}) + pydrive_retry(lambda: file1.Upload({'convert': True})) # Download string as plain text. downloaded_string = file1.GetContentString(mimetype='text/plain') @@ -416,16 +461,17 @@ def test_GFile_Conversion_Lossless_String(self): # Download content into file and ensure that file content matches original # content string. downloaded_file_name = '_tmp_downloaded_file_name.txt' - file1.GetContentFile(downloaded_file_name, mimetype='text/plain') + pydrive_retry(lambda: file1.GetContentFile(downloaded_file_name, mimetype='text/plain')) downloaded_string = open(downloaded_file_name).read() self.assertEqual(test_string, downloaded_string, "Strings do not match") # Delete temp file. - os.remove(downloaded_file_name) + delete_file(downloaded_file_name) # Tests for GDrive conversion. # ============================ + def setup_gfile_conversion_test(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() @@ -441,12 +487,14 @@ def setup_gfile_conversion_test(self): return file1, file_name, original_file_content, downloaded_file_name + def cleanup_gfile_conversion_test(self, file1, file_name, downloaded_file_name): # Delete temporary files. os.path.exists(file_name) and os.remove(file_name) os.path.exists(downloaded_file_name) and os.remove(downloaded_file_name) - file1.Delete() # Delete uploaded file. + pydrive_retry(file1.Delete) # Delete uploaded file. + def test_GFile_Conversion_Remove_BOM(self): (file1, file_name, original_file_content, downloaded_file_name) = \ @@ -454,7 +502,7 @@ def test_GFile_Conversion_Remove_BOM(self): try: # Upload source_file and convert into Google Doc format. file1.SetContentFile(file_name) - file1.Upload({'convert': True}) + pydrive_retry(lambda: file1.Upload({'convert': True})) # Download as string. downloaded_content_no_bom = file1.GetContentString(mimetype='text/plain', @@ -464,7 +512,7 @@ def test_GFile_Conversion_Remove_BOM(self): self.assertEqual(original_file_content, downloaded_content_no_bom) # Download as file. - file1.GetContentFile(downloaded_file_name, remove_bom=True) + pydrive_retry(lambda: file1.GetContentFile(downloaded_file_name, remove_bom=True)) downloaded_content = open(downloaded_file_name).read() downloaded_content = test_util.StripNewlines(downloaded_content) self.assertEqual(original_file_content, downloaded_content) @@ -472,6 +520,7 @@ def test_GFile_Conversion_Remove_BOM(self): finally: self.cleanup_gfile_conversion_test(file1, file_name, downloaded_file_name) + def test_Gfile_Conversion_Add_Remove_BOM(self): """Tests whether you can switch between the BOM appended and removed version on the fly.""" @@ -479,7 +528,7 @@ def test_Gfile_Conversion_Add_Remove_BOM(self): self.setup_gfile_conversion_test() try: file1.SetContentFile(file_name) - file1.Upload({'convert': True}) + pydrive_retry(lambda: file1.Upload({'convert': True})) content_bom = file1.GetContentString(mimetype='text/plain') content_no_bom = file1.GetContentString(mimetype='text/plain', @@ -493,9 +542,10 @@ def test_Gfile_Conversion_Add_Remove_BOM(self): finally: self.cleanup_gfile_conversion_test(file1, file_name, downloaded_file_name) + def test_InsertPrefix(self): # Create BytesIO. - file_obj = BytesIO('abc') + file_obj = BytesIO('abc'.encode('utf8')) original_length = len(file_obj.getvalue()) char_to_insert = u'\ufeff'.encode('utf8') @@ -505,10 +555,11 @@ def test_InsertPrefix(self): self.assertGreater(modified_length, original_length) self.assertEqual(file_obj.getvalue(), u'\ufeffabc'.encode('utf8')) + def test_InsertPrefixLarge(self): # Create BytesIO. test_content = 'abc' * 800 - file_obj = BytesIO(test_content) + file_obj = BytesIO(test_content.encode('utf-8')) original_length = len(file_obj.getvalue()) char_to_insert = u'\ufeff'.encode('utf8') @@ -519,6 +570,7 @@ def test_InsertPrefixLarge(self): expected_content = u'\ufeff' + test_content self.assertEqual(file_obj.getvalue(), expected_content.encode('utf8')) + def test_RemovePrefix(self): # Create BytesIO. file_obj = BytesIO(u'\ufeffabc'.encode('utf8')) @@ -529,7 +581,8 @@ def test_RemovePrefix(self): GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) modified_length = len(file_obj.getvalue()) self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), 'abc') + self.assertEqual(file_obj.getvalue(), 'abc'.encode('utf8')) + def test_RemovePrefixLarge(self): # Create BytesIO. @@ -542,7 +595,7 @@ def test_RemovePrefixLarge(self): GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) modified_length = len(file_obj.getvalue()) self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), test_content[1:]) + self.assertEqual(file_obj.getvalue(), test_content[1:].encode('utf8')) # Setup for concurrent upload testing. # ===================================== @@ -553,8 +606,11 @@ def __init__(self, gdrive_file, generate_http=False): if generate_http: self.param = {"http": gdrive_file.auth.Get_Http_Object()} + def run(self): - self.gdrive_file.Upload(param=self.param) + pydrive_retry(lambda: self.gdrive_file.Upload(param=self.param)) + + FILE_UPLOAD_COUNT = 10 def _parallel_uploader(self, num_of_uploads, num_of_workers, use_per_thread_http=False): @@ -572,8 +628,10 @@ def _parallel_uploader(self, num_of_uploads, num_of_workers, upload_files.append(up_file) # Ensure there are no files with the random file name. - files = drive.ListFile(param={'q': "title = '%s' and trashed = false" % + files = pydrive_retry( + lambda: drive.ListFile(param={'q': "title = '%s' and trashed = false" % remote_name}).GetList() + ) self.assertTrue(len(files) == 0) # Submit upload jobs to ThreadPoolExecutor. @@ -588,35 +646,34 @@ def _parallel_uploader(self, num_of_uploads, num_of_workers, self.assertIsNone(future.exception()) thread_pool.shutdown() - # Ensure 10 files were uploaded. - files = drive.ListFile(param={'q': "title = '%s' and trashed = false" % + # Ensure all files were uploaded. + files = pydrive_retry( + lambda: drive.ListFile(param={'q': "title = '%s' and trashed = false" % remote_name}).GetList() - self.assertTrue(len(files) == 10) + ) + self.assertTrue(len(files) == self.FILE_UPLOAD_COUNT) # Remove uploaded files. self.DeleteUploadedFiles(drive, [fi['id'] for fi in upload_files]) - @timeout_decorator.timeout(80) + @pytest.mark.skipif(sys.platform == 'win32', reason="timeout_decorator doesn't support Windows") + @timeout_decorator.timeout(160) def test_Parallel_Files_Insert_File_Auto_Generated_HTTP(self): - self._parallel_uploader(10, 10) + self._parallel_uploader(self.FILE_UPLOAD_COUNT, 10) - @timeout_decorator.timeout(80) + @pytest.mark.skipif(sys.platform == 'win32', reason="timeout_decorator doesn't support Windows") + @timeout_decorator.timeout(160) def test_Parallel_Insert_File_Passed_HTTP(self): - self._parallel_uploader(10, 10, True) + self._parallel_uploader(self.FILE_UPLOAD_COUNT, 10, True) # Helper functions. # ================= - def DeleteOldFile(self, file_name): - try: - os.remove(file_name) - except OSError: - pass def DeleteUploadedFiles(self, drive, ids): for element in ids: tmp_file = drive.CreateFile({'id': element}) - tmp_file.Delete() + pydrive_retry(tmp_file.Delete) if __name__ == '__main__': diff --git a/pydrive2/test/test_filelist.py b/pydrive2/test/test_filelist.py index 6d836059..4426163a 100644 --- a/pydrive2/test/test_filelist.py +++ b/pydrive2/test/test_filelist.py @@ -2,40 +2,45 @@ import os import sys import unittest +import pytest from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive from pydrive2.test import test_util +from pydrive2.test.test_util import pydrive_retry, pydrive_list_item, setup_credentials class GoogleDriveFileListTest(unittest.TestCase): """Tests operations of files.GoogleDriveFileList class. Equivalent to Files.list in Google Drive API. """ - ga = GoogleAuth('settings/test1.yaml') - ga.LocalWebserverAuth() - drive = GoogleDrive(ga) + + @classmethod + def setup_class(cls): + setup_credentials() + + cls.ga = GoogleAuth('pydrive2/test/settings/default.yaml') + cls.ga.ServiceAuth() + cls.drive = GoogleDrive(cls.ga) + def test_01_Files_List_GetList(self): drive = GoogleDrive(self.ga) - flist = drive.ListFile({'q': "title = '%s' and trashed = false" - % self.title}) - files = flist.GetList() # Auto iterate every file - for file1 in self.file_list: + query = "title = '{}' and trashed = false".format(self.title) + for file1 in pydrive_list_item(drive, query): found = False - for file2 in files: + for file2 in pydrive_list_item(drive, query): if file1['id'] == file2['id']: found = True self.assertEqual(found, True) + def test_02_Files_List_ForLoop(self): drive = GoogleDrive(self.ga) - flist = drive.ListFile({'q': "title = '%s' and trashed = false"%self.title, - 'maxResults': 2}) + query = "title = '{}' and trashed = false".format(self.title) files = [] - for x in flist: # Build iterator to access files simply with for loop - self.assertTrue(len(x) <= 2) - files.extend(x) + for x in pydrive_list_item(drive, query, 2): # Build iterator to access files simply with for loop + files.append(x) for file1 in self.file_list: found = False for file2 in files: @@ -43,6 +48,7 @@ def test_02_Files_List_ForLoop(self): found = True self.assertEqual(found, True) + def test_03_Files_List_GetList_Iterate(self): drive = GoogleDrive(self.ga) flist = drive.ListFile({'q': "title = '%s' and trashed = false"%self.title, @@ -50,7 +56,7 @@ def test_03_Files_List_GetList_Iterate(self): files = [] while True: try: - x = flist.GetList() + x = pydrive_retry(flist.GetList) self.assertTrue(len(x) <= 2) files.extend(x) except StopIteration: @@ -62,21 +68,19 @@ def test_03_Files_List_GetList_Iterate(self): found = True self.assertEqual(found, True) + def test_File_List_Folders(self): drive = GoogleDrive(self.ga) folder1 = drive.CreateFile( {'mimeType': 'application/vnd.google-apps.folder', 'title': self.title}) - folder1.Upload() + pydrive_retry(folder1.Upload) self.file_list.append(folder1) - - flist = drive.ListFile({'q': "title = '%s' and trashed = false" - % self.title}) + query = "title = '{}' and trashed = false".format(self.title) count = 0 - for _flist in flist: - for file1 in _flist: - self.assertFileInFileList(file1) - count += 1 + for file1 in pydrive_list_item(drive, query): + self.assertFileInFileList(file1) + count += 1 self.assertTrue(count == 11) @@ -88,7 +92,7 @@ def setUp(self): for x in range(0, 10): file1 = self.drive.CreateFile() file1['title'] = title - file1.Upload() + pydrive_retry(file1.Upload) file_list.append(file1) self.title = title @@ -97,7 +101,7 @@ def setUp(self): def tearDown(self): # Deleting uploaded files. for file1 in self.file_list: - file1.Delete() + pydrive_retry(file1.Delete) def assertFileInFileList(self, file_object): found = False diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index fb9fed6e..d6b8ae9d 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -1,71 +1,80 @@ import unittest import os import time +import pytest from pydrive2.auth import GoogleAuth +from pydrive2.test.test_util import setup_credentials, delete_file + class GoogleAuthTest(unittest.TestCase): """Tests basic OAuth2 operations of auth.GoogleAuth.""" + @pytest.mark.manual def test_01_LocalWebserverAuthWithClientConfigFromFile(self): # Delete old credentials file - self.DeleteOldCredentialsFile('credentials/1.dat') + delete_file('credentials/1.dat') # Test if authentication works with config read from file - ga = GoogleAuth('settings/test1.yaml') + ga = GoogleAuth('pydrive2/test/settings/test_oauth_test_01.yaml') ga.LocalWebserverAuth() self.assertEqual(ga.access_token_expired, False) # Test if correct credentials file is created self.CheckCredentialsFile('credentials/1.dat') time.sleep(1) + @pytest.mark.manual def test_02_LocalWebserverAuthWithClientConfigFromSettings(self): # Delete old credentials file - self.DeleteOldCredentialsFile('credentials/2.dat') + delete_file('credentials/2.dat') # Test if authentication works with config read from settings - ga = GoogleAuth('settings/test2.yaml') + ga = GoogleAuth('pydrive2/test/settings/test_oauth_test_02.yaml') ga.LocalWebserverAuth() self.assertEqual(ga.access_token_expired, False) # Test if correct credentials file is created self.CheckCredentialsFile('credentials/2.dat') time.sleep(1) + @pytest.mark.manual def test_03_LocalWebServerAuthWithNoCredentialsSaving(self): # Delete old credentials file - self.DeleteOldCredentialsFile('credentials/4.dat') + delete_file('credentials/4.dat') # Provide wrong credentials file - ga = GoogleAuth('settings/test3.yaml') + ga = GoogleAuth('pydrive2/test/settings/test_oauth_test_03.yaml') ga.LocalWebserverAuth() self.assertEqual(ga.access_token_expired, False) # Test if correct credentials file is created self.CheckCredentialsFile('credentials/4.dat', no_file=True) time.sleep(1) + @pytest.mark.manual def test_04_CommandLineAuthWithClientConfigFromFile(self): # Delete old credentials file - self.DeleteOldCredentialsFile('credentials/1.dat') + delete_file('credentials/1.dat') # Test if authentication works with config read from file - ga = GoogleAuth('settings/test4.yaml') + ga = GoogleAuth('pydrive2/test/settings/test_oauth_test_04.yaml') ga.CommandLineAuth() self.assertEqual(ga.access_token_expired, False) # Test if correct credentials file is created self.CheckCredentialsFile('credentials/1.dat') time.sleep(1) + @pytest.mark.manual def test_05_ConfigFromSettingsWithoutOauthScope(self): # Test if authentication works without oauth_scope - ga = GoogleAuth('settings/test5.yaml') + ga = GoogleAuth('pydrive2/test/settings/test_oauth_test_05.yaml') ga.LocalWebserverAuth() self.assertEqual(ga.access_token_expired, False) time.sleep(1) - def DeleteOldCredentialsFile(self, credentials): - try: - os.remove(credentials) - except OSError: - pass + def test_06_ServiceAuthFromSavedCredentialsFile(self): + setup_credentials("credentials/6.dat") + ga = GoogleAuth('pydrive2/test/settings/test_oauth_test_06.yaml') + ga.ServiceAuth() + self.assertEqual(ga.access_token_expired, False) + time.sleep(1) def CheckCredentialsFile(self, credentials, no_file=False): - ga = GoogleAuth('settings/default.yaml') + ga = GoogleAuth('pydrive2/test/settings/test_oauth_default.yaml') ga.LoadCredentialsFile(credentials) self.assertEqual(ga.access_token_expired, no_file) diff --git a/pydrive2/test/test_util.py b/pydrive2/test/test_util.py index 294d9188..abf0db0f 100644 --- a/pydrive2/test/test_util.py +++ b/pydrive2/test/test_util.py @@ -1,8 +1,61 @@ +import unittest import random import re +import os + +from funcy import retry +from funcy.py3 import cat +from pydrive2.files import ApiRequestError newline_pattern = re.compile(r'[\r\n]') +GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA" +DEFAULT_USER_CREDENTIALS_FILE = "credentials/default.dat" + + +def setup_credentials(credentials_path=DEFAULT_USER_CREDENTIALS_FILE): + if os.getenv(GDRIVE_USER_CREDENTIALS_DATA): + if not os.path.exists(os.path.dirname(credentials_path)): + os.makedirs(os.path.dirname(credentials_path), exist_ok=True) + with open( + credentials_path, "w" + ) as credentials_file: + credentials_file.write( + os.getenv(GDRIVE_USER_CREDENTIALS_DATA) + ) + + +class PyDriveRetriableError(Exception): + pass + + +# 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s +@retry(15, PyDriveRetriableError, timeout=lambda a: min(0.5 * 1.618 ** a, 20)) +def pydrive_retry(call): + try: + result = call() + except ApiRequestError as exception: + retry_codes = ["403", "500", "502", "503", "504"] + if any( + "HttpError {}".format(code) in str(exception) + for code in retry_codes + ): + raise PyDriveRetriableError("Google API request failed") + raise + return result + + +def pydrive_list_item(drive, query, max_results=1000): + param = {"q": query, "maxResults": max_results} + + file_list = drive.ListFile(param) + + # Isolate and decorate fetching of remote drive items in pages + get_list = lambda: pydrive_retry(lambda: next(file_list, None)) + + # Fetch pages until None is received, lazily flatten the thing + return cat(iter(get_list, None)) + def CreateRandomFileName(): hash = random.getrandbits(128) @@ -10,4 +63,14 @@ def CreateRandomFileName(): def StripNewlines(string): - return newline_pattern.sub("", string) \ No newline at end of file + return newline_pattern.sub("", string) + + +def create_file(path, content): + with open(path, "w") as f: + f.write(content) + + +def delete_file(path): + if os.path.exists(path): + os.remove(path) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..197f7632 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + manual: mark tests to be runnable only in local environment and require user manual actions. diff --git a/scripts/ci/script.sh b/scripts/ci/script.sh index 5dfafbc6..025be26d 100644 --- a/scripts/ci/script.sh +++ b/scripts/ci/script.sh @@ -3,4 +3,4 @@ set -x set -e -py.test -v -s +py.test -v -s -m "not manual" diff --git a/setup.py b/setup.py index 394ad110..d500ffc7 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ "pyOpenSSL >= 19.1.0" ], extras_require={ - "tests": ["timeout-decorator"], + "tests": ["pytest>=4.6.0", "timeout-decorator", "funcy>=1.14"], }, )