From 1bd5c963ce0226dee958b83a01c9c5bf8f023c75 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Wed, 3 Jul 2019 10:43:07 -0700 Subject: [PATCH 01/19] Adds VPC SC integration tests for Video Intelligence API --- ...deo_intelligence_service_grpc_transport.py | 16 +- .../video_intelligence_service_client.py | 31 ++-- videointelligence/tests/system.py | 159 ++++++++++++++++++ 3 files changed, 189 insertions(+), 17 deletions(-) create mode 100644 videointelligence/tests/system.py diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py index 0c3a472179cf..9bf9f3bd5024 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py @@ -19,6 +19,7 @@ import google.api_core.operations_v1 from google.cloud.videointelligence_v1.proto import video_intelligence_pb2_grpc +from google.longrunning import operations_pb2 class VideoIntelligenceServiceGrpcTransport(object): @@ -71,7 +72,8 @@ def __init__( self._stubs = { "video_intelligence_service_stub": video_intelligence_pb2_grpc.VideoIntelligenceServiceStub( channel - ) + ), + "operations_stub": operations_pb2.OperationsStub(channel) } # Because this API includes a method that returns a @@ -113,6 +115,18 @@ def channel(self): """ return self._channel + @property + def get_operation(self): + return self._operations_client.get_operation + + @property + def delete_operation(self): + return self._operations_client.delete_operation + + @property + def cancel_operation(self): + return self._operations_client.cancel_operation + @property def annotate_video(self): """Return the gRPC stub for :meth:`VideoIntelligenceServiceClient.annotate_video`. diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py index 890992f9fb65..ff5fdec70e21 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py @@ -20,7 +20,6 @@ import warnings from google.oauth2 import service_account -import google.api_core.client_options import google.api_core.gapic_v1.client_info import google.api_core.gapic_v1.config import google.api_core.gapic_v1.method @@ -83,7 +82,6 @@ def __init__( credentials=None, client_config=None, client_info=None, - client_options=None, ): """Constructor. @@ -114,9 +112,6 @@ def __init__( API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. - client_options (Union[dict, google.api_core.client_options.ClientOptions]): - Client options used to set user options on the client. API Endpoint - should be set through client_options. """ # Raise deprecation warnings for things we want to go away. if client_config is not None: @@ -135,15 +130,6 @@ def __init__( stacklevel=2, ) - api_endpoint = self.SERVICE_ADDRESS - if client_options: - if type(client_options) == dict: - client_options = google.api_core.client_options.from_dict( - client_options - ) - if client_options.api_endpoint: - api_endpoint = client_options.api_endpoint - # Instantiate the transport. # The transport is responsible for handling serialization and # deserialization and actually sending data to the service. @@ -152,7 +138,6 @@ def __init__( self.transport = transport( credentials=credentials, default_class=video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport, - address=api_endpoint, ) else: if credentials: @@ -163,7 +148,7 @@ def __init__( self.transport = transport else: self.transport = video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport( - address=api_endpoint, channel=channel, credentials=credentials + address=self.SERVICE_ADDRESS, channel=channel, credentials=credentials ) if client_info is None: @@ -187,7 +172,19 @@ def __init__( # transport methods, wrapped with `wrap_method` to add retry, # timeout, and the like. self._inner_api_calls = {} + + def channel(self): + return self.transport.channel + def get_operation(self, name=None): + return self.transport.get_operation(name) + + def delete_operation(self, name=None): + return self.transport.delete_operation(name) + + def cancel_operation(self, name=None): + return self.transport.cancel_operation(name) + # Service calls def annotate_video( self, @@ -304,3 +301,5 @@ def annotate_video( video_intelligence_pb2.AnnotateVideoResponse, metadata_type=video_intelligence_pb2.AnnotateVideoProgress, ) + + diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py new file mode 100644 index 000000000000..ea58a97286bb --- /dev/null +++ b/videointelligence/tests/system.py @@ -0,0 +1,159 @@ +# Copyright 2017, Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""System tests for VideoIntelligence API.""" + +import grpc +import io +import json +import os +import requests +import time +import unittest + +import google.api_core.exceptions +from google.cloud import exceptions +from google.cloud import videointelligence +from google.cloud.videointelligence_v1 import enums + +PROJECT_NUMBER = os.environ.get("PROJECT_NUMBER") +PROJECT_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT") +BUCKET_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") +OUTSIDE_PROJECT_API_KEY = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY") +OUTSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP") +INSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP") + +class VideoIntelligenceSystemTestBase(unittest.TestCase): + client = None + + def setUp(self): + self.to_delete_by_case = [] + self.input_uri = "gs://cloud-samples-data/video/cat.mp4" + + def tearDown(self): + for value in self.to_delete_by_case: + value.delete() + + +def setUpModule(): + VideoIntelligenceSystemTestBase.client = videointelligence.VideoIntelligenceServiceClient() + + +class TestVideoIntelligenceClient(VideoIntelligenceSystemTestBase): + def test_annotate_video(self): + features_element = enums.Feature.LABEL_DETECTION + features = [features_element] + response = self.client.annotate_video(input_uri=self.input_uri, features=features) + + # Wait for the operation to complete. + lro_timeout_seconds = 60 + start_time = time.time() + cnt = 0 + while not response.done() and (time.time() - start_time) < lro_timeout_seconds: + time.sleep(1) + cnt += 1 + if not response.done(): + self.fail( + "wait for operation timed out after {lro_timeout_seconds} seconds".format( + lro_timeout_seconds=lro_timeout_seconds + ) + ) + + result = response.result() + annotations = result.annotation_results[0] + assert len(annotations.segment_label_annotations) > 0 + + + +@unittest.skipUnless( + OUTSIDE_PROJECT_API_KEY, + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY not set in environment.", +) +class TestVideoIntelligenceClientVpcSc(VideoIntelligenceSystemTestBase): + # Tests to verify VideoIntelligence service requests blocked when trying to access resources outside of a secure perimeter. + def setUp(self): + VideoIntelligenceSystemTestBase.setUp(self) + # api-endpoint + self.url = "https://videointelligence.googleapis.com/v1/videos:annotate?key={}".format(OUTSIDE_PROJECT_API_KEY) + self.body = {"input_uri": self.input_uri, "features": ["LABEL_DETECTION"], "location_id": "us-west1"} + + def test_get_operation_from_different_project(self): + # sending request and saving the response as response object + r = requests.post(url=self.url, data=self.body) + outside_project_operation = json.loads(r.text) + + # Uses inside perimeter project to access operation resource created owned by outside project + with self.assertRaises(google.api_core.exceptions.NotFound) as cm: + self.client.get_operation(outside_project_operation["name"]) + + # Assert it returns not found. + self.assertEqual(cm.exception.code, 404) + self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) + + def test_delete_operation_from_different_project(self): + # sending request and saving the response as response object + r = requests.post(url=self.url, data=self.body) + outside_project_operation = json.loads(r.text) + + # Uses inside perimeter project to access operation resource created owned by outside project + with self.assertRaises(Exception) as cm: + self.client.delete_operation(outside_project_operation["name"]) + + # Assert it returns not found. + self.assertEqual(cm.exception.code, 404) + self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) + + def test_cancel_operation_from_different_project(self): + # sending request and saving the response as response object + r = requests.post(url=self.url, data=self.body) + outside_project_operation = json.loads(r.text) + + # Uses inside perimeter project to access operation resource created owned by outside project + with self.assertRaises(Exception) as cm: + self.client.cancel_operation(outside_project_operation["name"]) + + # Assert it returns not found. + self.assertEqual(cm.exception.code, 404) + self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) + + @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") + @unittest.skipUnless(OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment.") + def test_outside_ip_address_blocked(self): + headers = { + "Content-Type": "application/json", + "X-User-IP": OUTSIDE_IP, + "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, + } + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + outside_project_operation = json.loads(r.text) + print(outside_project_operation) + # Assert it returns permission denied from VPC SC + self.assertEqual(outside_project_operation["error"]["code"], 403) + self.assertEqual(outside_project_operation["error"]["status"], "PERMISSION_DENIED") + self.assertEqual(outside_project_operation["error"]["details"][0]["violations"][0]["type"], "VPC_SERVICE_CONTROLS") + self.assertEqual(outside_project_operation["error"]["message"], "Request is prohibited by organization's policy") + + @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") + @unittest.skipUnless(INSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP not set in environment.") + def test_inside_ip_address_allowed(self): + headers = { + "Content-Type": "application/json", + "X-User-IP": INSIDE_IP, + "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, + } + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + operation = json.loads(r.text) + # Assert it returns non-empty operation name. + self.assertNotEqual(operation["name"], "") + From 9a49e5407df17660e91f47f250df974f853589dd Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Mon, 8 Jul 2019 09:21:03 -0700 Subject: [PATCH 02/19] Addressing comments --- .../video_intelligence_service_grpc_transport.py | 4 +--- .../gapic/video_intelligence_service_client.py | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py index 9bf9f3bd5024..7dc775491d11 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py @@ -19,7 +19,6 @@ import google.api_core.operations_v1 from google.cloud.videointelligence_v1.proto import video_intelligence_pb2_grpc -from google.longrunning import operations_pb2 class VideoIntelligenceServiceGrpcTransport(object): @@ -72,8 +71,7 @@ def __init__( self._stubs = { "video_intelligence_service_stub": video_intelligence_pb2_grpc.VideoIntelligenceServiceStub( channel - ), - "operations_stub": operations_pb2.OperationsStub(channel) + ) } # Because this API includes a method that returns a diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py index ff5fdec70e21..611e45fc6ed7 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py @@ -172,9 +172,6 @@ def __init__( # transport methods, wrapped with `wrap_method` to add retry, # timeout, and the like. self._inner_api_calls = {} - - def channel(self): - return self.transport.channel def get_operation(self, name=None): return self.transport.get_operation(name) @@ -300,6 +297,4 @@ def annotate_video( self.transport._operations_client, video_intelligence_pb2.AnnotateVideoResponse, metadata_type=video_intelligence_pb2.AnnotateVideoProgress, - ) - - + ) From 133b008871466672017d863af9af95dfc92228df Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Wed, 3 Jul 2019 10:43:07 -0700 Subject: [PATCH 03/19] Adds VPC SC integration tests for Video Intelligence API --- ...deo_intelligence_service_grpc_transport.py | 16 +- .../video_intelligence_service_client.py | 31 ++-- videointelligence/tests/system.py | 159 ++++++++++++++++++ 3 files changed, 189 insertions(+), 17 deletions(-) create mode 100644 videointelligence/tests/system.py diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py index 0c3a472179cf..9bf9f3bd5024 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py @@ -19,6 +19,7 @@ import google.api_core.operations_v1 from google.cloud.videointelligence_v1.proto import video_intelligence_pb2_grpc +from google.longrunning import operations_pb2 class VideoIntelligenceServiceGrpcTransport(object): @@ -71,7 +72,8 @@ def __init__( self._stubs = { "video_intelligence_service_stub": video_intelligence_pb2_grpc.VideoIntelligenceServiceStub( channel - ) + ), + "operations_stub": operations_pb2.OperationsStub(channel) } # Because this API includes a method that returns a @@ -113,6 +115,18 @@ def channel(self): """ return self._channel + @property + def get_operation(self): + return self._operations_client.get_operation + + @property + def delete_operation(self): + return self._operations_client.delete_operation + + @property + def cancel_operation(self): + return self._operations_client.cancel_operation + @property def annotate_video(self): """Return the gRPC stub for :meth:`VideoIntelligenceServiceClient.annotate_video`. diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py index 890992f9fb65..ff5fdec70e21 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py @@ -20,7 +20,6 @@ import warnings from google.oauth2 import service_account -import google.api_core.client_options import google.api_core.gapic_v1.client_info import google.api_core.gapic_v1.config import google.api_core.gapic_v1.method @@ -83,7 +82,6 @@ def __init__( credentials=None, client_config=None, client_info=None, - client_options=None, ): """Constructor. @@ -114,9 +112,6 @@ def __init__( API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. - client_options (Union[dict, google.api_core.client_options.ClientOptions]): - Client options used to set user options on the client. API Endpoint - should be set through client_options. """ # Raise deprecation warnings for things we want to go away. if client_config is not None: @@ -135,15 +130,6 @@ def __init__( stacklevel=2, ) - api_endpoint = self.SERVICE_ADDRESS - if client_options: - if type(client_options) == dict: - client_options = google.api_core.client_options.from_dict( - client_options - ) - if client_options.api_endpoint: - api_endpoint = client_options.api_endpoint - # Instantiate the transport. # The transport is responsible for handling serialization and # deserialization and actually sending data to the service. @@ -152,7 +138,6 @@ def __init__( self.transport = transport( credentials=credentials, default_class=video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport, - address=api_endpoint, ) else: if credentials: @@ -163,7 +148,7 @@ def __init__( self.transport = transport else: self.transport = video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport( - address=api_endpoint, channel=channel, credentials=credentials + address=self.SERVICE_ADDRESS, channel=channel, credentials=credentials ) if client_info is None: @@ -187,7 +172,19 @@ def __init__( # transport methods, wrapped with `wrap_method` to add retry, # timeout, and the like. self._inner_api_calls = {} + + def channel(self): + return self.transport.channel + def get_operation(self, name=None): + return self.transport.get_operation(name) + + def delete_operation(self, name=None): + return self.transport.delete_operation(name) + + def cancel_operation(self, name=None): + return self.transport.cancel_operation(name) + # Service calls def annotate_video( self, @@ -304,3 +301,5 @@ def annotate_video( video_intelligence_pb2.AnnotateVideoResponse, metadata_type=video_intelligence_pb2.AnnotateVideoProgress, ) + + diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py new file mode 100644 index 000000000000..ea58a97286bb --- /dev/null +++ b/videointelligence/tests/system.py @@ -0,0 +1,159 @@ +# Copyright 2017, Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""System tests for VideoIntelligence API.""" + +import grpc +import io +import json +import os +import requests +import time +import unittest + +import google.api_core.exceptions +from google.cloud import exceptions +from google.cloud import videointelligence +from google.cloud.videointelligence_v1 import enums + +PROJECT_NUMBER = os.environ.get("PROJECT_NUMBER") +PROJECT_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT") +BUCKET_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") +OUTSIDE_PROJECT_API_KEY = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY") +OUTSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP") +INSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP") + +class VideoIntelligenceSystemTestBase(unittest.TestCase): + client = None + + def setUp(self): + self.to_delete_by_case = [] + self.input_uri = "gs://cloud-samples-data/video/cat.mp4" + + def tearDown(self): + for value in self.to_delete_by_case: + value.delete() + + +def setUpModule(): + VideoIntelligenceSystemTestBase.client = videointelligence.VideoIntelligenceServiceClient() + + +class TestVideoIntelligenceClient(VideoIntelligenceSystemTestBase): + def test_annotate_video(self): + features_element = enums.Feature.LABEL_DETECTION + features = [features_element] + response = self.client.annotate_video(input_uri=self.input_uri, features=features) + + # Wait for the operation to complete. + lro_timeout_seconds = 60 + start_time = time.time() + cnt = 0 + while not response.done() and (time.time() - start_time) < lro_timeout_seconds: + time.sleep(1) + cnt += 1 + if not response.done(): + self.fail( + "wait for operation timed out after {lro_timeout_seconds} seconds".format( + lro_timeout_seconds=lro_timeout_seconds + ) + ) + + result = response.result() + annotations = result.annotation_results[0] + assert len(annotations.segment_label_annotations) > 0 + + + +@unittest.skipUnless( + OUTSIDE_PROJECT_API_KEY, + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY not set in environment.", +) +class TestVideoIntelligenceClientVpcSc(VideoIntelligenceSystemTestBase): + # Tests to verify VideoIntelligence service requests blocked when trying to access resources outside of a secure perimeter. + def setUp(self): + VideoIntelligenceSystemTestBase.setUp(self) + # api-endpoint + self.url = "https://videointelligence.googleapis.com/v1/videos:annotate?key={}".format(OUTSIDE_PROJECT_API_KEY) + self.body = {"input_uri": self.input_uri, "features": ["LABEL_DETECTION"], "location_id": "us-west1"} + + def test_get_operation_from_different_project(self): + # sending request and saving the response as response object + r = requests.post(url=self.url, data=self.body) + outside_project_operation = json.loads(r.text) + + # Uses inside perimeter project to access operation resource created owned by outside project + with self.assertRaises(google.api_core.exceptions.NotFound) as cm: + self.client.get_operation(outside_project_operation["name"]) + + # Assert it returns not found. + self.assertEqual(cm.exception.code, 404) + self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) + + def test_delete_operation_from_different_project(self): + # sending request and saving the response as response object + r = requests.post(url=self.url, data=self.body) + outside_project_operation = json.loads(r.text) + + # Uses inside perimeter project to access operation resource created owned by outside project + with self.assertRaises(Exception) as cm: + self.client.delete_operation(outside_project_operation["name"]) + + # Assert it returns not found. + self.assertEqual(cm.exception.code, 404) + self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) + + def test_cancel_operation_from_different_project(self): + # sending request and saving the response as response object + r = requests.post(url=self.url, data=self.body) + outside_project_operation = json.loads(r.text) + + # Uses inside perimeter project to access operation resource created owned by outside project + with self.assertRaises(Exception) as cm: + self.client.cancel_operation(outside_project_operation["name"]) + + # Assert it returns not found. + self.assertEqual(cm.exception.code, 404) + self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) + + @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") + @unittest.skipUnless(OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment.") + def test_outside_ip_address_blocked(self): + headers = { + "Content-Type": "application/json", + "X-User-IP": OUTSIDE_IP, + "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, + } + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + outside_project_operation = json.loads(r.text) + print(outside_project_operation) + # Assert it returns permission denied from VPC SC + self.assertEqual(outside_project_operation["error"]["code"], 403) + self.assertEqual(outside_project_operation["error"]["status"], "PERMISSION_DENIED") + self.assertEqual(outside_project_operation["error"]["details"][0]["violations"][0]["type"], "VPC_SERVICE_CONTROLS") + self.assertEqual(outside_project_operation["error"]["message"], "Request is prohibited by organization's policy") + + @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") + @unittest.skipUnless(INSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP not set in environment.") + def test_inside_ip_address_allowed(self): + headers = { + "Content-Type": "application/json", + "X-User-IP": INSIDE_IP, + "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, + } + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + operation = json.loads(r.text) + # Assert it returns non-empty operation name. + self.assertNotEqual(operation["name"], "") + From 34e9c2b0178303b93a27de8d638bc005b5ec8dca Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Mon, 8 Jul 2019 09:21:03 -0700 Subject: [PATCH 04/19] Addressing comments --- .../video_intelligence_service_grpc_transport.py | 4 +--- .../gapic/video_intelligence_service_client.py | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py index 9bf9f3bd5024..7dc775491d11 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py @@ -19,7 +19,6 @@ import google.api_core.operations_v1 from google.cloud.videointelligence_v1.proto import video_intelligence_pb2_grpc -from google.longrunning import operations_pb2 class VideoIntelligenceServiceGrpcTransport(object): @@ -72,8 +71,7 @@ def __init__( self._stubs = { "video_intelligence_service_stub": video_intelligence_pb2_grpc.VideoIntelligenceServiceStub( channel - ), - "operations_stub": operations_pb2.OperationsStub(channel) + ) } # Because this API includes a method that returns a diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py index ff5fdec70e21..611e45fc6ed7 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py @@ -172,9 +172,6 @@ def __init__( # transport methods, wrapped with `wrap_method` to add retry, # timeout, and the like. self._inner_api_calls = {} - - def channel(self): - return self.transport.channel def get_operation(self, name=None): return self.transport.get_operation(name) @@ -300,6 +297,4 @@ def annotate_video( self.transport._operations_client, video_intelligence_pb2.AnnotateVideoResponse, metadata_type=video_intelligence_pb2.AnnotateVideoProgress, - ) - - + ) From 625ffa03972131b537ba1ef02216115d8368c778 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Tue, 9 Jul 2019 10:52:27 -0700 Subject: [PATCH 05/19] Removed changes in generated code. --- ...deo_intelligence_service_grpc_transport.py | 12 -- .../video_intelligence_service_client.py | 28 +++-- videointelligence/tests/system.py | 113 +++++++----------- 3 files changed, 63 insertions(+), 90 deletions(-) diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py index 7dc775491d11..0c3a472179cf 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py @@ -113,18 +113,6 @@ def channel(self): """ return self._channel - @property - def get_operation(self): - return self._operations_client.get_operation - - @property - def delete_operation(self): - return self._operations_client.delete_operation - - @property - def cancel_operation(self): - return self._operations_client.cancel_operation - @property def annotate_video(self): """Return the gRPC stub for :meth:`VideoIntelligenceServiceClient.annotate_video`. diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py index 611e45fc6ed7..890992f9fb65 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py @@ -20,6 +20,7 @@ import warnings from google.oauth2 import service_account +import google.api_core.client_options import google.api_core.gapic_v1.client_info import google.api_core.gapic_v1.config import google.api_core.gapic_v1.method @@ -82,6 +83,7 @@ def __init__( credentials=None, client_config=None, client_info=None, + client_options=None, ): """Constructor. @@ -112,6 +114,9 @@ def __init__( API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. + client_options (Union[dict, google.api_core.client_options.ClientOptions]): + Client options used to set user options on the client. API Endpoint + should be set through client_options. """ # Raise deprecation warnings for things we want to go away. if client_config is not None: @@ -130,6 +135,15 @@ def __init__( stacklevel=2, ) + api_endpoint = self.SERVICE_ADDRESS + if client_options: + if type(client_options) == dict: + client_options = google.api_core.client_options.from_dict( + client_options + ) + if client_options.api_endpoint: + api_endpoint = client_options.api_endpoint + # Instantiate the transport. # The transport is responsible for handling serialization and # deserialization and actually sending data to the service. @@ -138,6 +152,7 @@ def __init__( self.transport = transport( credentials=credentials, default_class=video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport, + address=api_endpoint, ) else: if credentials: @@ -148,7 +163,7 @@ def __init__( self.transport = transport else: self.transport = video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport( - address=self.SERVICE_ADDRESS, channel=channel, credentials=credentials + address=api_endpoint, channel=channel, credentials=credentials ) if client_info is None: @@ -173,15 +188,6 @@ def __init__( # timeout, and the like. self._inner_api_calls = {} - def get_operation(self, name=None): - return self.transport.get_operation(name) - - def delete_operation(self, name=None): - return self.transport.delete_operation(name) - - def cancel_operation(self, name=None): - return self.transport.cancel_operation(name) - # Service calls def annotate_video( self, @@ -297,4 +303,4 @@ def annotate_video( self.transport._operations_client, video_intelligence_pb2.AnnotateVideoResponse, metadata_type=video_intelligence_pb2.AnnotateVideoProgress, - ) + ) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index ea58a97286bb..bf4d5cdfc278 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -30,32 +30,34 @@ PROJECT_NUMBER = os.environ.get("PROJECT_NUMBER") PROJECT_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT") BUCKET_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") -OUTSIDE_PROJECT_API_KEY = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY") +OUTSIDE_PROJECT_API_KEY = os.environ.get( + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY" +) OUTSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP") INSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP") + class VideoIntelligenceSystemTestBase(unittest.TestCase): client = None def setUp(self): - self.to_delete_by_case = [] self.input_uri = "gs://cloud-samples-data/video/cat.mp4" - def tearDown(self): - for value in self.to_delete_by_case: - value.delete() - def setUpModule(): - VideoIntelligenceSystemTestBase.client = videointelligence.VideoIntelligenceServiceClient() + VideoIntelligenceSystemTestBase.client = ( + videointelligence.VideoIntelligenceServiceClient() + ) class TestVideoIntelligenceClient(VideoIntelligenceSystemTestBase): def test_annotate_video(self): features_element = enums.Feature.LABEL_DETECTION features = [features_element] - response = self.client.annotate_video(input_uri=self.input_uri, features=features) - + response = self.client.annotate_video( + input_uri=self.input_uri, features=features + ) + # Wait for the operation to complete. lro_timeout_seconds = 60 start_time = time.time() @@ -69,13 +71,12 @@ def test_annotate_video(self): lro_timeout_seconds=lro_timeout_seconds ) ) - + result = response.result() annotations = result.annotation_results[0] assert len(annotations.segment_label_annotations) > 0 - @unittest.skipUnless( OUTSIDE_PROJECT_API_KEY, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY not set in environment.", @@ -84,76 +85,54 @@ class TestVideoIntelligenceClientVpcSc(VideoIntelligenceSystemTestBase): # Tests to verify VideoIntelligence service requests blocked when trying to access resources outside of a secure perimeter. def setUp(self): VideoIntelligenceSystemTestBase.setUp(self) - # api-endpoint - self.url = "https://videointelligence.googleapis.com/v1/videos:annotate?key={}".format(OUTSIDE_PROJECT_API_KEY) - self.body = {"input_uri": self.input_uri, "features": ["LABEL_DETECTION"], "location_id": "us-west1"} - - def test_get_operation_from_different_project(self): - # sending request and saving the response as response object - r = requests.post(url=self.url, data=self.body) - outside_project_operation = json.loads(r.text) - - # Uses inside perimeter project to access operation resource created owned by outside project - with self.assertRaises(google.api_core.exceptions.NotFound) as cm: - self.client.get_operation(outside_project_operation["name"]) - - # Assert it returns not found. - self.assertEqual(cm.exception.code, 404) - self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) - - def test_delete_operation_from_different_project(self): - # sending request and saving the response as response object - r = requests.post(url=self.url, data=self.body) - outside_project_operation = json.loads(r.text) - - # Uses inside perimeter project to access operation resource created owned by outside project - with self.assertRaises(Exception) as cm: - self.client.delete_operation(outside_project_operation["name"]) - - # Assert it returns not found. - self.assertEqual(cm.exception.code, 404) - self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) - - def test_cancel_operation_from_different_project(self): - # sending request and saving the response as response object - r = requests.post(url=self.url, data=self.body) - outside_project_operation = json.loads(r.text) - - # Uses inside perimeter project to access operation resource created owned by outside project - with self.assertRaises(Exception) as cm: - self.client.cancel_operation(outside_project_operation["name"]) - - # Assert it returns not found. - self.assertEqual(cm.exception.code, 404) - self.assertEqual(cm.exception.message, "Operation not found: {}".format(outside_project_operation["name"])) - + # api-endpoint + self.url = "https://videointelligence.googleapis.com/v1/videos:annotate?key={}".format( + OUTSIDE_PROJECT_API_KEY + ) + self.body = { + "input_uri": self.input_uri, + "features": ["LABEL_DETECTION"], + "location_id": "us-west1", + } + @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") - @unittest.skipUnless(OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment.") + @unittest.skipUnless( + OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment." + ) def test_outside_ip_address_blocked(self): headers = { - "Content-Type": "application/json", - "X-User-IP": OUTSIDE_IP, - "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, + "Content-Type": "application/json", + "X-User-IP": OUTSIDE_IP, + "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, } r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) outside_project_operation = json.loads(r.text) print(outside_project_operation) # Assert it returns permission denied from VPC SC self.assertEqual(outside_project_operation["error"]["code"], 403) - self.assertEqual(outside_project_operation["error"]["status"], "PERMISSION_DENIED") - self.assertEqual(outside_project_operation["error"]["details"][0]["violations"][0]["type"], "VPC_SERVICE_CONTROLS") - self.assertEqual(outside_project_operation["error"]["message"], "Request is prohibited by organization's policy") - + self.assertEqual( + outside_project_operation["error"]["status"], "PERMISSION_DENIED" + ) + self.assertEqual( + outside_project_operation["error"]["details"][0]["violations"][0]["type"], + "VPC_SERVICE_CONTROLS", + ) + self.assertEqual( + outside_project_operation["error"]["message"], + "Request is prohibited by organization's policy", + ) + @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") - @unittest.skipUnless(INSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP not set in environment.") + @unittest.skipUnless( + INSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP not set in environment." + ) def test_inside_ip_address_allowed(self): headers = { - "Content-Type": "application/json", - "X-User-IP": INSIDE_IP, - "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, + "Content-Type": "application/json", + "X-User-IP": INSIDE_IP, + "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, } r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) operation = json.loads(r.text) # Assert it returns non-empty operation name. self.assertNotEqual(operation["name"], "") - From d6707d3b568ac536025bd11c83a40832d1a67dfa Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Tue, 9 Jul 2019 11:02:54 -0700 Subject: [PATCH 06/19] Reverting generated code changes. --- ...deo_intelligence_service_grpc_transport.py | 12 -------- .../video_intelligence_service_client.py | 28 +++++++++++-------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py index 7dc775491d11..0c3a472179cf 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/transports/video_intelligence_service_grpc_transport.py @@ -113,18 +113,6 @@ def channel(self): """ return self._channel - @property - def get_operation(self): - return self._operations_client.get_operation - - @property - def delete_operation(self): - return self._operations_client.delete_operation - - @property - def cancel_operation(self): - return self._operations_client.cancel_operation - @property def annotate_video(self): """Return the gRPC stub for :meth:`VideoIntelligenceServiceClient.annotate_video`. diff --git a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py index 611e45fc6ed7..890992f9fb65 100644 --- a/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py +++ b/videointelligence/google/cloud/videointelligence_v1/gapic/video_intelligence_service_client.py @@ -20,6 +20,7 @@ import warnings from google.oauth2 import service_account +import google.api_core.client_options import google.api_core.gapic_v1.client_info import google.api_core.gapic_v1.config import google.api_core.gapic_v1.method @@ -82,6 +83,7 @@ def __init__( credentials=None, client_config=None, client_info=None, + client_options=None, ): """Constructor. @@ -112,6 +114,9 @@ def __init__( API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. + client_options (Union[dict, google.api_core.client_options.ClientOptions]): + Client options used to set user options on the client. API Endpoint + should be set through client_options. """ # Raise deprecation warnings for things we want to go away. if client_config is not None: @@ -130,6 +135,15 @@ def __init__( stacklevel=2, ) + api_endpoint = self.SERVICE_ADDRESS + if client_options: + if type(client_options) == dict: + client_options = google.api_core.client_options.from_dict( + client_options + ) + if client_options.api_endpoint: + api_endpoint = client_options.api_endpoint + # Instantiate the transport. # The transport is responsible for handling serialization and # deserialization and actually sending data to the service. @@ -138,6 +152,7 @@ def __init__( self.transport = transport( credentials=credentials, default_class=video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport, + address=api_endpoint, ) else: if credentials: @@ -148,7 +163,7 @@ def __init__( self.transport = transport else: self.transport = video_intelligence_service_grpc_transport.VideoIntelligenceServiceGrpcTransport( - address=self.SERVICE_ADDRESS, channel=channel, credentials=credentials + address=api_endpoint, channel=channel, credentials=credentials ) if client_info is None: @@ -173,15 +188,6 @@ def __init__( # timeout, and the like. self._inner_api_calls = {} - def get_operation(self, name=None): - return self.transport.get_operation(name) - - def delete_operation(self, name=None): - return self.transport.delete_operation(name) - - def cancel_operation(self, name=None): - return self.transport.cancel_operation(name) - # Service calls def annotate_video( self, @@ -297,4 +303,4 @@ def annotate_video( self.transport._operations_client, video_intelligence_pb2.AnnotateVideoResponse, metadata_type=video_intelligence_pb2.AnnotateVideoProgress, - ) + ) From f88ce4f6be8dc386194fc7a21429d5cdca5785ad Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Wed, 10 Jul 2019 00:45:45 -0700 Subject: [PATCH 07/19] Set VPS SC environment variables. --- videointelligence/noxfile.py | 11 +++++++++-- videointelligence/tests/system.py | 2 -- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/videointelligence/noxfile.py b/videointelligence/noxfile.py index a2eefbb6765f..386bf4c64552 100644 --- a/videointelligence/noxfile.py +++ b/videointelligence/noxfile.py @@ -118,11 +118,18 @@ def system(session): session.install("-e", "../test_utils/") session.install("-e", ".") + # Additional set up for VPC SC. + env = { + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY": "AIzaSyCgRxOwpqt-lO6FD4sIqko23wwwexA4YsM", + "PROJECT_NUMBER": "570941833855", + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP": "10.1.1.1", + "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP": "55.55.0.0", + } # Run py.test against the system tests. if system_test_exists: - session.run("py.test", "--quiet", system_test_path, *session.posargs) + session.run("py.test", "--quiet", system_test_path, env=env, *session.posargs) if system_test_folder_exists: - session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) + session.run("py.test", "--quiet", system_test_folder_path, env=env *session.posargs) @nox.session(python="3.7") diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index bf4d5cdfc278..59c29dc97139 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -28,8 +28,6 @@ from google.cloud.videointelligence_v1 import enums PROJECT_NUMBER = os.environ.get("PROJECT_NUMBER") -PROJECT_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT") -BUCKET_OUTSIDE = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") OUTSIDE_PROJECT_API_KEY = os.environ.get( "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY" ) From e9271daf7d4387d5b92d16be7bc2e9a3b5490015 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Wed, 10 Jul 2019 01:00:25 -0700 Subject: [PATCH 08/19] Remove Api key from code (sensitive data) will have to be set through kokoro. --- videointelligence/noxfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/videointelligence/noxfile.py b/videointelligence/noxfile.py index 386bf4c64552..81cc32d7bc15 100644 --- a/videointelligence/noxfile.py +++ b/videointelligence/noxfile.py @@ -120,7 +120,6 @@ def system(session): # Additional set up for VPC SC. env = { - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY": "AIzaSyCgRxOwpqt-lO6FD4sIqko23wwwexA4YsM", "PROJECT_NUMBER": "570941833855", "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP": "10.1.1.1", "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP": "55.55.0.0", From c14396a22f1ecbdd07a3765d99d7af0db100bcb7 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Wed, 10 Jul 2019 16:19:46 -0700 Subject: [PATCH 09/19] Fix lint error. --- videointelligence/noxfile.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/videointelligence/noxfile.py b/videointelligence/noxfile.py index 81cc32d7bc15..9a0e46e83078 100644 --- a/videointelligence/noxfile.py +++ b/videointelligence/noxfile.py @@ -120,15 +120,17 @@ def system(session): # Additional set up for VPC SC. env = { - "PROJECT_NUMBER": "570941833855", - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP": "10.1.1.1", - "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP": "55.55.0.0", + "PROJECT_NUMBER": "570941833855", + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP": "10.1.1.1", + "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP": "55.55.0.0", } # Run py.test against the system tests. if system_test_exists: session.run("py.test", "--quiet", system_test_path, env=env, *session.posargs) if system_test_folder_exists: - session.run("py.test", "--quiet", system_test_folder_path, env=env *session.posargs) + session.run( + "py.test", "--quiet", system_test_folder_path, env=env * session.posargs + ) @nox.session(python="3.7") From f885db70f155fb70f43dec7d404a9018e2f437e0 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Wed, 10 Jul 2019 17:48:47 -0700 Subject: [PATCH 10/19] Remove unused imports. --- videointelligence/tests/system.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index 59c29dc97139..eb1c4bd5547d 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -14,16 +14,12 @@ """System tests for VideoIntelligence API.""" -import grpc -import io import json import os import requests import time import unittest -import google.api_core.exceptions -from google.cloud import exceptions from google.cloud import videointelligence from google.cloud.videointelligence_v1 import enums From e23764d41beafd321259c5da25116dd6fdf4a775 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Thu, 11 Jul 2019 12:33:11 -0700 Subject: [PATCH 11/19] Increase lro timeout to 180s. --- videointelligence/tests/system.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index eb1c4bd5547d..a8ad0a9b29bf 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -53,7 +53,8 @@ def test_annotate_video(self): ) # Wait for the operation to complete. - lro_timeout_seconds = 60 + # Long timeout value warranted due to https://github.com/grpc/grpc/issues/19173 + lro_timeout_seconds = 180 start_time = time.time() cnt = 0 while not response.done() and (time.time() - start_time) < lro_timeout_seconds: From 5dec9075560f16b9f284a74f664c53d2231b3e3d Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Mon, 9 Sep 2019 18:46:29 -0700 Subject: [PATCH 12/19] Enriches VPC SC tests. - Uses Access Token instead of API key. - Increases lro timeout to 300s. --- videointelligence/tests/system.py | 63 ++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index a8ad0a9b29bf..9cd31db3863d 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -20,16 +20,27 @@ import time import unittest +from google.auth.transport import requests as goog_auth_requests from google.cloud import videointelligence from google.cloud.videointelligence_v1 import enums +from google.oauth2 import service_account -PROJECT_NUMBER = os.environ.get("PROJECT_NUMBER") -OUTSIDE_PROJECT_API_KEY = os.environ.get( - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY" -) +CLOUD_PLATFORM_SCOPE = 'https://www.googleapis.com/auth/cloud-platform' +CREDENTIALS_FILE = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") OUTSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP") INSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP") +def get_access_token(): + """Returns an access token. + + Generates access tokens using the provided service account key file. + """ + creds = service_account.Credentials.from_service_account_file( + CREDENTIALS_FILE, scopes=[CLOUD_PLATFORM_SCOPE]) + with requests.Session() as session: + creds.refresh(goog_auth_requests.Request(session=session)) + return creds.token + class VideoIntelligenceSystemTestBase(unittest.TestCase): client = None @@ -54,7 +65,7 @@ def test_annotate_video(self): # Wait for the operation to complete. # Long timeout value warranted due to https://github.com/grpc/grpc/issues/19173 - lro_timeout_seconds = 180 + lro_timeout_seconds = 300 start_time = time.time() cnt = 0 while not response.done() and (time.time() - start_time) < lro_timeout_seconds: @@ -73,61 +84,69 @@ def test_annotate_video(self): @unittest.skipUnless( - OUTSIDE_PROJECT_API_KEY, - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_PROJECT_API_KEY not set in environment.", + CREDENTIALS_FILE, + "GOOGLE_APPLICATION_CREDENTIALS not set in environment.", ) class TestVideoIntelligenceClientVpcSc(VideoIntelligenceSystemTestBase): # Tests to verify VideoIntelligence service requests blocked when trying to access resources outside of a secure perimeter. def setUp(self): VideoIntelligenceSystemTestBase.setUp(self) # api-endpoint - self.url = "https://videointelligence.googleapis.com/v1/videos:annotate?key={}".format( - OUTSIDE_PROJECT_API_KEY - ) + self.url = "https://videointelligence.googleapis.com/v1/videos:annotate" self.body = { "input_uri": self.input_uri, "features": ["LABEL_DETECTION"], "location_id": "us-west1", } - @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") @unittest.skipUnless( OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment." ) def test_outside_ip_address_blocked(self): headers = { + "Authorization": "Bearer " + get_access_token(), "Content-Type": "application/json", "X-User-IP": OUTSIDE_IP, - "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, } r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) - outside_project_operation = json.loads(r.text) - print(outside_project_operation) + operation = json.loads(r.text) + print(operation) + + get_op_url = "https://videointelligence.googleapis.com/v1/" + operation["name"] + get_op = requests.get(url=get_op_url, headers=headers) + get_op_resp = json.loads(get_op.text) + print(get_op_resp) # Assert it returns permission denied from VPC SC - self.assertEqual(outside_project_operation["error"]["code"], 403) + self.assertEqual(get_op_resp["error"]["code"], 403) self.assertEqual( - outside_project_operation["error"]["status"], "PERMISSION_DENIED" + get_op_resp["error"]["status"], "PERMISSION_DENIED" ) self.assertEqual( - outside_project_operation["error"]["details"][0]["violations"][0]["type"], + get_op_resp["error"]["details"][0]["violations"][0]["type"], "VPC_SERVICE_CONTROLS", ) self.assertEqual( - outside_project_operation["error"]["message"], + get_op_resp["error"]["message"], "Request is prohibited by organization's policy", ) - @unittest.skipUnless(PROJECT_NUMBER, "PROJECT_NUMBER not set in environment.") @unittest.skipUnless( INSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP not set in environment." ) def test_inside_ip_address_allowed(self): headers = { + "Authorization": "Bearer " + get_access_token(), "Content-Type": "application/json", "X-User-IP": INSIDE_IP, - "X-Google-GFE-Cloud-Client-Network-Project-Number": PROJECT_NUMBER, } r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) operation = json.loads(r.text) - # Assert it returns non-empty operation name. - self.assertNotEqual(operation["name"], "") + print(operation) + + get_op_url = "https://videointelligence.googleapis.com/v1/" + operation["name"] + get_op = requests.get(url=get_op_url, headers=headers) + get_op_resp = json.loads(get_op.text) + print(get_op_resp) + # Assert that we do not get an error. + self.assertEqual(get_op_resp["name"], operation["name"]) + From 3a0f6aea4b1460b3c1232d6f16f2d93bf753ecae Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Thu, 12 Sep 2019 13:36:02 -0700 Subject: [PATCH 13/19] Only run VPC SC tests when GOOGLE_CLOUD_TESTS_IN_VPCSC is set to true --- videointelligence/tests/system.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index d8db6c1d2cc4..5d360ddc247e 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -29,6 +29,7 @@ CREDENTIALS_FILE = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") OUTSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP") INSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP") +IS_INSIDE_VPCSC = os.environ.get("GOOGLE_CLOUD_TESTS_IN_VPCSC") def get_access_token(): """Returns an access token. @@ -101,6 +102,9 @@ def setUp(self): @unittest.skipUnless( OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment." ) + @unittest.skipUnless( + IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment." + ) def test_outside_ip_address_blocked(self): headers = { "Authorization": "Bearer " + get_access_token(), From 6be1d009e3b5990fc6836ad50ac489d4d7bdc3f5 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Fri, 13 Sep 2019 15:35:41 -0700 Subject: [PATCH 14/19] Use public video --- videointelligence/tests/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index 5d360ddc247e..2198e43631fb 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -46,7 +46,7 @@ class VideoIntelligenceSystemTestBase(unittest.TestCase): client = None def setUp(self): - self.input_uri = "gs://cloud-samples-data/video/cat.mp4" + self.input_uri = "gs://videodemomaker/cat.mp4" def setUpModule(): From 4bde9bdb41b8a652ba620ed2147d8860035dc823 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Fri, 20 Sep 2019 14:08:48 -0700 Subject: [PATCH 15/19] Use Outside and inside perimeter buckets instead of IP address. --- videointelligence/tests/system.py | 182 +++++++++++++----------------- 1 file changed, 79 insertions(+), 103 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index 2198e43631fb..d855174cf68c 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """System tests for VideoIntelligence API.""" import json @@ -25,15 +24,18 @@ from google.cloud.videointelligence_v1 import enums from google.oauth2 import service_account -CLOUD_PLATFORM_SCOPE = 'https://www.googleapis.com/auth/cloud-platform' +CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" CREDENTIALS_FILE = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") -OUTSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP") -INSIDE_IP = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP") +OUTSIDE_BUCKET = os.environ.get( + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") +INSIDE_BUCKET = os.environ.get( + "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET") IS_INSIDE_VPCSC = os.environ.get("GOOGLE_CLOUD_TESTS_IN_VPCSC") + def get_access_token(): """Returns an access token. - + Generates access tokens using the provided service account key file. """ creds = service_account.Credentials.from_service_account_file( @@ -42,45 +44,14 @@ def get_access_token(): creds.refresh(goog_auth_requests.Request(session=session)) return creds.token -class VideoIntelligenceSystemTestBase(unittest.TestCase): - client = None - def setUp(self): - self.input_uri = "gs://videodemomaker/cat.mp4" +class VideoIntelligenceSystemTestBase(unittest.TestCase): + client = None def setUpModule(): - VideoIntelligenceSystemTestBase.client = ( - videointelligence.VideoIntelligenceServiceClient() - ) - - -class TestVideoIntelligenceClient(VideoIntelligenceSystemTestBase): - def test_annotate_video(self): - features_element = enums.Feature.LABEL_DETECTION - features = [features_element] - response = self.client.annotate_video( - input_uri=self.input_uri, features=features - ) - - # Wait for the operation to complete. - # Long timeout value warranted due to https://github.com/grpc/grpc/issues/19173 - lro_timeout_seconds = 300 - start_time = time.time() - cnt = 0 - while not response.done() and (time.time() - start_time) < lro_timeout_seconds: - time.sleep(1) - cnt += 1 - if not response.done(): - self.fail( - "wait for operation timed out after {lro_timeout_seconds} seconds".format( - lro_timeout_seconds=lro_timeout_seconds - ) - ) - - result = response.result() - annotations = result.annotation_results[0] - assert len(annotations.segment_label_annotations) > 0 + VideoIntelligenceSystemTestBase.client = ( + videointelligence.VideoIntelligenceServiceClient()) @unittest.skipUnless( @@ -88,68 +59,73 @@ def test_annotate_video(self): "GOOGLE_APPLICATION_CREDENTIALS not set in environment.", ) class TestVideoIntelligenceClientVpcSc(VideoIntelligenceSystemTestBase): - # Tests to verify VideoIntelligence service requests blocked when trying to access resources outside of a secure perimeter. - def setUp(self): - VideoIntelligenceSystemTestBase.setUp(self) - # api-endpoint - self.url = "https://videointelligence.googleapis.com/v1/videos:annotate" - self.body = { - "input_uri": self.input_uri, - "features": ["LABEL_DETECTION"], - "location_id": "us-west1", - } - - @unittest.skipUnless( - OUTSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_IP not set in environment." - ) - @unittest.skipUnless( - IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment." + # Tests to verify VideoIntelligence service requests blocked when trying to + # access resources outside of a secure perimeter. + def setUp(self): + VideoIntelligenceSystemTestBase.setUp(self) + # api-endpoint + self.url = "https://videointelligence.googleapis.com/v1/videos:annotate" + self.body = { + "features": ["LABEL_DETECTION"], + "location_id": "us-west1", + } + + @unittest.skipUnless( + INSIDE_BUCKET, + "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET not set in environment." + ) + @unittest.skipUnless(IS_INSIDE_VPCSC, + "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") + def test_outside_ip_address_blocked(self): + headers = { + "Authorization": "Bearer " + get_access_token(), + "Content-Type": "application/json", + } + self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( + bucket=INSIDE_BUCKET) + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + operation = json.loads(r.text) + print(operation) + + get_op_url = "https://videointelligence.googleapis.com/v1/" + operation[ + "name"] + get_op = requests.get(url=get_op_url, headers=headers) + get_op_resp = json.loads(get_op.text) + print(get_op_resp) + # Assert it returns permission denied from VPC SC + self.assertEqual(get_op_resp["error"]["code"], 403) + self.assertEqual(get_op_resp["error"]["status"], "PERMISSION_DENIED") + self.assertEqual( + get_op_resp["error"]["details"][0]["violations"][0]["type"], + "VPC_SERVICE_CONTROLS", ) - def test_outside_ip_address_blocked(self): - headers = { - "Authorization": "Bearer " + get_access_token(), - "Content-Type": "application/json", - "X-User-IP": OUTSIDE_IP, - } - r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) - operation = json.loads(r.text) - print(operation) - - get_op_url = "https://videointelligence.googleapis.com/v1/" + operation["name"] - get_op = requests.get(url=get_op_url, headers=headers) - get_op_resp = json.loads(get_op.text) - print(get_op_resp) - # Assert it returns permission denied from VPC SC - self.assertEqual(get_op_resp["error"]["code"], 403) - self.assertEqual( - get_op_resp["error"]["status"], "PERMISSION_DENIED" - ) - self.assertEqual( - get_op_resp["error"]["details"][0]["violations"][0]["type"], - "VPC_SERVICE_CONTROLS", - ) - self.assertEqual( - get_op_resp["error"]["message"], - "Request is prohibited by organization's policy", - ) - - @unittest.skipUnless( - INSIDE_IP, "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_IP not set in environment." + self.assertEqual( + get_op_resp["error"]["message"], + "Request is prohibited by organization's policy", ) - def test_inside_ip_address_allowed(self): - headers = { - "Authorization": "Bearer " + get_access_token(), - "Content-Type": "application/json", - "X-User-IP": INSIDE_IP, - } - r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) - operation = json.loads(r.text) - print(operation) - - get_op_url = "https://videointelligence.googleapis.com/v1/" + operation["name"] - get_op = requests.get(url=get_op_url, headers=headers) - get_op_resp = json.loads(get_op.text) - print(get_op_resp) - # Assert that we do not get an error. - self.assertEqual(get_op_resp["name"], operation["name"]) - + + @unittest.skipUnless( + OUTSIDE_BUCKET, + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET not set in environment." + ) + @unittest.skipUnless(IS_INSIDE_VPCSC, + "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") + def test_inside_ip_address_allowed(self): + headers = { + "Authorization": "Bearer " + get_access_token(), + "Content-Type": "application/json", + } + self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( + bucket=OUTSIDE_BUCKET) + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + operation = json.loads(r.text) + print(operation) + + get_op_url = "https://videointelligence.googleapis.com/v1/" + operation[ + "name"] + get_op = requests.get(url=get_op_url, headers=headers) + get_op_resp = json.loads(get_op.text) + print(get_op_resp) + # Assert that we do not get an error. + self.assertEqual(get_op_resp["name"], operation["name"]) + From a8bc5cca670096f65ae22d053ab056e8b10dfb44 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Fri, 20 Sep 2019 15:48:06 -0700 Subject: [PATCH 16/19] Change inside and outside buckets --- videointelligence/tests/system.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index d855174cf68c..d28aaae25277 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -71,8 +71,8 @@ def setUp(self): } @unittest.skipUnless( - INSIDE_BUCKET, - "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET not set in environment." + OUTSIDE_BUCKET, + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET not set in environment." ) @unittest.skipUnless(IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") @@ -105,8 +105,8 @@ def test_outside_ip_address_blocked(self): ) @unittest.skipUnless( - OUTSIDE_BUCKET, - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET not set in environment." + INSIDE_BUCKET, + "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET not set in environment." ) @unittest.skipUnless(IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") From 57c6d36ff3a745f9bb7d7cacef01dc5f61c47f23 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Mon, 23 Sep 2019 13:45:52 -0700 Subject: [PATCH 17/19] Change inside and outside buckets --- videointelligence/tests/system.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index d28aaae25277..4d9f67e7432b 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -76,13 +76,13 @@ def setUp(self): ) @unittest.skipUnless(IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") - def test_outside_ip_address_blocked(self): + def test_outside_perimeter_blocked(self): headers = { "Authorization": "Bearer " + get_access_token(), "Content-Type": "application/json", } self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( - bucket=INSIDE_BUCKET) + bucket=OUTSIDE_BUCKET) r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) operation = json.loads(r.text) print(operation) @@ -110,13 +110,13 @@ def test_outside_ip_address_blocked(self): ) @unittest.skipUnless(IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") - def test_inside_ip_address_allowed(self): + def test_inside_perimeter_allowed(self): headers = { "Authorization": "Bearer " + get_access_token(), "Content-Type": "application/json", } self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( - bucket=OUTSIDE_BUCKET) + bucket=INSIDE_BUCKET) r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) operation = json.loads(r.text) print(operation) From d7ad054b41ee15eb67accbe1822fa2e077ab67cf Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Mon, 23 Sep 2019 14:03:05 -0700 Subject: [PATCH 18/19] Update test --- videointelligence/tests/system.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index 4d9f67e7432b..defbb9d620dc 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -84,25 +84,11 @@ def test_outside_perimeter_blocked(self): self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( bucket=OUTSIDE_BUCKET) r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) - operation = json.loads(r.text) - print(operation) - - get_op_url = "https://videointelligence.googleapis.com/v1/" + operation[ - "name"] - get_op = requests.get(url=get_op_url, headers=headers) - get_op_resp = json.loads(get_op.text) - print(get_op_resp) + resp = json.loads(r.text) + print(resp) # Assert it returns permission denied from VPC SC - self.assertEqual(get_op_resp["error"]["code"], 403) - self.assertEqual(get_op_resp["error"]["status"], "PERMISSION_DENIED") - self.assertEqual( - get_op_resp["error"]["details"][0]["violations"][0]["type"], - "VPC_SERVICE_CONTROLS", - ) - self.assertEqual( - get_op_resp["error"]["message"], - "Request is prohibited by organization's policy", - ) + self.assertEqual(resp["error"]["code"], 403) + self.assertEqual(resp["error"]["status"], "PERMISSION_DENIED") @unittest.skipUnless( INSIDE_BUCKET, From 4eb0ed0e4cbbf34af85300e5db5aeb5aed1a57f6 Mon Sep 17 00:00:00 2001 From: Jimmy Mkude Date: Mon, 23 Sep 2019 15:01:22 -0700 Subject: [PATCH 19/19] Fix lint errors. --- videointelligence/tests/system.py | 138 ++++++++++++++---------------- 1 file changed, 65 insertions(+), 73 deletions(-) diff --git a/videointelligence/tests/system.py b/videointelligence/tests/system.py index defbb9d620dc..a73e4edc1634 100644 --- a/videointelligence/tests/system.py +++ b/videointelligence/tests/system.py @@ -16,102 +16,94 @@ import json import os import requests -import time import unittest from google.auth.transport import requests as goog_auth_requests from google.cloud import videointelligence -from google.cloud.videointelligence_v1 import enums from google.oauth2 import service_account CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" CREDENTIALS_FILE = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") -OUTSIDE_BUCKET = os.environ.get( - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") -INSIDE_BUCKET = os.environ.get( - "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET") +OUTSIDE_BUCKET = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET") +INSIDE_BUCKET = os.environ.get("GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET") IS_INSIDE_VPCSC = os.environ.get("GOOGLE_CLOUD_TESTS_IN_VPCSC") def get_access_token(): - """Returns an access token. + """Returns an access token. Generates access tokens using the provided service account key file. """ - creds = service_account.Credentials.from_service_account_file( - CREDENTIALS_FILE, scopes=[CLOUD_PLATFORM_SCOPE]) - with requests.Session() as session: - creds.refresh(goog_auth_requests.Request(session=session)) - return creds.token + creds = service_account.Credentials.from_service_account_file( + CREDENTIALS_FILE, scopes=[CLOUD_PLATFORM_SCOPE] + ) + with requests.Session() as session: + creds.refresh(goog_auth_requests.Request(session=session)) + return creds.token class VideoIntelligenceSystemTestBase(unittest.TestCase): - client = None + client = None def setUpModule(): - VideoIntelligenceSystemTestBase.client = ( - videointelligence.VideoIntelligenceServiceClient()) + VideoIntelligenceSystemTestBase.client = ( + videointelligence.VideoIntelligenceServiceClient() + ) @unittest.skipUnless( - CREDENTIALS_FILE, - "GOOGLE_APPLICATION_CREDENTIALS not set in environment.", + CREDENTIALS_FILE, "GOOGLE_APPLICATION_CREDENTIALS not set in environment." ) class TestVideoIntelligenceClientVpcSc(VideoIntelligenceSystemTestBase): - # Tests to verify VideoIntelligence service requests blocked when trying to - # access resources outside of a secure perimeter. - def setUp(self): - VideoIntelligenceSystemTestBase.setUp(self) - # api-endpoint - self.url = "https://videointelligence.googleapis.com/v1/videos:annotate" - self.body = { - "features": ["LABEL_DETECTION"], - "location_id": "us-west1", - } - - @unittest.skipUnless( - OUTSIDE_BUCKET, - "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET not set in environment." - ) - @unittest.skipUnless(IS_INSIDE_VPCSC, - "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") - def test_outside_perimeter_blocked(self): - headers = { - "Authorization": "Bearer " + get_access_token(), - "Content-Type": "application/json", - } - self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( - bucket=OUTSIDE_BUCKET) - r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) - resp = json.loads(r.text) - print(resp) - # Assert it returns permission denied from VPC SC - self.assertEqual(resp["error"]["code"], 403) - self.assertEqual(resp["error"]["status"], "PERMISSION_DENIED") - - @unittest.skipUnless( - INSIDE_BUCKET, - "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET not set in environment." - ) - @unittest.skipUnless(IS_INSIDE_VPCSC, - "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment.") - def test_inside_perimeter_allowed(self): - headers = { - "Authorization": "Bearer " + get_access_token(), - "Content-Type": "application/json", - } - self.body["input_uri"] = "gs://{bucket}/cat.mp4".format( - bucket=INSIDE_BUCKET) - r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) - operation = json.loads(r.text) - print(operation) - - get_op_url = "https://videointelligence.googleapis.com/v1/" + operation[ - "name"] - get_op = requests.get(url=get_op_url, headers=headers) - get_op_resp = json.loads(get_op.text) - print(get_op_resp) - # Assert that we do not get an error. - self.assertEqual(get_op_resp["name"], operation["name"]) - + # Tests to verify VideoIntelligence service requests blocked when trying to + # access resources outside of a secure perimeter. + def setUp(self): + VideoIntelligenceSystemTestBase.setUp(self) + # api-endpoint + self.url = "https://videointelligence.googleapis.com/v1/videos:annotate" + self.body = {"features": ["LABEL_DETECTION"], "location_id": "us-west1"} + + @unittest.skipUnless( + OUTSIDE_BUCKET, + "GOOGLE_CLOUD_TESTS_VPCSC_OUTSIDE_PERIMETER_BUCKET not set in environment.", + ) + @unittest.skipUnless( + IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment." + ) + def test_outside_perimeter_blocked(self): + headers = { + "Authorization": "Bearer " + get_access_token(), + "Content-Type": "application/json", + } + self.body["input_uri"] = "gs://{bucket}/cat.mp4".format(bucket=OUTSIDE_BUCKET) + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + resp = json.loads(r.text) + print(resp) + # Assert it returns permission denied from VPC SC + self.assertEqual(resp["error"]["code"], 403) + self.assertEqual(resp["error"]["status"], "PERMISSION_DENIED") + + @unittest.skipUnless( + INSIDE_BUCKET, + "GOOGLE_CLOUD_TESTS_VPCSC_INSIDE_PERIMETER_BUCKET not set in environment.", + ) + @unittest.skipUnless( + IS_INSIDE_VPCSC, "GOOGLE_CLOUD_TESTS_IN_VPCSC not set in environment." + ) + def test_inside_perimeter_allowed(self): + headers = { + "Authorization": "Bearer " + get_access_token(), + "Content-Type": "application/json", + } + self.body["input_uri"] = "gs://{bucket}/cat.mp4".format(bucket=INSIDE_BUCKET) + r = requests.post(url=self.url, data=json.dumps(self.body), headers=headers) + operation = json.loads(r.text) + print(operation) + + get_op_url = "https://videointelligence.googleapis.com/v1/" + operation["name"] + get_op = requests.get(url=get_op_url, headers=headers) + get_op_resp = json.loads(get_op.text) + print(get_op_resp) + # Assert that we do not get an error. + self.assertEqual(get_op_resp["name"], operation["name"])