From 923aa674db45301dc774392e6f7854a600beaf54 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Wed, 11 Mar 2020 12:14:20 +0530 Subject: [PATCH 1/4] feat(storage): add conformance tests --- tests/unit/test__signing.py | 42 +++++++++++++++++-------- tests/unit/url_signer_v4_test_data.json | 24 ++++++++++++++ 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index c3b911f1d..a9d603f00 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -778,8 +778,11 @@ def dummy_service_account(): return _DUMMY_SERVICE_ACCOUNT +_API_ACCESS_ENDPOINT = "https://storage.googleapis.com" + + def _run_conformance_test( - resource, test_data, api_access_endpoint="https://storage.googleapis.com" + resource, test_data, api_access_endpoint=_API_ACCESS_ENDPOINT ): credentials = dummy_service_account() url = Test_generate_signed_url_v4._call_fut( @@ -804,13 +807,20 @@ def test_conformance_client(test_data): @pytest.mark.parametrize("test_data", _BUCKET_TESTS) def test_conformance_bucket(test_data): - if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": - api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( - scheme=test_data["scheme"], - bucket_bound_hostname=test_data["bucketBoundHostname"], - ) + global _API_ACCESS_ENDPOINT + if "urlStyle" in test_data: + if test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) + elif test_data["urlStyle"] == "VIRTUAL_HOSTED_STYLE": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_name}.storage.googleapis.com".format( + scheme=test_data["scheme"], bucket_name=test_data["bucket"] + ) + resource = "/" - _run_conformance_test(resource, test_data, api_access_endpoint) + _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: resource = "/{}".format(test_data["bucket"]) _run_conformance_test(resource, test_data) @@ -818,13 +828,19 @@ def test_conformance_bucket(test_data): @pytest.mark.parametrize("test_data", _BLOB_TESTS) def test_conformance_blob(test_data): - if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": - api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( - scheme=test_data["scheme"], - bucket_bound_hostname=test_data["bucketBoundHostname"], - ) + global _API_ACCESS_ENDPOINT + if "urlStyle" in test_data: + if test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) + elif test_data["urlStyle"] == "VIRTUAL_HOSTED_STYLE": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_name}.storage.googleapis.com".format( + scheme=test_data["scheme"], bucket_name=test_data["bucket"] + ) resource = "/{}".format(test_data["object"]) - _run_conformance_test(resource, test_data, api_access_endpoint) + _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) diff --git a/tests/unit/url_signer_v4_test_data.json b/tests/unit/url_signer_v4_test_data.json index a43fc5cd5..a4b4a51f9 100644 --- a/tests/unit/url_signer_v4_test_data.json +++ b/tests/unit/url_signer_v4_test_data.json @@ -168,5 +168,29 @@ "scheme": "https", "urlStyle": "BUCKET_BOUND_HOSTNAME", "bucketBoundHostname": "mydomain.tld" + }, + + { + "description": "Virtual Hosted Style", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://test-bucket.storage.googleapis.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=06c633ea060b0eda81ee58dd3337b01b0d243a44f18cb03ec948861f533a129e579c7fd4c856d187f1c7b86e5715ea0abf6a1c6ba32b69274d22b1b0406df6847dc87f0d289fe8dc0682351574849b8b13e4b66922f39441af96becb73ea4c56cd5e3eeb30bc91fe84e8bd205adca8639253bdb65b2fcaf2598a230c6d8f6d8177c9e58a61b6e826767f594056b490184d676897c4bbedc15d6fbf08c3fa82a406c62e74db661e6c5d7d3ced29e0619ee719dce4b8136360345b8dce120b9f1debd511c8dac3e6d874ee05bfda8c8f1c4fedd0c07fc6d98f5f18a349bb204d8ff401402a025194e2792df8a09282141157e4ca51d26a8d0d142a01c805321911", + "scheme": "https", + "urlStyle": "VIRTUAL_HOSTED_STYLE" + }, + + { + "description": "Virtual Hosted Style", + "bucket": "test-bucket", + "object": "", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://test-bucket.storage.googleapis.com/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=79b49f12d21293882daaef1d7b0cf81529f302dc29e37d42e15ac1d55b6fe793efc4f53e29a7fe47ca71119fb82768cb34111410029fd5d1a542b8be7e38b54a4164f2fb2a330aa57d0c5762de7f31614c4dda3f1e29ff735a9886464e04e36445a043e5f2e9bac64a588cb339fa75d69dce97ca1322dd4bfe0e7a8fe33cf309f30b35f79e7d4cf78b1a9ed4eb2ce7ba91e662ade58b6e0d516c13d7bdc18244c191cf0bdd55e04bbaca01afb89df02123d0855ecec960d8c52bc792f8e5b36e0e3cd697bf7c9e390bd165b7767d94ba2d5938dc555e3918ac05a9d5c66293b204a104776c03a43d59764cce4a28e85f1e54b36b315367ed8f37ac691ef6a11b", + "scheme": "https", + "urlStyle": "VIRTUAL_HOSTED_STYLE" } ] From b094210041ba374e7b5da0890a10fab7e3277cc1 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Wed, 11 Mar 2020 13:10:49 +0530 Subject: [PATCH 2/4] feat(storage): cover fix --- tests/unit/test__signing.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index a9d603f00..40f3d6dc1 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -814,11 +814,12 @@ def test_conformance_bucket(test_data): scheme=test_data["scheme"], bucket_bound_hostname=test_data["bucketBoundHostname"], ) - elif test_data["urlStyle"] == "VIRTUAL_HOSTED_STYLE": + + # For the VIRTUAL_HOSTED_STYLE + else: _API_ACCESS_ENDPOINT = "{scheme}://{bucket_name}.storage.googleapis.com".format( scheme=test_data["scheme"], bucket_name=test_data["bucket"] ) - resource = "/" _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: @@ -835,14 +836,15 @@ def test_conformance_blob(test_data): scheme=test_data["scheme"], bucket_bound_hostname=test_data["bucketBoundHostname"], ) - elif test_data["urlStyle"] == "VIRTUAL_HOSTED_STYLE": + + # For the VIRTUAL_HOSTED_STYLE + else: _API_ACCESS_ENDPOINT = "{scheme}://{bucket_name}.storage.googleapis.com".format( scheme=test_data["scheme"], bucket_name=test_data["bucket"] ) resource = "/{}".format(test_data["object"]) _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: - resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) _run_conformance_test(resource, test_data) From 0870ee097f73acdcd60d95850c95d71f17fc85d7 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Thu, 12 Mar 2020 11:16:34 +0530 Subject: [PATCH 3/4] feat(storage): nit --- tests/unit/url_signer_v4_test_data.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/url_signer_v4_test_data.json b/tests/unit/url_signer_v4_test_data.json index a4b4a51f9..550f372f0 100644 --- a/tests/unit/url_signer_v4_test_data.json +++ b/tests/unit/url_signer_v4_test_data.json @@ -179,7 +179,9 @@ "timestamp": "20190201T090000Z", "expectedUrl": "https://test-bucket.storage.googleapis.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=06c633ea060b0eda81ee58dd3337b01b0d243a44f18cb03ec948861f533a129e579c7fd4c856d187f1c7b86e5715ea0abf6a1c6ba32b69274d22b1b0406df6847dc87f0d289fe8dc0682351574849b8b13e4b66922f39441af96becb73ea4c56cd5e3eeb30bc91fe84e8bd205adca8639253bdb65b2fcaf2598a230c6d8f6d8177c9e58a61b6e826767f594056b490184d676897c4bbedc15d6fbf08c3fa82a406c62e74db661e6c5d7d3ced29e0619ee719dce4b8136360345b8dce120b9f1debd511c8dac3e6d874ee05bfda8c8f1c4fedd0c07fc6d98f5f18a349bb204d8ff401402a025194e2792df8a09282141157e4ca51d26a8d0d142a01c805321911", "scheme": "https", - "urlStyle": "VIRTUAL_HOSTED_STYLE" + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n89eeae48258eccdcb1f592fb908008e3f5d36a949c002c1e614c94356dc18fc6" }, { @@ -191,6 +193,8 @@ "timestamp": "20190201T090000Z", "expectedUrl": "https://test-bucket.storage.googleapis.com/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=79b49f12d21293882daaef1d7b0cf81529f302dc29e37d42e15ac1d55b6fe793efc4f53e29a7fe47ca71119fb82768cb34111410029fd5d1a542b8be7e38b54a4164f2fb2a330aa57d0c5762de7f31614c4dda3f1e29ff735a9886464e04e36445a043e5f2e9bac64a588cb339fa75d69dce97ca1322dd4bfe0e7a8fe33cf309f30b35f79e7d4cf78b1a9ed4eb2ce7ba91e662ade58b6e0d516c13d7bdc18244c191cf0bdd55e04bbaca01afb89df02123d0855ecec960d8c52bc792f8e5b36e0e3cd697bf7c9e390bd165b7767d94ba2d5938dc555e3918ac05a9d5c66293b204a104776c03a43d59764cce4a28e85f1e54b36b315367ed8f37ac691ef6a11b", "scheme": "https", - "urlStyle": "VIRTUAL_HOSTED_STYLE" + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "expectedCanonicalRequest": "GET\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4a3352bc39ec2a3eec47d568fb05688e66b0d0f88bbe9890fa83f53bf756483e" } ] From 6669e6eb462b09ee164dd2d2727f1126623dc203 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Thu, 12 Mar 2020 13:46:23 +0530 Subject: [PATCH 4/4] feat(storage): remove unnecessary test --- tests/unit/test__signing.py | 17 +++++------------ tests/unit/url_signer_v4_test_data.json | 14 -------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index 40f3d6dc1..c2c8eba54 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -808,18 +808,11 @@ def test_conformance_client(test_data): @pytest.mark.parametrize("test_data", _BUCKET_TESTS) def test_conformance_bucket(test_data): global _API_ACCESS_ENDPOINT - if "urlStyle" in test_data: - if test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": - _API_ACCESS_ENDPOINT = "{scheme}://{bucket_bound_hostname}".format( - scheme=test_data["scheme"], - bucket_bound_hostname=test_data["bucketBoundHostname"], - ) - - # For the VIRTUAL_HOSTED_STYLE - else: - _API_ACCESS_ENDPOINT = "{scheme}://{bucket_name}.storage.googleapis.com".format( - scheme=test_data["scheme"], bucket_name=test_data["bucket"] - ) + if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) resource = "/" _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: diff --git a/tests/unit/url_signer_v4_test_data.json b/tests/unit/url_signer_v4_test_data.json index 550f372f0..2c587f737 100644 --- a/tests/unit/url_signer_v4_test_data.json +++ b/tests/unit/url_signer_v4_test_data.json @@ -182,19 +182,5 @@ "urlStyle": "VIRTUAL_HOSTED_STYLE", "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n89eeae48258eccdcb1f592fb908008e3f5d36a949c002c1e614c94356dc18fc6" - }, - - { - "description": "Virtual Hosted Style", - "bucket": "test-bucket", - "object": "", - "method": "GET", - "expiration": 10, - "timestamp": "20190201T090000Z", - "expectedUrl": "https://test-bucket.storage.googleapis.com/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=79b49f12d21293882daaef1d7b0cf81529f302dc29e37d42e15ac1d55b6fe793efc4f53e29a7fe47ca71119fb82768cb34111410029fd5d1a542b8be7e38b54a4164f2fb2a330aa57d0c5762de7f31614c4dda3f1e29ff735a9886464e04e36445a043e5f2e9bac64a588cb339fa75d69dce97ca1322dd4bfe0e7a8fe33cf309f30b35f79e7d4cf78b1a9ed4eb2ce7ba91e662ade58b6e0d516c13d7bdc18244c191cf0bdd55e04bbaca01afb89df02123d0855ecec960d8c52bc792f8e5b36e0e3cd697bf7c9e390bd165b7767d94ba2d5938dc555e3918ac05a9d5c66293b204a104776c03a43d59764cce4a28e85f1e54b36b315367ed8f37ac691ef6a11b", - "scheme": "https", - "urlStyle": "VIRTUAL_HOSTED_STYLE", - "expectedCanonicalRequest": "GET\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", - "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4a3352bc39ec2a3eec47d568fb05688e66b0d0f88bbe9890fa83f53bf756483e" } ]