From 6f5f17a9985e3c31694829f3c871cb91ee6ccd9e Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Wed, 21 Apr 2021 00:30:24 +0000 Subject: [PATCH 1/7] fix local development with the Pub/Sub emulator --- src/functions_framework/__init__.py | 2 +- src/functions_framework/event_conversion.py | 62 ++++++++- tests/test_convert.py | 125 ++++++++++++++++-- tests/test_functions.py | 28 ++++ .../test_functions/background_trigger/main.py | 5 +- 5 files changed, 207 insertions(+), 15 deletions(-) diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index 3bff83f0..9f5cf34e 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -126,7 +126,7 @@ def view_func(path): function(data, context) else: # This is a regular CloudEvent - event_data = request.get_json() + event_data = event_conversion.marshal_background_event_data(request) if not event_data: flask.abort(400) event_object = BackgroundEvent(**event_data) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index 4d914802..c5ebeae7 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -13,7 +13,8 @@ # limitations under the License. import re -from typing import Tuple +from datetime import datetime +from typing import Optional, Tuple from cloudevents.http import CloudEvent @@ -55,6 +56,12 @@ _PUBSUB_CE_SERVICE = "pubsub.googleapis.com" _STORAGE_CE_SERVICE = "storage.googleapis.com" +# Raw pubsub types +_PUBSUB_EVENT_TYPE = 'google.pubsub.topic.publish' +_PUBSUB_MESSAGE_TYPE = 'type.googleapis.com/google.pubsub.v1.PubsubMessage' + +_PUBSUB_TOPIC_REQUEST_PATH = re.compile(r"projects\/[^/?]+\/topics\/[^/?]+") + # Maps background event services to their equivalent CloudEvent services. _SERVICE_BACKGROUND_TO_CE = { "providers/cloud.firestore/": _FIRESTORE_CE_SERVICE, @@ -90,7 +97,7 @@ def background_event_to_cloudevent(request) -> CloudEvent: """Converts a background event represented by the given HTTP request into a CloudEvent. """ - event_data = request.get_json() + event_data = marshal_background_event_data(request) if not event_data: raise EventConversionException("Failed to parse JSON") @@ -109,6 +116,10 @@ def background_event_to_cloudevent(request) -> CloudEvent: # Handle Pub/Sub events. if service == _PUBSUB_CE_SERVICE: data = {"message": data} + # It is possible to configure a Pub/Sub subscription to push directly to this function + # without passing the topic name in the URL path. + if resource is None: + resource = "" # Handle Firebase Auth events. if service == _FIREBASE_AUTH_CE_SERVICE: @@ -168,3 +179,50 @@ def _split_resource(context: Context) -> Tuple[str, str, str]: raise EventConversionException("Resource regex did not match") return service, match.group(1), match.group(2) + + +def marshal_background_event_data(request): + """Marshal the request body of a raw Pub/Sub HTTP request into the schema that is expected of + a background event""" + request_data = request.get_json() + if not _is_raw_pubsub_payload(request_data): + # If this in not a raw Pub/Sub request, return the unaltered request data. + return request_data + + return { + "context": { + "eventId": request_data["message"]["messageId"], + "timestamp": datetime.utcnow().isoformat() + "Z", + "eventType": _PUBSUB_EVENT_TYPE, + "resource": { + "service": _PUBSUB_CE_SERVICE, + "type": _PUBSUB_MESSAGE_TYPE, + "name": _parse_pubsub_topic(request.path), + }, + }, + "data": { + '@type': _PUBSUB_MESSAGE_TYPE, + "data": request_data["message"]["data"], + "attributes": request_data["message"]["attributes"], + } + } + + +def _is_raw_pubsub_payload(request_data) -> bool: + """Does the given request body match the schema of a unmarshalled Pub/Sub request""" + return ( + request_data is not None and + "context" not in request_data and + "subscription" in request_data and + "message" in request_data and + "data" in request_data["message"] and + "messageId" in request_data["message"] + ) + + +def _parse_pubsub_topic(request_path) -> Optional[str]: + match = _PUBSUB_TOPIC_REQUEST_PATH.search(request_path) + if match: + return match.group(0) + else: + return None \ No newline at end of file diff --git a/tests/test_convert.py b/tests/test_convert.py index d3fe3e22..5f31fdbd 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -65,23 +65,67 @@ BACKGROUND_RESOURCE_STRING = "projects/_/buckets/some-bucket/objects/folder/Test.cs" +PUBSUB_CLOUD_EVENT = { + "specversion": "1.0", + "id": "1215011316659232", + "source": "//pubsub.googleapis.com/projects/sample-project/topics/gcf-test", + "time": "2020-05-18T12:13:19Z", + "type": "google.cloud.pubsub.topic.v1.messagePublished", + "datacontenttype": "application/json", + "data": { + "message": { + "data": "10", + }, + }, +} + @pytest.fixture def pubsub_cloudevent_output(): - event = { - "specversion": "1.0", - "id": "1215011316659232", - "source": "//pubsub.googleapis.com/projects/sample-project/topics/gcf-test", - "time": "2020-05-18T12:13:19Z", - "type": "google.cloud.pubsub.topic.v1.messagePublished", - "datacontenttype": "application/json", + return from_json(json.dumps(PUBSUB_CLOUD_EVENT)) + +@pytest.fixture +def raw_pubsub_request(): + return { + "subscription": "projects/sample-project/subscriptions/gcf-test-sub", + "message": { + "data": "eyJmb28iOiJiYXIifQ==", + "messageId": "1215011316659232", + "attributes": { + "test": "123" + } + } + } + +@pytest.fixture +def marshalled_pubsub_request(): + return { "data": { - "message": { - "data": "10", - }, + "@type": "type.googleapis.com/google.pubsub.v1.PubsubMessage", + "data": "eyJmb28iOiJiYXIifQ==", + "attributes": { + "test": "123" + } }, + "context": { + "eventId": "1215011316659232", + "eventType": "google.pubsub.topic.publish", + "resource": { + "name": "projects/sample-project/topics/gcf-test", + "service": "pubsub.googleapis.com", + "type": "type.googleapis.com/google.pubsub.v1.PubsubMessage" + }, + "timestamp": "2021-04-17T07:21:18.249Z", + } } +@pytest.fixture +def raw_pubsub_cloudevent_output(marshalled_pubsub_request): + event = PUBSUB_CLOUD_EVENT.copy() + # the data payload is more complex for the raw pubsub request + event["data"] = { + "message": marshalled_pubsub_request["data"] + } return from_json(json.dumps(event)) @@ -212,3 +256,64 @@ def test_split_resource_no_resource_regex_match(): with pytest.raises(EventConversionException) as exc_info: event_conversion._split_resource(context) assert "Resource regex did not match" in exc_info.value.args[0] + + +def test_marshal_background_event_data_without_topic_in_path( + raw_pubsub_request, marshalled_pubsub_request +): + req = flask.Request.from_values(json=raw_pubsub_request, path="/myfunc/") + payload = event_conversion.marshal_background_event_data(req) + + # Remove timestamps as they get generates on the fly + del marshalled_pubsub_request["context"]["timestamp"] + del payload["context"]["timestamp"] + + # Resource name is set to None when it cannot be parsed from the request path + marshalled_pubsub_request["context"]["resource"]["name"] = None + + assert payload == marshalled_pubsub_request + +def test_marshal_background_event_data_with_topic_path( + raw_pubsub_request, marshalled_pubsub_request +): + req = flask.Request.from_values( + json=raw_pubsub_request, path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true" + ) + payload = event_conversion.marshal_background_event_data(req) + + # Remove timestamps as they are generated on the fly. + del marshalled_pubsub_request["context"]["timestamp"] + del payload["context"]["timestamp"] + + assert payload == marshalled_pubsub_request + +def test_pubsub_emulator_request_to_cloudevent(raw_pubsub_request, raw_pubsub_cloudevent_output): + req = flask.Request.from_values( + json=raw_pubsub_request, + path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true" + ) + cloudevent = event_conversion.background_event_to_cloudevent(req) + + # Remove timestamps as they are generated on the fly. + del raw_pubsub_cloudevent_output['time'] + del cloudevent['time'] + + assert cloudevent == raw_pubsub_cloudevent_output + + +def test_pubsub_emulator_request_to_cloudevent_without_topic_path( + raw_pubsub_request, raw_pubsub_cloudevent_output +): + req = flask.Request.from_values( + json=raw_pubsub_request, path="/" + ) + cloudevent = event_conversion.background_event_to_cloudevent(req) + + # Remove timestamps as they are generated on the fly. + del raw_pubsub_cloudevent_output['time'] + del cloudevent['time'] + + # Default to the service name, when the topic is not configured subscription's pushEndpoint. + raw_pubsub_cloudevent_output['source'] = "//pubsub.googleapis.com/" + + assert cloudevent == raw_pubsub_cloudevent_output \ No newline at end of file diff --git a/tests/test_functions.py b/tests/test_functions.py index 5f746931..375ea192 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -48,6 +48,19 @@ def background_json(tmpdir): "data": {"filename": str(tmpdir / "filename.txt"), "value": "some-value"}, } +@pytest.fixture +def pubsub_emulator_request_payload(tmpdir): + return { + "subscription": "projects/FOO/subscriptions/BAR_SUB", + "message": { + "data": 'eyJmb28iOiJiYXIifQ==', + "messageId": "1", + "attributes": { + "filename": str(tmpdir / "filename.txt"), + "value": "some-value" + } + } + } def test_http_function_executes_success(): source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" @@ -242,6 +255,21 @@ def test_pubsub_payload(background_json): background_json["data"]["value"] ) +def test_pubsub_emulator_payload(pubsub_emulator_request_payload): + source = TEST_FUNCTIONS_DIR / "background_trigger" / "main.py" + target = "function" + + client = create_app(target, source, "event").test_client() + + resp = client.post("/", json=pubsub_emulator_request_payload) + + assert resp.status_code == 200 + assert resp.data == b"OK" + + with open(pubsub_emulator_request_payload["message"]["attributes"]["filename"]) as f: + assert f.read() == '{{"entryPoint": "function", "value": "{}"}}'.format( + pubsub_emulator_request_payload["message"]["attributes"]["value"] + ) def test_background_function_no_data(background_json): source = TEST_FUNCTIONS_DIR / "background_trigger" / "main.py" diff --git a/tests/test_functions/background_trigger/main.py b/tests/test_functions/background_trigger/main.py index 842c4889..00d60bb9 100644 --- a/tests/test_functions/background_trigger/main.py +++ b/tests/test_functions/background_trigger/main.py @@ -29,8 +29,9 @@ def function( data dictionary. context (google.cloud.functions.Context): The Cloud Functions event context. """ - filename = event["filename"] - value = event["value"] + attributes = event.get("attributes", {}) + filename = event.get("filename", attributes.get("filename")) + value = event.get("value", attributes.get("value")) f = open(filename, "w") f.write('{{"entryPoint": "function", "value": "{}"}}'.format(value)) f.close() From b82500a03c3c2ed7e57511a979fff5385994beda Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Thu, 22 Apr 2021 18:01:55 +0000 Subject: [PATCH 2/7] black formatting fixes --- src/functions_framework/event_conversion.py | 24 +++++------ tests/test_convert.py | 48 ++++++++++----------- tests/test_functions.py | 16 ++++--- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index c5ebeae7..99951af1 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -57,8 +57,8 @@ _STORAGE_CE_SERVICE = "storage.googleapis.com" # Raw pubsub types -_PUBSUB_EVENT_TYPE = 'google.pubsub.topic.publish' -_PUBSUB_MESSAGE_TYPE = 'type.googleapis.com/google.pubsub.v1.PubsubMessage' +_PUBSUB_EVENT_TYPE = "google.pubsub.topic.publish" +_PUBSUB_MESSAGE_TYPE = "type.googleapis.com/google.pubsub.v1.PubsubMessage" _PUBSUB_TOPIC_REQUEST_PATH = re.compile(r"projects\/[^/?]+\/topics\/[^/?]+") @@ -188,7 +188,7 @@ def marshal_background_event_data(request): if not _is_raw_pubsub_payload(request_data): # If this in not a raw Pub/Sub request, return the unaltered request data. return request_data - + return { "context": { "eventId": request_data["message"]["messageId"], @@ -201,22 +201,22 @@ def marshal_background_event_data(request): }, }, "data": { - '@type': _PUBSUB_MESSAGE_TYPE, + "@type": _PUBSUB_MESSAGE_TYPE, "data": request_data["message"]["data"], "attributes": request_data["message"]["attributes"], - } + }, } def _is_raw_pubsub_payload(request_data) -> bool: """Does the given request body match the schema of a unmarshalled Pub/Sub request""" return ( - request_data is not None and - "context" not in request_data and - "subscription" in request_data and - "message" in request_data and - "data" in request_data["message"] and - "messageId" in request_data["message"] + request_data is not None + and "context" not in request_data + and "subscription" in request_data + and "message" in request_data + and "data" in request_data["message"] + and "messageId" in request_data["message"] ) @@ -225,4 +225,4 @@ def _parse_pubsub_topic(request_path) -> Optional[str]: if match: return match.group(0) else: - return None \ No newline at end of file + return None diff --git a/tests/test_convert.py b/tests/test_convert.py index 5f31fdbd..e9db2407 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -84,6 +84,7 @@ def pubsub_cloudevent_output(): return from_json(json.dumps(PUBSUB_CLOUD_EVENT)) + @pytest.fixture def raw_pubsub_request(): return { @@ -91,21 +92,18 @@ def raw_pubsub_request(): "message": { "data": "eyJmb28iOiJiYXIifQ==", "messageId": "1215011316659232", - "attributes": { - "test": "123" - } - } + "attributes": {"test": "123"}, + }, } + @pytest.fixture def marshalled_pubsub_request(): return { "data": { "@type": "type.googleapis.com/google.pubsub.v1.PubsubMessage", "data": "eyJmb28iOiJiYXIifQ==", - "attributes": { - "test": "123" - } + "attributes": {"test": "123"}, }, "context": { "eventId": "1215011316659232", @@ -113,19 +111,18 @@ def marshalled_pubsub_request(): "resource": { "name": "projects/sample-project/topics/gcf-test", "service": "pubsub.googleapis.com", - "type": "type.googleapis.com/google.pubsub.v1.PubsubMessage" + "type": "type.googleapis.com/google.pubsub.v1.PubsubMessage", }, "timestamp": "2021-04-17T07:21:18.249Z", - } + }, } + @pytest.fixture def raw_pubsub_cloudevent_output(marshalled_pubsub_request): event = PUBSUB_CLOUD_EVENT.copy() # the data payload is more complex for the raw pubsub request - event["data"] = { - "message": marshalled_pubsub_request["data"] - } + event["data"] = {"message": marshalled_pubsub_request["data"]} return from_json(json.dumps(event)) @@ -273,11 +270,13 @@ def test_marshal_background_event_data_without_topic_in_path( assert payload == marshalled_pubsub_request + def test_marshal_background_event_data_with_topic_path( raw_pubsub_request, marshalled_pubsub_request ): req = flask.Request.from_values( - json=raw_pubsub_request, path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true" + json=raw_pubsub_request, + path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true", ) payload = event_conversion.marshal_background_event_data(req) @@ -287,16 +286,19 @@ def test_marshal_background_event_data_with_topic_path( assert payload == marshalled_pubsub_request -def test_pubsub_emulator_request_to_cloudevent(raw_pubsub_request, raw_pubsub_cloudevent_output): + +def test_pubsub_emulator_request_to_cloudevent( + raw_pubsub_request, raw_pubsub_cloudevent_output +): req = flask.Request.from_values( json=raw_pubsub_request, - path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true" + path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true", ) cloudevent = event_conversion.background_event_to_cloudevent(req) # Remove timestamps as they are generated on the fly. - del raw_pubsub_cloudevent_output['time'] - del cloudevent['time'] + del raw_pubsub_cloudevent_output["time"] + del cloudevent["time"] assert cloudevent == raw_pubsub_cloudevent_output @@ -304,16 +306,14 @@ def test_pubsub_emulator_request_to_cloudevent(raw_pubsub_request, raw_pubsub_cl def test_pubsub_emulator_request_to_cloudevent_without_topic_path( raw_pubsub_request, raw_pubsub_cloudevent_output ): - req = flask.Request.from_values( - json=raw_pubsub_request, path="/" - ) + req = flask.Request.from_values(json=raw_pubsub_request, path="/") cloudevent = event_conversion.background_event_to_cloudevent(req) # Remove timestamps as they are generated on the fly. - del raw_pubsub_cloudevent_output['time'] - del cloudevent['time'] + del raw_pubsub_cloudevent_output["time"] + del cloudevent["time"] # Default to the service name, when the topic is not configured subscription's pushEndpoint. - raw_pubsub_cloudevent_output['source'] = "//pubsub.googleapis.com/" + raw_pubsub_cloudevent_output["source"] = "//pubsub.googleapis.com/" - assert cloudevent == raw_pubsub_cloudevent_output \ No newline at end of file + assert cloudevent == raw_pubsub_cloudevent_output diff --git a/tests/test_functions.py b/tests/test_functions.py index 375ea192..8d560ed4 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -48,20 +48,22 @@ def background_json(tmpdir): "data": {"filename": str(tmpdir / "filename.txt"), "value": "some-value"}, } + @pytest.fixture def pubsub_emulator_request_payload(tmpdir): return { "subscription": "projects/FOO/subscriptions/BAR_SUB", "message": { - "data": 'eyJmb28iOiJiYXIifQ==', + "data": "eyJmb28iOiJiYXIifQ==", "messageId": "1", "attributes": { "filename": str(tmpdir / "filename.txt"), - "value": "some-value" - } - } + "value": "some-value", + }, + }, } + def test_http_function_executes_success(): source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" target = "function" @@ -255,6 +257,7 @@ def test_pubsub_payload(background_json): background_json["data"]["value"] ) + def test_pubsub_emulator_payload(pubsub_emulator_request_payload): source = TEST_FUNCTIONS_DIR / "background_trigger" / "main.py" target = "function" @@ -266,11 +269,14 @@ def test_pubsub_emulator_payload(pubsub_emulator_request_payload): assert resp.status_code == 200 assert resp.data == b"OK" - with open(pubsub_emulator_request_payload["message"]["attributes"]["filename"]) as f: + with open( + pubsub_emulator_request_payload["message"]["attributes"]["filename"] + ) as f: assert f.read() == '{{"entryPoint": "function", "value": "{}"}}'.format( pubsub_emulator_request_payload["message"]["attributes"]["value"] ) + def test_background_function_no_data(background_json): source = TEST_FUNCTIONS_DIR / "background_trigger" / "main.py" target = "function" From 6cdef95d332deaab51bd11b7af8189c4064ee96d Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Thu, 22 Apr 2021 18:05:46 +0000 Subject: [PATCH 3/7] set name to empty string when it cannot be parsed --- src/functions_framework/event_conversion.py | 8 +++----- tests/test_convert.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index 99951af1..27ceb8d4 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -116,10 +116,6 @@ def background_event_to_cloudevent(request) -> CloudEvent: # Handle Pub/Sub events. if service == _PUBSUB_CE_SERVICE: data = {"message": data} - # It is possible to configure a Pub/Sub subscription to push directly to this function - # without passing the topic name in the URL path. - if resource is None: - resource = "" # Handle Firebase Auth events. if service == _FIREBASE_AUTH_CE_SERVICE: @@ -225,4 +221,6 @@ def _parse_pubsub_topic(request_path) -> Optional[str]: if match: return match.group(0) else: - return None + # It is possible to configure a Pub/Sub subscription to push directly to this function + # without passing the topic name in the URL path. + return "" diff --git a/tests/test_convert.py b/tests/test_convert.py index e9db2407..03b2f3b8 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -265,8 +265,8 @@ def test_marshal_background_event_data_without_topic_in_path( del marshalled_pubsub_request["context"]["timestamp"] del payload["context"]["timestamp"] - # Resource name is set to None when it cannot be parsed from the request path - marshalled_pubsub_request["context"]["resource"]["name"] = None + # Resource name is set to empty string when it cannot be parsed from the request path + marshalled_pubsub_request["context"]["resource"]["name"] = "" assert payload == marshalled_pubsub_request From ad9f94f75e807e13242386ee5057d7663bd61fa4 Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Thu, 22 Apr 2021 18:20:42 +0000 Subject: [PATCH 4/7] convert key error to EventConversionException --- src/functions_framework/event_conversion.py | 35 +++++++++++---------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index 27ceb8d4..4e365b70 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -185,23 +185,26 @@ def marshal_background_event_data(request): # If this in not a raw Pub/Sub request, return the unaltered request data. return request_data - return { - "context": { - "eventId": request_data["message"]["messageId"], - "timestamp": datetime.utcnow().isoformat() + "Z", - "eventType": _PUBSUB_EVENT_TYPE, - "resource": { - "service": _PUBSUB_CE_SERVICE, - "type": _PUBSUB_MESSAGE_TYPE, - "name": _parse_pubsub_topic(request.path), + try: + return { + "context": { + "eventId": request_data["message"]["messageId"], + "timestamp": datetime.utcnow().isoformat() + "Z", + "eventType": _PUBSUB_EVENT_TYPE, + "resource": { + "service": _PUBSUB_CE_SERVICE, + "type": _PUBSUB_MESSAGE_TYPE, + "name": _parse_pubsub_topic(request.path), + }, }, - }, - "data": { - "@type": _PUBSUB_MESSAGE_TYPE, - "data": request_data["message"]["data"], - "attributes": request_data["message"]["attributes"], - }, - } + "data": { + "@type": _PUBSUB_MESSAGE_TYPE, + "data": request_data["message"]["data"], + "attributes": request_data["message"]["attributes"], + }, + } + except (KeyError, AttributeError): + raise EventConversionException("Failed to convert Pub/Sub payload to event") def _is_raw_pubsub_payload(request_data) -> bool: From 17baac22caad69f2bb81582f03ff6c06c9f72c6a Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Thu, 22 Apr 2021 21:57:53 +0000 Subject: [PATCH 5/7] used payload timestamp if it exists, fix test coverage --- src/functions_framework/event_conversion.py | 13 ++++++------- tests/test_convert.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index 4e365b70..d99e6d5e 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -180,16 +180,15 @@ def _split_resource(context: Context) -> Tuple[str, str, str]: def marshal_background_event_data(request): """Marshal the request body of a raw Pub/Sub HTTP request into the schema that is expected of a background event""" - request_data = request.get_json() - if not _is_raw_pubsub_payload(request_data): - # If this in not a raw Pub/Sub request, return the unaltered request data. - return request_data - try: + request_data = request.get_json() + if not _is_raw_pubsub_payload(request_data): + # If this in not a raw Pub/Sub request, return the unaltered request data. + return request_data return { "context": { "eventId": request_data["message"]["messageId"], - "timestamp": datetime.utcnow().isoformat() + "Z", + "timestamp": request_data["message"].get("publishTime", datetime.utcnow().isoformat() + "Z"), "eventType": _PUBSUB_EVENT_TYPE, "resource": { "service": _PUBSUB_CE_SERVICE, @@ -203,7 +202,7 @@ def marshal_background_event_data(request): "attributes": request_data["message"]["attributes"], }, } - except (KeyError, AttributeError): + except (AttributeError, KeyError, TypeError): raise EventConversionException("Failed to convert Pub/Sub payload to event") diff --git a/tests/test_convert.py b/tests/test_convert.py index 03b2f3b8..a6e18f28 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -317,3 +317,15 @@ def test_pubsub_emulator_request_to_cloudevent_without_topic_path( raw_pubsub_cloudevent_output["source"] = "//pubsub.googleapis.com/" assert cloudevent == raw_pubsub_cloudevent_output + + +def test_pubsub_emulator_request_with_invalid_message( + raw_pubsub_request, raw_pubsub_cloudevent_output +): + # Create an invalid message payload + raw_pubsub_request["message"] = None + req = flask.Request.from_values(json=raw_pubsub_request, path="/") + + with pytest.raises(EventConversionException) as exc_info: + cloudevent = event_conversion.background_event_to_cloudevent(req) + assert "Failed to convert Pub/Sub payload to event" in exc_info.value.args[0] From 93181a306da935e9781ab1b04ceeccb04d2cbaea Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Thu, 22 Apr 2021 22:18:39 +0000 Subject: [PATCH 6/7] remove test that required update to test_function --- src/functions_framework/event_conversion.py | 4 ++- tests/test_convert.py | 2 +- tests/test_functions.py | 34 ------------------- .../test_functions/background_trigger/main.py | 5 ++- 4 files changed, 6 insertions(+), 39 deletions(-) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index d99e6d5e..a935c5b5 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -188,7 +188,9 @@ def marshal_background_event_data(request): return { "context": { "eventId": request_data["message"]["messageId"], - "timestamp": request_data["message"].get("publishTime", datetime.utcnow().isoformat() + "Z"), + "timestamp": request_data["message"].get( + "publishTime", datetime.utcnow().isoformat() + "Z" + ), "eventType": _PUBSUB_EVENT_TYPE, "resource": { "service": _PUBSUB_CE_SERVICE, diff --git a/tests/test_convert.py b/tests/test_convert.py index a6e18f28..07580991 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -325,7 +325,7 @@ def test_pubsub_emulator_request_with_invalid_message( # Create an invalid message payload raw_pubsub_request["message"] = None req = flask.Request.from_values(json=raw_pubsub_request, path="/") - + with pytest.raises(EventConversionException) as exc_info: cloudevent = event_conversion.background_event_to_cloudevent(req) assert "Failed to convert Pub/Sub payload to event" in exc_info.value.args[0] diff --git a/tests/test_functions.py b/tests/test_functions.py index 8d560ed4..5f746931 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -49,21 +49,6 @@ def background_json(tmpdir): } -@pytest.fixture -def pubsub_emulator_request_payload(tmpdir): - return { - "subscription": "projects/FOO/subscriptions/BAR_SUB", - "message": { - "data": "eyJmb28iOiJiYXIifQ==", - "messageId": "1", - "attributes": { - "filename": str(tmpdir / "filename.txt"), - "value": "some-value", - }, - }, - } - - def test_http_function_executes_success(): source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" target = "function" @@ -258,25 +243,6 @@ def test_pubsub_payload(background_json): ) -def test_pubsub_emulator_payload(pubsub_emulator_request_payload): - source = TEST_FUNCTIONS_DIR / "background_trigger" / "main.py" - target = "function" - - client = create_app(target, source, "event").test_client() - - resp = client.post("/", json=pubsub_emulator_request_payload) - - assert resp.status_code == 200 - assert resp.data == b"OK" - - with open( - pubsub_emulator_request_payload["message"]["attributes"]["filename"] - ) as f: - assert f.read() == '{{"entryPoint": "function", "value": "{}"}}'.format( - pubsub_emulator_request_payload["message"]["attributes"]["value"] - ) - - def test_background_function_no_data(background_json): source = TEST_FUNCTIONS_DIR / "background_trigger" / "main.py" target = "function" diff --git a/tests/test_functions/background_trigger/main.py b/tests/test_functions/background_trigger/main.py index 00d60bb9..842c4889 100644 --- a/tests/test_functions/background_trigger/main.py +++ b/tests/test_functions/background_trigger/main.py @@ -29,9 +29,8 @@ def function( data dictionary. context (google.cloud.functions.Context): The Cloud Functions event context. """ - attributes = event.get("attributes", {}) - filename = event.get("filename", attributes.get("filename")) - value = event.get("value", attributes.get("value")) + filename = event["filename"] + value = event["value"] f = open(filename, "w") f.write('{{"entryPoint": "function", "value": "{}"}}'.format(value)) f.close() From 77df302a2194b08bad87aa6067e2258e2cef0771 Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Tue, 27 Apr 2021 19:07:49 +0000 Subject: [PATCH 7/7] fix lint error --- src/functions_framework/event_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions_framework/event_conversion.py b/src/functions_framework/event_conversion.py index a935c5b5..596bee2c 100644 --- a/src/functions_framework/event_conversion.py +++ b/src/functions_framework/event_conversion.py @@ -96,7 +96,7 @@ def background_event_to_cloudevent(request) -> CloudEvent: - """Converts a background event represented by the given HTTP request into a CloudEvent. """ + """Converts a background event represented by the given HTTP request into a CloudEvent.""" event_data = marshal_background_event_data(request) if not event_data: raise EventConversionException("Failed to parse JSON")