From faafe4469018ade22e44c413e51fd4786c4ed5c3 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Thu, 10 Oct 2024 13:20:06 +0200 Subject: [PATCH 1/5] #20 Add Swagger definitions --- .gitignore | 5 ++ scripts/notebook.ipynb | 15 +++- src/event_gate_lambda.py | 145 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5c2fa73..dc63698 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ .ipynb_checkpoints .venv __pycache__ +/dependencies +/lambda_function.zip +/terraform/*.tfvars +/terraform/*.tfstate* +/terraform/.terraform* diff --git a/scripts/notebook.ipynb b/scripts/notebook.ipynb index 13ea40b..bb56c04 100644 --- a/scripts/notebook.ipynb +++ b/scripts/notebook.ipynb @@ -30,6 +30,19 @@ "jwtToken = \"eyJhb...\"" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ae218c5-8174-41bc-be5d-9487d68260c5", + "metadata": {}, + "outputs": [], + "source": [ + "src.event_gate_lambda.lambda_handler({\n", + " \"httpMethod\": \"GET\",\n", + " \"resource\": \"/api\"\n", + "}, {})" + ] + }, { "cell_type": "code", "execution_count": null, @@ -128,7 +141,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.12.6" } }, "nbformat": 4, diff --git a/src/event_gate_lambda.py b/src/event_gate_lambda.py index a86723d..b3b3818 100644 --- a/src/event_gate_lambda.py +++ b/src/event_gate_lambda.py @@ -82,6 +82,149 @@ def kafkaWrite(topicName, message): logger.info("OK") return 202 +def getApi(): + return { + "statusCode": 200, + "body": """openapi: 3.0.0 +info: + title: Event Gate + version: 0.0.0 + description: This API provides topic management for an event bus. + +servers: + - url: https://{id}-vpce-{vpce}.execute-api.{region}.amazonaws.com/DEV + variables: + id: + default: 01234567ab + description: API Gateway ID + vpce: + default: '01234567abcdef012' + description: VPC endpoint + region: + default: 'af-south-1' + description: AWS Region + +paths: + /token: + get: + summary: Login to the service + description: Allows a user to obtain credentials (JWT token) from LoginService for the service + responses: + '303': + description: Redirect to actual address of Loing service which performs auth up to its capabilities + + /topics: + get: + summary: Get a list of topics + description: Returns a list of all available topics. + responses: + '200': + description: A list of topics + content: + application/json: + schema: + type: array + items: + type: string + + /topics/{topicName}: + get: + summary: Get schema for a specific topic + description: Returns the schema for a specified topic using [JSON Schema](https://json-schema.org/). + parameters: + - name: topicName + in: path + required: true + schema: + type: string + description: Name of the topic + responses: + '200': + description: Key-value pairs representing the schema for the topic + content: + application/json: + schema: + type: object + additionalProperties: + type: string + '404': + description: Topic not found + content: + application/json: + schema: + type: object + properties: + error: + type: string + + put: + summary: Publish an event to a topic + description: Publishes an event to the event bus under the specified topic. User must be authenticated with a JWT token. + security: + - bearerAuth: [] + parameters: + - name: topicName + in: path + required: true + schema: + type: string + description: Name of the topic + requestBody: + description: Event data to be published + required: true + content: + application/json: + schema: + type: object + additionalProperties: true + responses: + '200': + description: Event successfully published + '400': + description: Invalid event data + content: + application/json: + schema: + type: object + properties: + error: + type: string + '401': + description: Unauthorized + content: + application/json: + schema: + type: object + properties: + error: + type: string + '403': + description: Forbidden + content: + application/json: + schema: + type: object + properties: + error: + type: string + '404': + description: Topic not found + content: + application/json: + schema: + type: object + properties: + error: + type: string + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT""" + } + def getToken(): logger.info("Handling GET Token") return { @@ -139,6 +282,8 @@ def postTopicMessage(topicName, topicMessage, tokenEncoded): def lambda_handler(event, context): try: + if event["resource"].lower() == "/api": + return getApi() if event["resource"].lower() == "/token": return getToken() if event["resource"].lower() == "/topics": From ff8c7adcaadd7ce20291f470d3b5081446852650 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Thu, 10 Oct 2024 13:34:00 +0200 Subject: [PATCH 2/5] terraform update as well --- src/event_gate_lambda.py | 14 +++++++++++++- terraform/api_gateway.tf | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/event_gate_lambda.py b/src/event_gate_lambda.py index b3b3818..c546b99 100644 --- a/src/event_gate_lambda.py +++ b/src/event_gate_lambda.py @@ -105,6 +105,18 @@ def getApi(): description: AWS Region paths: + /api: + get: + summary: Open API specification + description: Open API specification + responses: + '200': + description: Open API specification + content: + application/vnd.oai.openapi: + schema: + type: string + /token: get: summary: Login to the service @@ -282,7 +294,7 @@ def postTopicMessage(topicName, topicMessage, tokenEncoded): def lambda_handler(event, context): try: - if event["resource"].lower() == "/api": + if event["resource"].lower() == "/oas3": return getApi() if event["resource"].lower() == "/token": return getToken() diff --git a/terraform/api_gateway.tf b/terraform/api_gateway.tf index 029aae5..591f3e7 100644 --- a/terraform/api_gateway.tf +++ b/terraform/api_gateway.tf @@ -24,6 +24,28 @@ resource "aws_api_gateway_rest_api" "event_gate_api" { }) } +resource "aws_api_gateway_resource" "event_gate_api_api" { + rest_api_id = aws_api_gateway_rest_api.event_gate_api.id + parent_id = aws_api_gateway_rest_api.event_gate_api.root_resource_id + path_part = "api" +} + +resource "aws_api_gateway_method" "event_gate_api_api_get" { + rest_api_id = aws_api_gateway_rest_api.event_gate_api.id + resource_id = aws_api_gateway_resource.event_gate_api_api.id + authorization = "NONE" + http_method = "GET" +} + +resource "aws_api_gateway_integration" "event_gate_api_api_get_integration" { + rest_api_id = aws_api_gateway_rest_api.event_gate_api.id + resource_id = aws_api_gateway_resource.event_gate_api_api.id + http_method = aws_api_gateway_method.event_gate_api_api_get.http_method + integration_http_method = "POST" + type = "AWS_PROXY" + uri = aws_lambda_function.event_gate_lambda.invoke_arn +} + resource "aws_api_gateway_resource" "event_gate_api_token" { rest_api_id = aws_api_gateway_rest_api.event_gate_api.id parent_id = aws_api_gateway_rest_api.event_gate_api.root_resource_id From 3a4d6cb18f356c2fcaf1de8154eb06f1bdf89876 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Thu, 10 Oct 2024 15:20:28 +0200 Subject: [PATCH 3/5] missed terraform dependency --- terraform/api_gateway.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/api_gateway.tf b/terraform/api_gateway.tf index 591f3e7..ed86457 100644 --- a/terraform/api_gateway.tf +++ b/terraform/api_gateway.tf @@ -167,6 +167,7 @@ resource "aws_api_gateway_deployment" "event_gate_api_deployment" { rest_api_id = aws_api_gateway_rest_api.event_gate_api.id triggers = { redeployment = sha1(jsonencode([ + aws_api_gateway_integration.event_gate_api_api_get_integration, aws_api_gateway_integration.event_gate_api_token_get_integration, aws_api_gateway_integration.event_gate_api_topics_get_integration, aws_api_gateway_integration.event_gate_api_topic_name_get_integration, From 8ad8886d7817648eff0a3724e2b1d7e41ee29ead Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Thu, 10 Oct 2024 16:04:45 +0200 Subject: [PATCH 4/5] lambda sourced from S3 --- README.md | 13 ++++++++++--- terraform/lambda.tf | 4 ++-- terraform/variables.tf | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f286a3c..b0be3ac 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ POST 🔒 method is guarded by JWT token in standard header "bearer" | Method | Endpoint | Info | |---------|-----------------------|------------------------------------------------------------------------------| +| GET | `/api` | OpenAPI 3 definition | | GET | `/token` | forwards (HTTP303) caller to where to obtain JWT token for posting to topic | | GET | `/topics` | lists available topics | | GET | `/topics/{topicName}` | schema for given topic | @@ -41,11 +42,16 @@ There are 3 configs for this solution (in conf folder) ## Terraform Deplyoment Whole solution expects to be deployed as lambda in AWS, -there are prepared terraform scripts to make initial deplyoment, and can be found in "terraform" fodler -All that is needed is supplementing variables for +there are prepared terraform scripts to make initial deplyoment, and can be found in "terraform" folder + +Resulting lambda_function zip file needs to be uploaded to aws s3 bucket (since direct upload of zip likes to fail, might be related to poor network though) + +All that is needed afterwards is supplementing variables for - aws_region - vpc_id + - vpc_endpoint - resource prefix - all terraform resources would be prefixed my this prefix, usefull when mixed-in with something else + - lambda_source_bucket - the bucket where "lambda_function.zip" is already uploaded - lambda_role_arn - the role for the lambda, should be able to make HTTP calls to wherever kafka server lives - lambda_vpc_subnet_ids @@ -59,4 +65,5 @@ Jupyter notebook, with one cell for lambda initialization and one cell per metho Obviously using it requires correct configs to be in place (PUBLIC key is being loaded during initilization) ### Preapare Deployment -shell script for fetching pithon requirements and ziping it together with sources and config into lambda archive, ready to be used by terraform +shell script for fetching pithon requirements and ziping it together with sources and config into lambda archive +it needs to be uploaded to s3 bucket first before running the terraform diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 03bb64b..2cc94b5 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -12,11 +12,11 @@ resource "aws_vpc_security_group_egress_rule" "allow_all_traffic_ipv4" { } resource "aws_lambda_function" "event_gate_lambda" { - filename = "../lambda_function.zip" + s3_bucket = var.lambda_source_bucket + s3_key = "lambda_function.zip" function_name = "${var.resource_prefix}event-gate-lambda" role = var.lambda_role_arn handler = "event_gate_lambda.lambda_handler" - source_code_hash = filebase64sha256("../lambda_function.zip") runtime = "python3.12" vpc_config { subnet_ids = var.lambda_vpc_subnet_ids diff --git a/terraform/variables.tf b/terraform/variables.tf index a8e8010..c749624 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -4,3 +4,4 @@ variable "vpc_endpoint" {} variable "resource_prefix" {} variable "lambda_role_arn" {} variable "lambda_vpc_subnet_ids" {} +variable "lambda_source_bucket" {} From 9ddb9cd4988dd1003ca4aa7b2f5561d54daadaf2 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Thu, 17 Oct 2024 15:04:52 +0200 Subject: [PATCH 5/5] OpenAPI definition in config file --- conf/api.yaml | 165 +++++++++++++++++++++++++++++++++++++++ src/event_gate_lambda.py | 156 ++---------------------------------- 2 files changed, 170 insertions(+), 151 deletions(-) create mode 100644 conf/api.yaml diff --git a/conf/api.yaml b/conf/api.yaml new file mode 100644 index 0000000..660d825 --- /dev/null +++ b/conf/api.yaml @@ -0,0 +1,165 @@ +openapi: 3.0.0 +info: + title: Event Gate + version: 0.0.0 + description: This API provides topic management for an event bus. + +servers: + - url: https://{id}-vpce-{vpce}.execute-api.{region}.amazonaws.com/DEV + variables: + id: + default: 01234567ab + description: API Gateway ID + vpce: + default: '01234567abcdef012' + description: VPC endpoint + region: + default: 'af-south-1' + description: AWS Region + +paths: + /api: + get: + summary: Open API specification + description: Open API specification + responses: + '200': + description: Open API specification + content: + application/vnd.oai.openapi: + schema: + type: string + + /token: + get: + summary: Login to the service + description: Allows a user to obtain credentials (JWT token) from LoginService for the service + responses: + '303': + description: Redirect to actual address of Loing service which performs auth up to its capabilities + + /topics: + get: + summary: Get a list of topics + description: Returns a list of all available topics. + responses: + '200': + description: A list of topics + content: + application/json: + schema: + type: array + items: + type: string + + /topics/{topicName}: + get: + summary: Get schema for a specific topic + description: Returns the schema for a specified topic using [JSON Schema](https://json-schema.org/). + parameters: + - name: topicName + in: path + required: true + schema: + type: string + description: Name of the topic + responses: + '200': + description: Key-value pairs representing the schema for the topic + content: + application/json: + schema: + type: object + additionalProperties: + type: string + '404': + description: Topic not found + content: + application/json: + schema: + type: object + properties: + error: + type: string + + put: + summary: Publish an event to a topic + description: Publishes an event to the event bus under the specified topic. User must be authenticated with a JWT token. + security: + - bearerAuth: [] + parameters: + - name: topicName + in: path + required: true + schema: + type: string + description: Name of the topic + requestBody: + description: Event data to be published + required: true + content: + application/json: + schema: + type: object + additionalProperties: true + responses: + '200': + description: Event successfully published + '400': + description: Invalid event data + content: + application/json: + schema: + type: object + properties: + error: + type: string + '401': + description: Unauthorized + content: + application/json: + schema: + type: object + properties: + error: + type: string + '403': + description: Forbidden + content: + application/json: + schema: + type: object + properties: + error: + type: string + '404': + description: Topic not found + content: + application/json: + schema: + type: object + properties: + error: + type: string + + /terminate: + get: + summary: Terminates lambda environment + description: Facilitates fresh start of lambda environment on next invocation (i.e. loads fresh public keys and configs) + responses: + '502': + description: Internal server error or bad gateway error + content: + application/json: + schema: + type: object + properties: + error: + type: string + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT \ No newline at end of file diff --git a/src/event_gate_lambda.py b/src/event_gate_lambda.py index c546b99..1cf876c 100644 --- a/src/event_gate_lambda.py +++ b/src/event_gate_lambda.py @@ -33,6 +33,9 @@ logger.setLevel(logging.INFO) logger.addHandler(logging.StreamHandler()) +with open("conf/api.yaml", "r") as file: + API = file.read() + with open("conf/config.json", "r") as file: CONFIG = json.load(file) @@ -85,156 +88,7 @@ def kafkaWrite(topicName, message): def getApi(): return { "statusCode": 200, - "body": """openapi: 3.0.0 -info: - title: Event Gate - version: 0.0.0 - description: This API provides topic management for an event bus. - -servers: - - url: https://{id}-vpce-{vpce}.execute-api.{region}.amazonaws.com/DEV - variables: - id: - default: 01234567ab - description: API Gateway ID - vpce: - default: '01234567abcdef012' - description: VPC endpoint - region: - default: 'af-south-1' - description: AWS Region - -paths: - /api: - get: - summary: Open API specification - description: Open API specification - responses: - '200': - description: Open API specification - content: - application/vnd.oai.openapi: - schema: - type: string - - /token: - get: - summary: Login to the service - description: Allows a user to obtain credentials (JWT token) from LoginService for the service - responses: - '303': - description: Redirect to actual address of Loing service which performs auth up to its capabilities - - /topics: - get: - summary: Get a list of topics - description: Returns a list of all available topics. - responses: - '200': - description: A list of topics - content: - application/json: - schema: - type: array - items: - type: string - - /topics/{topicName}: - get: - summary: Get schema for a specific topic - description: Returns the schema for a specified topic using [JSON Schema](https://json-schema.org/). - parameters: - - name: topicName - in: path - required: true - schema: - type: string - description: Name of the topic - responses: - '200': - description: Key-value pairs representing the schema for the topic - content: - application/json: - schema: - type: object - additionalProperties: - type: string - '404': - description: Topic not found - content: - application/json: - schema: - type: object - properties: - error: - type: string - - put: - summary: Publish an event to a topic - description: Publishes an event to the event bus under the specified topic. User must be authenticated with a JWT token. - security: - - bearerAuth: [] - parameters: - - name: topicName - in: path - required: true - schema: - type: string - description: Name of the topic - requestBody: - description: Event data to be published - required: true - content: - application/json: - schema: - type: object - additionalProperties: true - responses: - '200': - description: Event successfully published - '400': - description: Invalid event data - content: - application/json: - schema: - type: object - properties: - error: - type: string - '401': - description: Unauthorized - content: - application/json: - schema: - type: object - properties: - error: - type: string - '403': - description: Forbidden - content: - application/json: - schema: - type: object - properties: - error: - type: string - '404': - description: Topic not found - content: - application/json: - schema: - type: object - properties: - error: - type: string - -components: - securitySchemes: - bearerAuth: - type: http - scheme: bearer - bearerFormat: JWT""" + "body": API } def getToken(): @@ -294,7 +148,7 @@ def postTopicMessage(topicName, topicMessage, tokenEncoded): def lambda_handler(event, context): try: - if event["resource"].lower() == "/oas3": + if event["resource"].lower() == "/api": return getApi() if event["resource"].lower() == "/token": return getToken()