From 5d5b6f50ab13c3cf40f3de49cff562d94bd3b274 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Wed, 18 Sep 2024 13:48:34 +0200 Subject: [PATCH 1/7] moved (helper) scripts to aforementioned folder --- notebook.ipynb => scripts/notebook.ipynb | 8 ++++++++ prepare.deplyoment.sh => scripts/prepare.deplyoment.sh | 1 + 2 files changed, 9 insertions(+) rename notebook.ipynb => scripts/notebook.ipynb (91%) rename prepare.deplyoment.sh => scripts/prepare.deplyoment.sh (97%) diff --git a/notebook.ipynb b/scripts/notebook.ipynb similarity index 91% rename from notebook.ipynb rename to scripts/notebook.ipynb index ca762b9..13ea40b 100644 --- a/notebook.ipynb +++ b/scripts/notebook.ipynb @@ -9,6 +9,14 @@ "source": [ "%load_ext autoreload\n", "%autoreload 2\n", + "\n", + "# Jump out of the \"Scripts\" box for the lambda source\n", + "import sys\n", + "import os\n", + "parent_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir))\n", + "sys.path.insert(0, parent_dir)\n", + "if not os.path.exists('src'):\n", + " os.chdir('..')\n", "import src.event_gate_lambda" ] }, diff --git a/prepare.deplyoment.sh b/scripts/prepare.deplyoment.sh similarity index 97% rename from prepare.deplyoment.sh rename to scripts/prepare.deplyoment.sh index 0e0a35b..4d65ba7 100644 --- a/prepare.deplyoment.sh +++ b/scripts/prepare.deplyoment.sh @@ -1,3 +1,4 @@ +cd .. pip3 install -r src/requirements.txt -t dependencies/ --platform manylinux2014_x86_64 --python-version 3.12 --only-binary=:all: zip -r lambda_function.zip conf cd src From 25a949a899cffc86396b19fe9e4e24a8b47f58dd Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Wed, 18 Sep 2024 13:51:29 +0200 Subject: [PATCH 2/7] added codeowners --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..4d9d138 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @ABMC831 @OlivieFranklova @Zejnilovic From 7166c8318b29e63e76af6b5d2d9f59deb6a3533e Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Wed, 18 Sep 2024 14:01:11 +0200 Subject: [PATCH 3/7] long-term borrowed issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 25 ++++++++++++++++ .github/ISSUE_TEMPLATE/epic_task.md | 16 ++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 21 +++++++++++++ .github/ISSUE_TEMPLATE/question.md | 13 ++++++++ .github/ISSUE_TEMPLATE/spike_task.md | 36 +++++++++++++++++++++++ 5 files changed, 111 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/epic_task.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/ISSUE_TEMPLATE/spike_task.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..3aa610b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,25 @@ +--- +name: Bug report +about: Create a report to help us improve +labels: 'bug' + +--- + +## Describe the bug +A clear and concise description of what the bug is. + +## To Reproduce +Steps to reproduce the behavior OR commands run: +1. Go to '...' +2. Click on '....' +3. Enter value '...' +4. See error + +## Expected behavior +A clear and concise description of what you expected to happen. + +## Screenshots +If applicable, add screenshots to help explain your problem. + +## Additional context +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/epic_task.md b/.github/ISSUE_TEMPLATE/epic_task.md new file mode 100644 index 0000000..f7411e2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/epic_task.md @@ -0,0 +1,16 @@ +--- +name: Epic +about: A larger task consisting of more deliverables +labels: 'epic' +--- + +## Background +A clear and concise intro into the situation. + +## Goal +The goal that epic wants to achieve. + +```[tasklist] +### Tasks +- [ ] Add a draft title or issue reference here +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..74a56c3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,21 @@ +--- +name: Feature request +about: Suggest an idea for this project +labels: 'enhancement' + +--- + +## Background +A clear and concise description of where the limitation lies. + +## Feature +A description of the requested feature. + +## Example [Optional] +A simple example if applicable. + +## Proposed Solution [Optional] +Solution Ideas: +1. +2. +3. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..24ba89d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,13 @@ +--- +name: Question +about: Ask a question +labels: 'question' + +--- + +## Background [Optional] +A clear explanation of the reason for raising the question. +This gives us a better understanding of your use cases and how we might accommodate them. + +## Question +A clear and concise inquiry diff --git a/.github/ISSUE_TEMPLATE/spike_task.md b/.github/ISSUE_TEMPLATE/spike_task.md new file mode 100644 index 0000000..d071e81 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/spike_task.md @@ -0,0 +1,36 @@ +--- +name: Spike +about: Issue template for spikes, research and investigation tasks +labels: 'spike' + +--- + +## Background +A clear and concise description of the problem or a topic we need to understand. + +Feel free to add information about why it's needed and what assumptions you have at the moment. + +## Questions To Answer + +1. +2. +3. + +## Desired Outcome + +The list of desired outcomes of this spike ticket. + +```[tasklist] +### Tasks +- [ ] Questions have been answered or we have a clearer idea of how to get to our goal +- [ ] Discussion with the team +- [ ] Documentation +- [ ] Create recommendations and new implementation tickets +- [ ] item here.. +``` + +## Additional Info/Resources [Optional] + +1. +2. +3. From 3d5f53ed55b6e21e02d634dcc77b9986c0ceacce Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Wed, 18 Sep 2024 14:20:38 +0200 Subject: [PATCH 4/7] provide readme file --- README.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eadb7fa..3727e45 100644 --- a/README.md +++ b/README.md @@ -1 +1,61 @@ -# EventGate \ No newline at end of file +# EventGate +Python lambda for sending well-defined messages to confluent kafka +assumes AWS Deployment with API Gateway exposure of endpoint + + +- [Lambda itself](#lambda-itself) +- [API](#api) +- [Config](#config) +- [Terraform Deplyoment](#terraform-deplyoment) +- [Scripts](#scripts) + + +## Lambda itself +Hearth of the solution lies in the Src folder + +## API +POST 🔒 method is guarded by JWT token in standard header "bearer" + +| Method | Endpoint | Info | +|---------|-----------------------|------------------------------------------------------------------------------| +| GET | `/token` | forwards (HTTP3003) caller to where to obtain JWT token for posting to topic | +| GET | `/topics` | lists available topics | +| GET | `/topics/{topicName}` | schema for given topic | +| POST 🔒 | `/topics/{topicName}` | posts payload (after authorization and schema validation) into kafka topic | +| POST | `terminate` | kills lambda - useful for when forcing config reload is desired | + + +## Config +There are 3 configs for this solution (in conf folder) + +- config.json +- - this one needs to live in the conf folder +- - defines where are other resources/configs +- access.json +- - this one could be local or in AWS S3 +- - defines who has access to post to individual topics +- topics.json +- - this one could be local or in AWS S3 +- - defines schema of the topics, as well as enumerates those + +## Terraform Deplyoment +Whole solution expects to be deployed as lambda in AWS, +there are prepared terraform scripts to make initial deplyoment, and can be found in "terraform" fodler +All that is needed is supplementing variables for + - aws_region + - vpc_id + - resource prefix - all terraform resources would be prefixed my this prefix, usefull when mixed-in with something else + - lambda_role_arn - the role for the lambda, should be able to make HTTP calls to wherever kafka server lives + - lambda_vpc_subnet_ids + +Once tfvars are supplied, go terraform apply and you are done + +## Scripts +Useful scripts for dev and Deployment + +### Notebook +Jupyter notebook, with one cell for lambda initialization and one cell per method, for testing purposes +Obviously using it requires correct configs to be in place (PUBLIC key is being loaded during initilization) + +### Preapare Deployment +shell script for fetching pithon requirements and ziping it together with sources and config into lambda archive, ready to be used by terraform From bde6a6818ae00c240b73a3aada8e1c6062f5d8ce Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Wed, 18 Sep 2024 14:31:43 +0200 Subject: [PATCH 5/7] schema consolidation --- conf/topics.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/topics.json b/conf/topics.json index f051455..d1c9a1e 100644 --- a/conf/topics.json +++ b/conf/topics.json @@ -13,7 +13,7 @@ "description": "Environment", "type": "string" }, - "id": { + "guid": { "description": "Unique identifier for the event (GUID)", "type": "string" }, @@ -76,7 +76,7 @@ "description": "Format of the data", "type": "string" }, - "id": { + "guid": { "description": "Unique identifier for the event (GUID)", "type": "string" }, From d2fc60a8e6b51eb7cb0099627f56e3dc16108551 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Fri, 20 Sep 2024 09:01:29 +0200 Subject: [PATCH 6/7] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Saša Zejnilović --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3727e45..33ed7de 100644 --- a/README.md +++ b/README.md @@ -29,14 +29,15 @@ POST 🔒 method is guarded by JWT token in standard header "bearer" There are 3 configs for this solution (in conf folder) - config.json -- - this one needs to live in the conf folder -- - defines where are other resources/configs + - this one needs to live in the conf folder + - defines where are other resources/configs - access.json -- - this one could be local or in AWS S3 -- - defines who has access to post to individual topics + - this one could be local or in AWS S3 + - defines who has access to post to individual topics - topics.json -- - this one could be local or in AWS S3 -- - defines schema of the topics, as well as enumerates those + - this one could be local or in AWS S3 + - defines schema of the topics, as well as enumerates those + ## Terraform Deplyoment Whole solution expects to be deployed as lambda in AWS, From 4103c3622f9814ee0d1ee101378a03d18f61cb44 Mon Sep 17 00:00:00 2001 From: "Miroslav Chomut (CZ)" Date: Fri, 20 Sep 2024 09:01:40 +0200 Subject: [PATCH 7/7] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Saša Zejnilović --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 33ed7de..2f85d24 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ POST 🔒 method is guarded by JWT token in standard header "bearer" | Method | Endpoint | Info | |---------|-----------------------|------------------------------------------------------------------------------| -| GET | `/token` | forwards (HTTP3003) caller to where to obtain JWT token for posting to topic | +| GET | `/token` | forwards (HTTP303) caller to where to obtain JWT token for posting to topic | + | GET | `/topics` | lists available topics | | GET | `/topics/{topicName}` | schema for given topic | | POST 🔒 | `/topics/{topicName}` | posts payload (after authorization and schema validation) into kafka topic |