From fda040af25ae1c847bed5b6a693608f03afcc473 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 15 Jun 2023 14:28:59 -0600 Subject: [PATCH 1/2] Added SWE packet definition and dockerfile to packet decom and for future CDK development too. --- Docker-compose.yml | 12 + Dockerfile | 32 ++ README.docker.md | 167 ++++++ .../science_block_20221116_163611Z_idle.bin | Bin 0 -> 29762 bytes in_situ/swe/swe_decom.py | 20 + in_situ/swe/swe_packet_definition.xml | 502 ++++++++++++++++++ 6 files changed, 733 insertions(+) create mode 100644 Docker-compose.yml create mode 100644 Dockerfile create mode 100644 README.docker.md create mode 100644 in_situ/swe/science_block_20221116_163611Z_idle.bin create mode 100644 in_situ/swe/swe_decom.py create mode 100644 in_situ/swe/swe_packet_definition.xml diff --git a/Docker-compose.yml b/Docker-compose.yml new file mode 100644 index 00000000..706ab21e --- /dev/null +++ b/Docker-compose.yml @@ -0,0 +1,12 @@ +version: "3.9" +services: + imap_processing_dev: + build: + context: . + dockerfile: Dockerfile # path to Dockerfile + image: imap_processing_workspace + stdin_open: true # This line is same as docker run -i + tty: true # This line is same as docker run -t + volumes: # mount volumes + - ./:/workspaces/cdk-workspace/ + - $HOME/.aws:/workspaces/cdk-workspace/.aws/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b2bd23af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# OS +ARG VARIANT=bullseye +FROM --platform=linux/amd64 mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} + +# Updates +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install -y firefox-esr +RUN sudo apt-get update +RUN sudo apt-get install -y libgtk-3-dev + +# Download and install NodeJS which helps to install AWS CDK library +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - && sudo apt-get install -y nodejs +RUN npm install -g aws-cdk + +# Download and install AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN unzip awscliv2.zip +RUN sudo ./aws/install + +# Install Python and pip +RUN sudo apt-get install -y python3 python3-pip + +# Install Pip libraries +RUN pip install aws-cdk-lib==2.48.0 +RUN pip install "constructs>=10.0.0,<11.0.0" +RUN pip install boto3==1.26.143 +RUN pip install pyyaml==6.0 +RUN pip install space_packet_parser==4.0.2 + +# Set environment and working directory +ENV AWS_CONFIG_FILE=/workspaces/cdk-workspace/.aws/config +ENV AWS_SHARED_CREDENTIALS_FILE=/workspaces/cdk-workspace/.aws/credentials +WORKDIR /workspaces/cdk-workspace/ diff --git a/README.docker.md b/README.docker.md new file mode 100644 index 00000000..cf671c43 --- /dev/null +++ b/README.docker.md @@ -0,0 +1,167 @@ +# Docker + +# Build ECR Image Locally Using Docker + +From my personal and professional experience, I have encountered challenges when trying to access AWS resources while developing code locally within a Docker image. I have observed individuals resorting to copying their AWS credentials directly into the repository path where the Dockerfile is located and then incorporating those credentials into the Docker image. However, this approach poses security risks for several reasons. + +Recently, I devised a safer solution. Instead of storing AWS credentials within the Docker image, I recommend mounting the path where your AWS credentials reside into Docker's home path during runtime. This approach ensures that the credentials are not embedded within the Docker image itself, reducing the potential exposure of sensitive information. + +By adopting this method, you can securely access AWS resources during development within a Docker environment without compromising the integrity of your credentials. + +## Pre Setup + +To ensure proper AWS credential setup, follow these steps: + +* Confirm that your AWS credentials are correctly configured in the `~/.aws/` directory. + +To install the AWS Command Line Interface (CLI) within a Docker environment, you have a few options: + +* If you are using a Dockerfile, add a line to install the awscli directly. +* If your project uses a requirements.txt file to install pip libraries, include awscli as a dependency in the requirements.txt file. +* Alternatively, if the docker build process takes a long time when awscli is added to the requirements.txt, you can install it manually inside the Docker container. Once inside the container, run the command `pip install awscli`. + +These steps ensure that the AWS CLI is properly installed within the Docker environment, allowing seamless interaction with AWS services. + +## Local Development Using Docker + +To build a Docker image using the standard approach, execute the command: +``` +docker build -t : . +``` +This assumes that you are in the folder where the Dockerfile resides. + +If you need to build the Docker image from a location outside the folder where the Dockerfile is located, utilize the -f argument. For example: +``` +docker build -t : -f . +``` +These commands enable the creation of Docker images for local development. The standard approach assumes that the Dockerfile is present in the current directory, while the second command allows you to specify the path to the Dockerfile if it is located outside the current folder. + +### Docker with Volumn Mount, AWS credentials, and Environment Variable +To achieve live code editing within a Docker container and temporary AWS resource access, you can utilize the following Docker command: + +``` +docker run -it -v : -v $HOME/.aws:/.aws/ --env-file :latest /bin/bash +``` + +This command provides the following functionalities: + +1. The `-v :` option mounts the local content into the Docker image, allowing you to make code changes from your preferred IDE. Any modifications made to the files within the IDE will be automatically reflected inside the Docker container. This eliminates the need to rebuild the image each time you want to test code changes. + +2. The `-v $HOME/.aws:/.aws/` option mounts your AWS credentials into the Docker container, facilitating temporary access to AWS resources from within the container. This enables you to bypass the steps of uploading the image to ECR, triggering Batch jobs, and waiting for their completion to determine if your code is successful or not. By setting up this local environment, you can accelerate your development cycle and save time. Please note that you might need to replace `` with the absolute home path by running the image once and determining the correct value. + +3. The `--env-file ` option allows you to set up runtime environment variables, similar to how you would configure them when running a Batch job. Create a file named `[name].env`, such as `batch.env`, and define all the required environment variables along with their corresponding values. For example: + ``` + XTCE_SCHEMA_S3_PATH=L0_data_products/sci_packet_xtce_schema.xml + DYNAMODB_TABLE_NAME=ProcessingEvents + OUTPUT_FILE_PREFIX=tenzin_test + ``` + These variables can be passed into the Batch job during runtime. + +This Docker command provides a convenient way to edit code live, access AWS resources within the container, and configure necessary environment variables for smooth development and testing. + +## Docker-Compose Approach +We can enhance and simplify the aforementioned approach by incorporating Docker Compose. By utilizing Docker Compose, the setup of AWS credentials becomes more streamlined. + +Here is an example of the contents of a docker-compose.yml file: +``` +version: "3.9" +services: + imap_processing_cdk: + build: + context: ./ + dockerfile: . # path to Dockerfile + image: imap_processing_workspace + stdin_open: true # This line is same as docker run -i + tty: true # This line is same as docker run -t + volumes: # mount volumes + - ./:/workspaces/cdk-workspace/ + - $HOME/.aws:/workspaces/cdk-workspace/.aws/ + environment: + - AWS_PROFILE=$AWS_PROFILE + - AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN + - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + env_file: # Set environment variables using .env file + - .//batch.env +``` + +To build the image, execute the following command: +``` +docker-compose build +``` + +To run an interactive Docker container locally, use this command: +``` +docker-compose run /bin/bash +``` + +### Build Context Explanation + +If the Dockerfile is being built with files located within the same directory, there is no need to set context and dockerfile. Instead, simply set build to the path where the Dockerfile resides. + +If files are being copied from a different folder outside of where the Dockerfile is located, you can build the Docker images using either of the following methods: + +1. Create the docker-compose.yml file in the same path where the Dockerfile exists. +2. Create the docker-compose.yml file in the same directory level as the folder from which the files are being copied. + +In the first case, we define the build location for Docker using the following syntax. The "context" parameter keeps track of the directory path, while the "dockerfile" parameter specifies the exact path to the Dockerfile. + +In this particular case, the context is set to `./../../../../`. The context path starts from the directory where the docker-compose.yml file is located. By setting it to `./../../../../`, we are instructing it to start from the path of the docker-compose.yml file and navigate four directories up to reach the desired directory. + +Once we have changed the build directory by setting the context value, we specify the path to the Dockerfile using the "dockerfile" parameter. The "dockerfile" value is set to the path where the Dockerfile exists. + +Eg. + +``` +version: "3.9" +services: + service_name: + build: + context: ./../../../../ + dockerfile: src/processing/batch/some_job/Dockerfile + image: image_name + stdin_open: true # This line is same as docker run -i + tty: true # This line is same as docker run -t + volumes: # mount volumes + - ./:/src/ + - $HOME/.aws:/root/.aws/ + env_file: # Set environment variables using .env file + - batch.env +``` + + +In the second case, we place the docker-compose.yml file in the same folder as the directory from which we are copying files. Then, we set the context to `.` which represents the current directory. By doing so, we indicate that we want to start the build process from the current directory. + +Next, we specify the path to the Dockerfile using the "dockerfile" parameter. The "dockerfile" value is set to the location where the Dockerfile exists, allowing Docker to find and use the specified Dockerfile for building the container. + +Eg. + +``` +version: "3.9" +services: + service_name: + build: + context: . + dockerfile: src/processing/batch/some_job/Dockerfile + image: pixel_binning # docker image name + stdin_open: true # This line is same as docker run -i + tty: true # This line is same as docker run -t + volumes: # mount volumes + - ./:/var/task/ + - $HOME/.aws:/root/.aws/ + env_file: # Set environment variables using .env file + - l1a_env.env +``` + + +## Steps to test AWS Lambda Image locally + +1. Build image same standard way +2. Run lambda image using this command (with value changed as needed) + ``` + docker run -p 9000:8080 -v $HOME/.aws:/root/.aws/ --env-file aws_env.env + ``` +3. To invoke the lambda image, run this curl command (with event input value as needed) + ``` + curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"Bucket": "sth", ..., "otherKey": "otherValue"}' + ``` diff --git a/in_situ/swe/science_block_20221116_163611Z_idle.bin b/in_situ/swe/science_block_20221116_163611Z_idle.bin new file mode 100644 index 0000000000000000000000000000000000000000..0d6b5133d3e68e067dd8834848cf90b2bc8e61c1 GIT binary patch literal 29762 zcmeI5Ux-vy9LH~3^bi(ekhvlvg3+I?2VtBvovm#gvz!&0hNW63D654nBcWQtUC6*; zYw@9nhFHUTSeOSvwrP4PG$=@=2Vd+(!R>iBXoK)U?VR)ZuKyl%H}BkYcQ|t&J`cs2 znP9(t?)U!AnI*Yn*}LvuzTx4s%Qvj}a__(AR@}2;xVyXAaYtlci&jT7_Uh|(uNvqA z6)*-GUtZSFaZWZ5#&H0sDUm}0b`&6hCt$L{<_~@eJ3M()j$`h zfHBYjL!bvtfI(0MIpBHC-pF1(U;+$+BFKS0Py=0{0>(fC3;|zjMw9lcfi6%1W1s? zPuny+d$!rJ=-M>wC&0!*0StgT7y)IF1yxW28IT8_*Ce^JpbAPL1M;9BjDrFg0Cg|| z%D~r}1lTwzfB{ekBcKejpbAPL1M;9BB);Y+k-f?!R~A%331mPX^n-Cw00W>7Mu6uv zNv?5F00W>7MnD;4K^2rh2IN6M@V2H)!z=+-1tpLHdC(8WK>-YaIv4?Eko20xH@BNK z-00HqqDjN4b#J|6)9~`;X2;@d(@=6J)sy&110{u0Ly#GB( zE~%cxPZ}sGlp;zPrH_(HX`rM~iYQ_9wI%^3)sy&110{u0L_|VU#{f{G@@BLMftz(c79X4W)V#KWU((P>LvFls-!Qq=Aw`DWZfi z=`}NdMJ5e5*))7)iAlqYAO7LeaOu)!hYiCkcgm7Gsa}?hlLkr(rHB$n>7!&)8Yn50 zB6?mEkCW<2{G@@B!VLe4gi-n^@hkJsB!yB$31i}G{#Le^gi-n^nUn@f3Z;k=Mqg`^hEhF=pEOWXC`FVoN*^VD z(m+X}6j8$HZB3VkQay>EG*D6~MU*f~A0>X$KuMt#QNoz?nz_m%lZH>ZG+bp9@kGI; z;hHr!_J+8tqCbWA{PXW7h_l2yKHjz^T_kQ8dc!F6hLZ_)kuD9B?)c{a_fCS{>~rSJ z-`=lHP6?`JH=7?tFE8gkrp1q^FLLw{1xm zi5rIAa5Bxkp?fa00Q${ae$L|7mrWXOacQ{4q+#dhuU@jfVX1VrH*|}%VaG=v=@D`5 zI%U(tWIbBegJwN;mQ3pLu^uMt(Xt*i`&u*fhM_l{Oumb3pWg6?QIm!_mxe`?hFhY& zhin?|-hHJveDS2cJa>G!VZ2==`ImUd$J@4~izE$0Zy2SxH^g?+-*3+Sb8_>TbjR0< zE;6@&xk9j?Cq0 zHn1hVVec7}hEKaR++)&k*Wo`$Z5q~Uv%TT!_UjEv^fD0=dc!D<^M)<#B3W9G+!O!q zo@WDZ*`5858Af-fob+s9D`psr(@&T*e8#2Wut~!`%c`qv8rJJqdqekSLvPp&yR4~v^(>{ zK4m>xZpmyQDWgZ(G3|YBqu%g?$X=fJ6q9k<0u$2=BWb9uC1IcPOa$Z!j6wT3iF1ko;Pft-te;l zlZIPe8Xh-kc=V%vKiM?2JM-FopE7plYc`N~kEx41vBO@T_Y@O~@$P*l4cnqOCD82d-*zxkPvQ37uiO= zAzvl*hO=8s(tN(5drrLo&SAXe=gd!iWzz6DmxePY4JSwM?6YZTcjnFZhHjB|+NZ3J zk5~%xhH0KR487q3*#Fx;z2QGUnl#*Q)9}&7CJkqzy{ zsKWV%P03^#wM@tPhI|&!XAyF98}FHk{7r^=!^u>hZ|HXBc{+=5{?7?W?VIhV&b_#)Z!*jqMv}=?{VvLV=k4YDH%#0^fS9|; z_UR4ZJ7UtX=+dy;q~ZEEzCL8r(C*9&^M(uUd_xj8rWr;S^L(}=cgkVjaP~*`!tbKY N= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CCSDS Packet Version Number + + + CCSDS Packet Type Indicator + + + CCSDS Packet Secondary Header Flag + + + CCSDS Packet Application Process ID + + + CCSDS Packet Grouping Flag + + + CCSDS Packet Sequence Count + + + CCSDS Packet length + + + + Mission Elasped Time + + + Acquisition Start time coarse in seconds + + + Acquisition Start time fine in microseconds + + + + CEM Nominal status bit: + '1' -- nominal, + '0' -- not nomimal + + + + + Spin phase valid bit: + '1' -- valid, + '0' -- invalid + + + + + Spin period source bit: + '1' -- sun sensor (safing), + '0' -- star tracker (nominal) + + + + Spare + + + Spin phase + + + Spin period + + + ESA steps in packet + + + ESA table identifier + + + ESA acquisition configuration + + + HVPS settling duration + + + Acquisition duration + + + Threshold DAC value + + + Stim pulse configuration register. Bits 0-3, period, 4-7, duration, 8-9 mode + + + + + Data for a science acquisiton quarter cycle. (56 bit * 180 array) + Seven consecutive bytes represent data for a single step: + CEM1 - 8 bit counter + CEM2 - 8 bit counter + CEM3 - 8 bit counter + CEM4 - 8 bit counter + CEM5 - 8 bit counter + CEM6 - 8 bit counter + CEM7 - 8 bit counter + 180 steps are for the 180 ESA voltages used in a quarter cycle + + + + + + Operation mode + + + Command accumulation count + + + Command reject count + + + Last opcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Checksum + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 4519efa9e6e7d68c83e55e023e19b71b71de2536 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Fri, 16 Jun 2023 08:28:30 -0600 Subject: [PATCH 2/2] couple minor comments update --- Docker-compose.yml | 4 ++-- in_situ/swe/swe_packet_definition.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Docker-compose.yml b/Docker-compose.yml index 706ab21e..733a128b 100644 --- a/Docker-compose.yml +++ b/Docker-compose.yml @@ -5,8 +5,8 @@ services: context: . dockerfile: Dockerfile # path to Dockerfile image: imap_processing_workspace - stdin_open: true # This line is same as docker run -i - tty: true # This line is same as docker run -t + stdin_open: true # This line is same as `docker run -i` + tty: true # This line is same as `docker run -t` volumes: # mount volumes - ./:/workspaces/cdk-workspace/ - $HOME/.aws:/workspaces/cdk-workspace/.aws/ diff --git a/in_situ/swe/swe_packet_definition.xml b/in_situ/swe/swe_packet_definition.xml index cb6f15ff..2b91ef26 100644 --- a/in_situ/swe/swe_packet_definition.xml +++ b/in_situ/swe/swe_packet_definition.xml @@ -402,7 +402,7 @@ + this particular packet definition. Same logic is used in all SequenceContainer after this. -->