diff --git a/devops/01 - git/README.md b/devops/01 - git/README.md index 1fc3f573..bc0fcdd1 100644 --- a/devops/01 - git/README.md +++ b/devops/01 - git/README.md @@ -6,8 +6,12 @@ * fork the current repository +Done + * after passing other test tasks (02 - dockerfile, 03 - docker-compose, 04 - bash, 05 - cloud-ops) need to squash all your commits to one +Done + * make a pull-request with the results of the tasks and tag Done-current date (e.g. `Done-01-01-2021`) @@ -15,14 +19,97 @@ 1. What command can I use to view the commit history? -1. What command can I use to undo the last commit? +git log +<<<<<<< HEAD + +2. What command can I use to undo the last commit? + +git reset HEAD~1 + +3. What command can I use to create a new branch and a new tag? + +git checkout -b NEWBRANCHNAME + +git tag -a NEWTAG + +4. How do I exclude a file / folder from a commit? + +You should to add a name of this file/folder to file .gitignore or ./git/info/exclude + +5. In case of a merge conflict, what commands can be used to resolve it? + +git status + +#to see which files are unmerged + +git diff branch1..branch2 + +#to see difference between branches + +git log branch1...branch2 + +#to see log/commit history after making branch + +git reset HEAD + +#to cancel merge, if it will be useful + +git restore * + +#to restore all files, if it will be useful + +6. `*` What are pre-commit hooks and post-commit hooks, and what are they for? + +pre-commit - checking the correctness of the data before commit. + +post-commit - used to send notifications after commit. + +7. `*` How do I change the last commit without adding a new commit? + +======= + +2. What command can I use to undo the last commit? + +git reset HEAD^1 + +3. What command can I use to create a new branch and a new tag? + +git checkout -b NEWBRANCHNAME + +git tag -a NEWTAG + +4. How do I exclude a file / folder from a commit? + +You should to add a name of this file/folder to file .gitignore or ./git/info/exclude + +5. In case of a merge conflict, what commands can be used to resolve it? + +git status + +#to see which files are unmerged + +git diff branch1..branch2 + +#to see difference between branches + +git log branch1...branch2 + +#to see log/commit history after making branch + +git reset HEAD + +#to cancel merge, if it will be useful + +git restore * + +#to restore all files, if it will be useful -1. What command can I use to create a new branch and a new tag? +6. `*` What are pre-commit hooks and post-commit hooks, and what are they for? -1. How do I exclude a file / folder from a commit? +pre-commit - checking the correctness of the data before commit. -1. In case of a merge conflict, what commands can be used to resolve it? +post-commit - used to send notifications after commit. -1. `*` What are pre-commit hooks and post-commit hooks, and what are they for? +7. `*` How do I change the last commit without adding a new commit? -1. `*` How do I change the last commit without adding a new commit? +git commit --amend diff --git a/devops/02 - dockerfile/.env b/devops/02 - dockerfile/.env deleted file mode 100644 index 65398b6e..00000000 --- a/devops/02 - dockerfile/.env +++ /dev/null @@ -1 +0,0 @@ -PORT=8888 diff --git a/devops/02 - dockerfile/Dockerfile b/devops/02 - dockerfile/Dockerfile index f3258b32..326c6cb3 100644 --- a/devops/02 - dockerfile/Dockerfile +++ b/devops/02 - dockerfile/Dockerfile @@ -1,8 +1,12 @@ -FROM golang:1.15 +FROM golang:1.15 AS builder WORKDIR /usr/src/app +RUN echo PORT=8888>./.env COPY main.go . COPY go.mod . -COPY .env . RUN go get -d github.com/joho/godotenv RUN CGO_ENABLED=0 GOOS=linux go build -o simple-webpage . + +FROM scratch +WORKDIR /usr/src/app +COPY --from=builder /usr/src/app/simple-webpage /usr/src/app/.env ./ CMD ["./simple-webpage"] diff --git a/devops/02 - dockerfile/README.md b/devops/02 - dockerfile/README.md index 7fddb4e8..cdc014fa 100644 --- a/devops/02 - dockerfile/README.md +++ b/devops/02 - dockerfile/README.md @@ -12,18 +12,52 @@ You need to optimize the Dockerfile by correcting or adding steps. 1. What is Docker? Which technology is it based on? +Docker is an application manager with container support. It is based on the LXC technology. + 2. Look at the Docker file – what would you change in it? +I've added Multi-staging and creating .env file inside container. + 3. How do I pass variables to the Docker file when building and running the container? +Through the .env file. + 4. Why do we need multistage build ? +Because we don't need golang environment in a resulting image. One container - one application. + ## Tasks * Dockerfile - generate .env file inside Dockerfile, provide value of port at build step. +Done + * Multi-stage build – change the Dockerfile to make it multi-stage. Try to get the lowest container size. +6.7 MB + * Compare size of docker images with and without multistage build. +without multi-staging it was 858 MB, but with this one it became 6.7 MB + * Write down all commands which you have used. + +cat .env + +nano Dockerfile + +docker pull golang:1.15 + +rm .env + +docker build -t webcounter:1.0 . + +docker run -d --name worker -p80:8888 webcounter:1.0 + +docker stop worker + +nano README.md + +git add . + +git commit -m "02 - dockerfile" diff --git a/devops/03 - docker-compose/README.md b/devops/03 - docker-compose/README.md index 2f2a4f23..4f58f4b2 100644 --- a/devops/03 - docker-compose/README.md +++ b/devops/03 - docker-compose/README.md @@ -10,38 +10,79 @@ Docker compose with 3 applications (frontend + backend + DB). ### Instructions for running -1. Bootstrap the DB: - -`docker-compose up -d db` - -`docker-compose run --rm flaskapp /bin/bash -c "cd /opt/services/flaskapp/src && python -c 'import database; database.init_db()'"` - -2. Boot up the cluster +1. Bootstrap the DB and boot up the cluster `docker-compose up -d` -3. Browse to localhost:8080 to see the app in action. +2. Browse to localhost:8080 to see the app in action. ## Questions 1. What is the difference between Docker Compose and dockerfile? Why do I need Docker Compose? +dockerfile is needed to describe how to build images, but docker compose is needed to describe how to run containers. + 2. How do I parameterize compose for different environments? +create .env file with environment variables and use the vars from this file in docker-compose.yml file. + 3. What types of entities are used in docker-compose and for what purpose? +version - version of docker-compose + +services - services/applications to start in docker containers + +db/flaskapp/nginx - containers names + +image - docker images for services + +volumes - volumes that storaged in docker outside containers for using inside containers + +env_file - file in which described environment variables to export them inside container. + +networks - networks in which container will have interface + +depends_on - services which needed to be started before starting current service + +command - command which will be executed after starting conatiner + +ports - describing port translations + +db_nw/web_nw - describing networks + +driver - selecting driver for network interfaces in used network + 4. `*` How to output application logs? -4. `*` How to copy\upload a file from host machine to the container? +sudo docker-compose logs SERVICE + +5. `*` How to copy\upload a file from host machine to the container? -5. `*` How to save file changes made inside the container? +in Dockerfile: COPY /srcPath/srcFile ./dstPath + +in dockercompose: volumes: - /srcPath/srcFile:/dstPath/dstFile + +on run: docker cp /srcPath/srcFile containerName:/dstPath + +6. `*` How to save file changes made inside the container? + +we can use volume, upload file from container to host machine, use shared folder, save container to image ## Tasks * Docker-compose has a bug - investigate it! What would you improve? +There is a little bug with NGINX config. Also I edited Dockerfile and docker-compose.yml +to build and run application with only "docker-compose up -d" +(but I added wait-for-it.sh script, recommended on docs.docker.com to make dependences between services and +execute commands on flaskapp to create DB after database start) + * Docker-compose with an environment file. Create 2 different environment files for docker-compose +I added environments to docker-compose file. Now it uses env_file and .env files. + * `*` Change the `docker-compose.yml` to run through dockerstack with code reuse (don't repeat yourself) +I edited docker-compose.yml, so now it can work with docker-compose and docker-stack (check this with +docker swarm with 1 host) diff --git a/devops/03 - docker-compose/example/.env b/devops/03 - docker-compose/example/.env new file mode 100644 index 00000000..66c93e20 --- /dev/null +++ b/devops/03 - docker-compose/example/.env @@ -0,0 +1,15 @@ +DB_TAG=9.6.5 +DB_VOLUME=dbdata +ENV_FILE=env_file +DB_NW=db_nw +WEB_NW=web_nw +FLASK_DOCKERFILE=. +DB_DIR=/var/lib/postgresql/data +FLASK_DIR=/opt/services/flaskapp/src +FLASK_PORT=5432 +FLASK_CMD=bash -c "./wait-for-it.sh db:${FLASK_PORT} -- python -c 'import database; database.init_db()' && python app.py" +NGINX_TAG=1.13.5 +PUB_PORT=8080 +NGINX_PORT=80 +SRC_CONFD=./conf.d +NGINX_CONFD=/etc/nginx/conf.d diff --git a/devops/03 - docker-compose/example/Dockerfile b/devops/03 - docker-compose/example/Dockerfile index bcbd0d6f..6024a8d8 100644 --- a/devops/03 - docker-compose/example/Dockerfile +++ b/devops/03 - docker-compose/example/Dockerfile @@ -8,6 +8,4 @@ RUN mkdir -p /opt/services/flaskapp/src COPY requirements.txt /opt/services/flaskapp/src/ WORKDIR /opt/services/flaskapp/src RUN pip install -r requirements.txt -COPY . /opt/services/flaskapp/src EXPOSE 5090 -CMD ["python", "app.py"] diff --git a/devops/03 - docker-compose/example/conf.d/flaskapp.conf b/devops/03 - docker-compose/example/conf.d/flaskapp.conf index 66f28037..f0c14473 100644 --- a/devops/03 - docker-compose/example/conf.d/flaskapp.conf +++ b/devops/03 - docker-compose/example/conf.d/flaskapp.conf @@ -3,11 +3,10 @@ server { server_name localhost; location / { - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; proxy_pass http://flaskapp:5090; } diff --git a/devops/03 - docker-compose/example/docker-compose.yml b/devops/03 - docker-compose/example/docker-compose.yml index 917b4f65..4321aff9 100644 --- a/devops/03 - docker-compose/example/docker-compose.yml +++ b/devops/03 - docker-compose/example/docker-compose.yml @@ -1,38 +1,45 @@ version: '3' services: db: - image: "postgres:9.6.5" + image: "postgres:${DB_TAG:-9.6.5}" + ports: + - "5432:5432" volumes: - - "dbdata:/var/lib/postgresql/data" - env_file: - - env_file + - "dbdata:${DB_DIR:-/var/lib/postgresql/data}" + env_file: &environ_file + - ${ENV_FILE:-env_file} networks: - - db_nw + - ${DB_NW:-db_nw} flaskapp: - build: . - env_file: - - env_file + build: ${FLASK_DOCKERFILE:-.} + image: "${FLASKAPP_IMAGE:-example_flaskapp}:latest" + env_file: *environ_file + ports: + - "5090:5090" volumes: - - .:/opt/services/flaskapp/src + - .:${FLASK_DIR:-/opt/services/flaskapp/src} networks: - - db_nw - - web_nw + - ${DB_NW:-db_nw} + - ${WEB_NW:-web_nw} depends_on: - db + command: ${FLASK_CMD:-bash -c " + ./wait-for-it.sh db:5432 -- python -c 'import database; database.init_db()' + && python app.py" + } nginx: - image: "nginx:1.13.5" + image: "nginx:${NGINX_TAG:-1.13.5}" ports: - - "8080:80" + - "${PUB_PORT:-8080}:${NGINX_PORT:-80}" volumes: - - ./conf.d:/etc/nginx/conf.d + - ${SRC_CONFD:-./conf.d}:${NGINX_CONFD:-/etc/nginx/conf.d} networks: - - web_nw + - ${WEB_NW:-web_nw} depends_on: - flaskapp networks: - db_nw: - driver: bridge - web_nw: - driver: bridge + db_nw: &bridgedriver + driver: overlay + web_nw: *bridgedriver volumes: dbdata: \ No newline at end of file diff --git a/devops/03 - docker-compose/example/wait-for-it.sh b/devops/03 - docker-compose/example/wait-for-it.sh new file mode 100755 index 00000000..d990e0d3 --- /dev/null +++ b/devops/03 - docker-compose/example/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/devops/04 - bash/README.md b/devops/04 - bash/README.md index 3782c2ef..54993c50 100644 --- a/devops/04 - bash/README.md +++ b/devops/04 - bash/README.md @@ -3,9 +3,100 @@ ## Questions * Mention the advantages and disadvantages of bash scripts + +Advantages - high speed and ease of scripting non-difficult task. + +Disadvantages - it may have low speed of script execution and hard to write difficult task. + * What types of variables are used in bash? + +System and user defined variables. Also there are two data types: character and integer. + * What is pipes in shell script and how to use it? + +Pipes are used to redirect stdout of command/application before pipe to stdin of command/application after pipe. +Each command/application will be executed sequentially from first to last separated by pipes. + * How to show unique values of the second column from the CSV comma-separated file and sort results with the alphabet? +``` bash +awk -F "\"*,\"*" FILE.CSV | sort | uniq +``` +It won't work for eggs containing comma. +I.e. for row which equal 'substr1,"substr2,substr3",substr4' according to task we have to print 'substr2,substr3', +but this one will print only 'substr2' +In this way I can offer a little C program for print only second column according to task: +``` bash +./print2col FILE.CSV | sort | uniq +``` +where print2col is program.c file compiled by gcc (`gcc program.c -o print2col`), and `program.c`: +``` C +#include +#include +#include + +int main (int argx, char **argy) +{ + char buf; //buffer for 1 character + char sep; //separator + int fd; //file descriptor + int rd; //state of read execution + int notendline; + + buf = 0; + sep = ','; + if (argx != 2) + return (0); + fd = open(argy[1], O_RDONLY); + if (fd < 0) + return (0); + rd = read(fd, &buf, 1); + while (rd > 0) + { + notendline = 1; + while (rd > 0 && buf != sep) + { + if (buf == '\n') + { + notendline = 0; + break ; + } + rd = read(fd, &buf, 1); + } + if (notendline) + rd = read(fd, &buf, 1); + if (notendline && rd > 0 && buf == '"') + { + rd = read(fd, &buf, 1); + while (rd > 0 && buf != '"') + { + if (buf == '\n') + { + notendline = 0; + break ; + } + write(1, &buf, 1); + rd = read(fd, &buf, 1); + } + if (rd > 0 && buf == '"') + rd = read(fd, &buf, 1); + } + while (notendline && rd > 0 && buf != sep && buf != '\n') + { + write(1, &buf, 1); + rd = read(fd, &buf, 1); + } + while (notendline && rd > 0 && buf != '\n') + rd = read(fd, &buf, 1); + if (rd > 0) + { + write(1, "\n" ,1); + rd = read(fd, &buf, 1); + } + } + close(fd); + return (0); +} +``` ## Task Create script which provide aggregated information about analysises and they datasets. Information about each analysis/deployment could be retrieved from `awscli.sh`: @@ -53,5 +144,35 @@ Need to provide aggregated information for next analysis: `users, billings, usag ## How to If you use OS different than macOS/Linux need to use Docker to launch your script: ``` bash -docker run -it -rm -v :/src -w /src --entrypoint bash cfmanteiga/alpine-bash-curl-jq -- +docker run -it --rm -v :/src -w /src --entrypoint bash cfmanteiga/alpine-bash-curl-jq -- +``` +## Solution +I add `myscript.sh` (without jq) which is working this way: +``` bash +./myscript.sh users +``` +And another one file `script.sh` (with jq) which is working the same way: +``` bash +.\script.sh users +``` + +And each files will display something like this: +``` json +{ + "analysises": { + "users": { + "datasetsCount": 2 + "datasets": [ + { + "arn": "arn:us-west-2:reports:dataset/Zmlyc3REYXRhc2V0Cg==", + "name": "firstDataset" + } + { + "arn": "arn:us-west-2:reports:dataset/c2Vjb25kRGF0YXNldAo=", + "name": "secondDataset" + } + ] + } + } +} ``` diff --git a/devops/04 - bash/myscript.sh b/devops/04 - bash/myscript.sh new file mode 100755 index 00000000..82aaefe2 --- /dev/null +++ b/devops/04 - bash/myscript.sh @@ -0,0 +1,30 @@ +#! /bin/bash +ID=$(echo $1 | base64) +DATASETS=$(./awscli.sh get_analysis $1 |xargs| sed 's/.*datasetsIds\: \[ \(.*\)\].*/\1/'|wc -w) +cat << EOF +{ + "analysises": { + "${1}": { + "datasetsCount": $DATASETS + "datasets": [ +EOF +count=1 +while [ $count -le $DATASETS ] +do + DATASETID=$(./awscli.sh get_analysis users |xargs|\ + sed 's/.*datasetsIds\: \[\(.*\)\].*/\1/'|xargs|\ + awk '{print $('$count')}'|sed 's/,$//') + cat << EOF + { + "arn": "$(./awscli.sh get_dataset $DATASETID |xargs|sed 's/.*datasetArn\: //'|sed 's/ }//')", + "name": "$(./awscli.sh get_dataset $DATASETID |xargs|sed 's/.*name\: //'|sed 's/, datasetArn.*//')" + } +EOF + ((count++)) +done +cat << EOF + ] + } + } +} +EOF diff --git a/devops/04 - bash/script.sh b/devops/04 - bash/script.sh new file mode 100755 index 00000000..70cfe0ad --- /dev/null +++ b/devops/04 - bash/script.sh @@ -0,0 +1,28 @@ +#! /bin/bash +ID=$(echo $1 | base64) +DATASETS=$(./awscli.sh get_analysis $1 | jq '.datasetsIds' | jq length) +cat << EOF +{ + "analysises": { + "${1}": { + "datasetsCount": $DATASETS + "datasets": [ +EOF +count=0 +while [ $count -lt $DATASETS ] +do + DATASETID=$(./awscli.sh get_analysis $1 | jq '.datasetsIds['$count']'|xargs) + cat << EOF + { + "arn": $(./awscli.sh get_dataset $DATASETID | jq '.datasetArn'), + "name": $(./awscli.sh get_dataset $DATASETID | jq '.name') + } +EOF + ((count++)) +done +cat << EOF + ] + } + } +} +EOF diff --git a/devops/05 - cloud-ops/README.md b/devops/05 - cloud-ops/README.md index aa9d9773..f1a8fd90 100644 --- a/devops/05 - cloud-ops/README.md +++ b/devops/05 - cloud-ops/README.md @@ -13,11 +13,17 @@ https://aws.amazon.com/free/?all-free-tier.sort-by=item.additionalFields.SortRan * Create an AWS account and protect the root user with MFA https://docs.aws.amazon.com/IAM/latest/UserGuide/id_root-user.html#id_root-user_manage_mfa +Already done (in May 2021). + * Create an EC2 instance, connect to it through SSH and install apache or nginx manually, terminate it then. +Done. + * Create an EC2 instance, provision software (apache/nginx) using Cloudformation, validate the installation, finally delete the stack. Provide the resulting template as outcome of this task. +My configuration is in `aws_cf_nginx_template.json`. + * (*) Given that we have a properly written Cloudformation template come up with an idea how we can create and update a cloudformation stack based on this template automatically. Try to implement it. @@ -25,6 +31,23 @@ https://aws.amazon.com/free/?all-free-tier.sort-by=item.additionalFields.SortRan 1. What the benefits of the cloud computing? +With the cloud we don't need to buy expensive infrastructure. +It won't be useless while we aren't using them. +It will not get loss in cost for us cause we don't own this. +When we need to do some computing, just take access to camputing power in some cloud service, +and other cloud service users can use it the rest of the time. +We don't have to maintain hardware infrastructure. +Finally, the cloud system can be distributed so we can get +access to our services or infrastructure wherever we are. +And this one can be high perfomance, scaleable and fault resistant. + 2. What is the Infrastructure As Code? Pros/cons of it? +We can have hardware infrastructure in our own. We will support this with all the consequences. +Or we can have configuration file which describes components of hardware infrastructure and their interaction. +When we have to do something with our "hardware" we just edit this config file or operate with some interface. +It will save our funds, time and many other resources. +But it can be difficult to work for some users. We can't implement some new features until it will be accessable on +your cloud service. Also there are some troubles to diagnostic, and troubleshoot this. +IaC can be really complex, difficult to understand and maintain this one. diff --git a/devops/05 - cloud-ops/aws_cf_nginx_template.json b/devops/05 - cloud-ops/aws_cf_nginx_template.json new file mode 100644 index 00000000..c9c863df --- /dev/null +++ b/devops/05 - cloud-ops/aws_cf_nginx_template.json @@ -0,0 +1,177 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "AWS nginx", + "Parameters": { + "KeyName": { + "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance", + "Type": "AWS::EC2::KeyPair::KeyName", + "ConstraintDescription": "must be the name of an existing EC2 KeyPair." + }, + "InstanceType": { + "Description": "WebServer EC2 instance type", + "Type": "String", + "Default": "t2.micro", + "AllowedValues": [ + "t2.micro", + ], + "ConstraintDescription": "must be a valid EC2 instance type." + }, + "SSHLocation": { + "Description": " The IP address range that can be used to SSH to the EC2 instances", + "Type": "String", + "MinLength": "9", + "MaxLength": "18", + "Default": "0.0.0.0/0", + "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", + "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." + } + }, + "Mappings": { + "AWSInstanceType2Arch": { + "t2.micro": { + "Arch": "HVM64" + }, + }, + "AWSRegionArch2AMI": { + "us-east-2": { + "HVM64": "ami-027cab9a7bf0155df", + "HVMG2": "NOT_SUPPORTED" + } + } + }, + "Resources": { + "WebServerInstance": { + "Type": "AWS::EC2::Instance", + "Metadata": { + "AWS::CloudFormation::Init": { + "config": { + "packages": { + "yum": { + "nginx": [] + } + }, + "services": { + "sysvinit": { + "nginx": { + "enabled": "true", + "ensureRunning": "true" + } + } + } + }, + } + }, + "Properties": { + "ImageId": { + "Fn::FindInMap": [ + "AWSRegionArch2AMI", + { + "Ref": "AWS::Region" + }, + { + "Fn::FindInMap": [ + "AWSInstanceType2Arch", + { + "Ref": "InstanceType" + }, + "Arch" + ] + } + ] + }, + "InstanceType": { + "Ref": "InstanceType" + }, + "SecurityGroups": [ + { + "Ref": "WebServerSecurityGroup" + } + ], + "KeyName": { + "Ref": "KeyName" + }, + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash -xe\n", + "yum update -y aws-cfn-bootstrap\n", + "# Install the files and packages from the metadata\n", + "/opt/aws/bin/cfn-init -v ", + " --stack ", + { + "Ref": "AWS::StackName" + }, + " --resource WebServerInstance ", + " --region ", + { + "Ref": "AWS::Region" + }, + "\n", + "# Signal the status from cfn-init\n", + "/opt/aws/bin/cfn-signal -e $? ", + " --stack ", + { + "Ref": "AWS::StackName" + }, + " --resource WebServerInstance ", + " --region ", + { + "Ref": "AWS::Region" + }, + "\n" + ] + ] + } + } + }, + "CreationPolicy": { + "ResourceSignal": { + "Timeout": "PT5M" + } + } + }, + "WebServerSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Enable HTTP access via port 80", + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": "80", + "ToPort": "80", + "CidrIp": "0.0.0.0/0" + }, + { + "IpProtocol": "tcp", + "FromPort": "22", + "ToPort": "22", + "CidrIp": { + "Ref": "SSHLocation" + } + } + ] + } + } + }, + "Outputs": { + "WebsiteURL": { + "Description": "URL for newly created LAMP stack", + "Value": { + "Fn::Join": [ + "", + [ + "http://", + { + "Fn::GetAtt": [ + "WebServerInstance", + "PublicDnsName" + ] + } + ] + ] + } + } + } +}