From 09ff84c1b11b2adcf991cabb0fe5c8c89d9077a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 23 Apr 2020 13:13:43 +0200 Subject: [PATCH 01/44] play: test machine user --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03baac51d..2b6e04bdd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,23 @@ jobs: - run: name: Run tests command: make test + - add_ssh_keys: + fingerprints: + - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" + - run: + name: TEST - Create test branch on api-data # TODO: move to the deploy phase TODO: rename + command: | + mkdir -p ./testpr + cd testpr + git clone https://github.com/PokeAPI/api-data.git api-data + cd api-data + git checkout -b testbranch + touch .gitkeeptestpr + git config --global user.name "pokeapi-machine-user" + git config --global user.email pokeapi.co@gmail.com + git add . + git commit -m "play: add test file" + git push -uf origin testbranch workflows: version: 2 From 96478a2fcd9e8c60989f6b65408f09d5d2cf68ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 23 Apr 2020 13:21:18 +0200 Subject: [PATCH 02/44] fix: add indentation --- .circleci/config.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b6e04bdd..d0540aef0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,19 +19,19 @@ jobs: fingerprints: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" - run: - name: TEST - Create test branch on api-data # TODO: move to the deploy phase TODO: rename - command: | - mkdir -p ./testpr - cd testpr - git clone https://github.com/PokeAPI/api-data.git api-data - cd api-data - git checkout -b testbranch - touch .gitkeeptestpr - git config --global user.name "pokeapi-machine-user" - git config --global user.email pokeapi.co@gmail.com - git add . - git commit -m "play: add test file" - git push -uf origin testbranch + name: TEST - Create test branch on api-data # TODO: move to the deploy phase TODO: rename + command: | + mkdir -p ./testpr + cd testpr + git clone https://github.com/PokeAPI/api-data.git api-data + cd api-data + git checkout -b testbranch + touch .gitkeeptestpr + git config --global user.name "pokeapi-machine-user" + git config --global user.email pokeapi.co@gmail.com + git add . + git commit -m "play: add test file" + git push -uf origin testbranch workflows: version: 2 From 0951d79dc77bd81d739984447e4c9d9ba5370851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 23 Apr 2020 21:48:55 +0200 Subject: [PATCH 03/44] play: open PR towards api-data --- .circleci/config.yml | 15 +------ Resources/scripts/updater.sh | 76 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 Resources/scripts/updater.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d0540aef0..85e5232ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,19 +19,8 @@ jobs: fingerprints: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" - run: - name: TEST - Create test branch on api-data # TODO: move to the deploy phase TODO: rename - command: | - mkdir -p ./testpr - cd testpr - git clone https://github.com/PokeAPI/api-data.git api-data - cd api-data - git checkout -b testbranch - touch .gitkeeptestpr - git config --global user.name "pokeapi-machine-user" - git config --global user.email pokeapi.co@gmail.com - git add . - git commit -m "play: add test file" - git push -uf origin testbranch + name: TEST - Create test branch on api-data and relative PR # TODO: move to the deploy phase TODO: rename + command: bash Resources/scripts/updater.sh workflows: version: 2 diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh new file mode 100644 index 000000000..6961cef5a --- /dev/null +++ b/Resources/scripts/updater.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +set -o pipefail + +org='PokeAPI' +data_repo='api-data' +branch_name='testbranch' + +prepare() { + mkdir -p ./testpr + cd testpr || exit +} + +clone() { + git clone "https://github.com/$org/$data_repo.git" "$data_repo" + cd "$data_repo" || exit +} + +configure_git() { + git config --global user.name "pokeapi-machine-user" + git config --global user.email pokeapi.co@gmail.com +} + +push() { + git checkout -b "$branch_name" + touch .gitkeeptestpr + git add . + git commit -m "play: add test file" + git push -uf origin "$branch_name" +} + +pr_content() { + cat < Date: Thu, 23 Apr 2020 21:51:17 +0200 Subject: [PATCH 04/44] play: better token name --- Resources/scripts/updater.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 6961cef5a..a911c09b2 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -57,13 +57,13 @@ EOF } create_pr() { - pr_number=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -X POST --data "$(pr_content)" "https://api.github.com/repos/$org/$data_repo/pulls" | jq '.number') + pr_number=$(curl -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(pr_content)" "https://api.github.com/repos/$org/$data_repo/pulls" | jq '.number') echo pr_number } assign_pr() { pr_number=$1 - curl -H "Authorization: token $GITHUB_API_TOKEN" -X POST --data "$(reviewers)" "https://api.github.com/repos/$org/$data_repo/pulls/$pr_number/requested_reviewers" + curl -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(reviewers)" "https://api.github.com/repos/$org/$data_repo/pulls/$pr_number/requested_reviewers" } prepare From cf3802cc7fca47f8a5c32ebce6b40bd1bdf23e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 23 Apr 2020 21:59:36 +0200 Subject: [PATCH 05/44] play: add -x flag --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 85e5232ee..5d0a15c50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" - run: name: TEST - Create test branch on api-data and relative PR # TODO: move to the deploy phase TODO: rename - command: bash Resources/scripts/updater.sh + command: bash -x Resources/scripts/updater.sh workflows: version: 2 From 016f4be8749703798a25d51e33fee2fdae0cc7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 23 Apr 2020 22:02:53 +0200 Subject: [PATCH 06/44] play: invoke create_pr function --- Resources/scripts/updater.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index a911c09b2..9b76be03a 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -70,7 +70,7 @@ prepare clone configure_git push -sleep 5 -pr_number=create_pr -sleep 5 +sleep 10 +pr_number=$(create_pr) +sleep 10 assign_pr "$pr_number" From 773a0429ed9160516dd72100dde0b9e7f27e2cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 23 Apr 2020 22:31:08 +0200 Subject: [PATCH 07/44] play: use " in JSON data/modify PR --- Resources/scripts/updater.sh | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 9b76be03a..14a956487 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -33,7 +33,7 @@ pr_content() { cat < Date: Fri, 24 Apr 2020 21:18:26 +0200 Subject: [PATCH 08/44] feat: add checks on curl requests --- Resources/scripts/updater.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 14a956487..55fba9c86 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -59,7 +59,7 @@ assignees_and_labels() { EOF } -reviewers() { +reviewers() { # TODO: Add core team cat < Date: Fri, 24 Apr 2020 21:27:18 +0200 Subject: [PATCH 09/44] play: split circle test phase --- .circleci/config.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d0a15c50..a95408b2e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,14 +7,20 @@ jobs: steps: - checkout - run: - name: Install deps - command: sudo make dev-install + name: Install deps + command: sudo make dev-install - run: - name: Linting - command: make format-check + name: Linting + command: make format-check - run: - name: Run tests - command: make test + name: Run tests + command: make test + + generate_data: + docker: + - image: circleci/python:3.7 + steps: + - checkout - add_ssh_keys: fingerprints: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" @@ -27,3 +33,6 @@ workflows: test: jobs: - test + - generate_data: + requires: + - test From d04da768f2d64baedf3343204f6ae28957eadafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 21:31:39 +0200 Subject: [PATCH 10/44] play: format --- .circleci/config.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a95408b2e..6c2027bec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,14 +7,14 @@ jobs: steps: - checkout - run: - name: Install deps - command: sudo make dev-install + name: Install deps + command: sudo make dev-install - run: - name: Linting - command: make format-check + name: Linting + command: make format-check - run: - name: Run tests - command: make test + name: Run tests + command: make test generate_data: docker: @@ -32,7 +32,7 @@ workflows: version: 2 test: jobs: + - test + - generate_data: + requires: - test - - generate_data: - requires: - - test From 6cebda5382aa6676ba5ad395dfe4b36ac90da97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 21:33:05 +0200 Subject: [PATCH 11/44] play: format --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c2027bec..da891b035 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,5 +34,5 @@ workflows: jobs: - test - generate_data: - requires: - - test + requires: + - test From 55ee13be54ba3b5e090b2174f140bb6864575565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 22:49:53 +0200 Subject: [PATCH 12/44] feat: run the updater on CircleCI --- .circleci/config.yml | 3 +-- Resources/scripts/updater.sh | 33 ++++++++++++++++++++------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da891b035..90ffa2854 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,8 +17,7 @@ jobs: command: make test generate_data: - docker: - - image: circleci/python:3.7 + machine: true steps: - checkout - add_ssh_keys: diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 55fba9c86..37e57cfe3 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -7,13 +7,12 @@ data_repo='api-data' branch_name='testbranch' prepare() { - mkdir -p ./testpr - cd testpr || exit + mkdir -p ./repositories + cd repositories || exit } clone() { git clone "https://github.com/$org/$data_repo.git" "$data_repo" - cd "$data_repo" || exit } configure_git() { @@ -21,14 +20,22 @@ configure_git() { git config --global user.email pokeapi.co@gmail.com } -push() { - git checkout -b "$branch_name" - touch .gitkeeptestpr - git add . - git commit -m "play: add test file" - git push -uf origin "$branch_name" +run_updater() { + sleep 10 # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content + cd "${data_repo}/updater" || exit + docker build -t pokeapi-updater . + docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" pokeapi-updater + cd .. || exit } +# push() { +# git checkout -b "$branch_name" +# touch .gitkeeptestpr +# git add . +# git commit -m "play: add test file" +# git push -uf origin "$branch_name" +# } + pr_content() { cat < Date: Fri, 24 Apr 2020 22:56:16 +0200 Subject: [PATCH 13/44] play: fix ssh bad credentials --- Resources/scripts/updater.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 37e57cfe3..f6e9b4d0a 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -18,6 +18,8 @@ clone() { configure_git() { git config --global user.name "pokeapi-machine-user" git config --global user.email pokeapi.co@gmail.com + chown "$USER" ~/.ssh/config + chmod 644 ~/.ssh/config } run_updater() { From c3b1e4fbeabe91155bac66c20ed2c426d22b26b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 23:05:17 +0200 Subject: [PATCH 14/44] play: use https for cloning --- Resources/scripts/updater.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index f6e9b4d0a..53c9c577e 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -18,15 +18,15 @@ clone() { configure_git() { git config --global user.name "pokeapi-machine-user" git config --global user.email pokeapi.co@gmail.com - chown "$USER" ~/.ssh/config - chmod 644 ~/.ssh/config + # chown "$USER" ~/.ssh/config + # chmod 644 ~/.ssh/config } run_updater() { sleep 10 # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content cd "${data_repo}/updater" || exit docker build -t pokeapi-updater . - docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" pokeapi-updater + docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI 'https://github.com/PokeAPI/pokeapi.git' -e REPO_DATA 'https://github.com/PokeAPI/api-data.git' pokeapi-updater cd .. || exit } From 5be98b385f8d69ff70dbbb856c48e4a8d57f34a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 23:10:16 +0200 Subject: [PATCH 15/44] play: assign env variables to their values --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 53c9c577e..59dc67c1f 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -26,7 +26,7 @@ run_updater() { sleep 10 # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content cd "${data_repo}/updater" || exit docker build -t pokeapi-updater . - docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI 'https://github.com/PokeAPI/pokeapi.git' -e REPO_DATA 'https://github.com/PokeAPI/api-data.git' pokeapi-updater + docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" pokeapi-updater cd .. || exit } From c1af30439d275c9f3b1c209b4b87dcb3d1844a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 23:30:30 +0200 Subject: [PATCH 16/44] play: use test branch --- Resources/scripts/updater.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 59dc67c1f..c85cef131 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -13,6 +13,7 @@ prepare() { clone() { git clone "https://github.com/$org/$data_repo.git" "$data_repo" + git checkout test } configure_git() { From 628058e37237915a4b70a3e2eb7a2e7a8cf10d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 23:40:57 +0200 Subject: [PATCH 17/44] play: fetches remotes --- Resources/scripts/updater.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index c85cef131..9a40ecd56 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -13,6 +13,7 @@ prepare() { clone() { git clone "https://github.com/$org/$data_repo.git" "$data_repo" + git fetch git checkout test } From a84a48e8f245e36fa21044fed41d77ebda8a0bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 23:45:09 +0200 Subject: [PATCH 18/44] play: move git commands --- Resources/scripts/updater.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 9a40ecd56..6061efc6c 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -13,8 +13,6 @@ prepare() { clone() { git clone "https://github.com/$org/$data_repo.git" "$data_repo" - git fetch - git checkout test } configure_git() { @@ -27,6 +25,8 @@ configure_git() { run_updater() { sleep 10 # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content cd "${data_repo}/updater" || exit + git fetch + git checkout test docker build -t pokeapi-updater . docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" pokeapi-updater cd .. || exit From 03a32c667847c610ca12268afe4fe4d93f9eac75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 24 Apr 2020 23:57:03 +0200 Subject: [PATCH 19/44] play: ask CircleCI to use newer version of Docker --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90ffa2854..10eaacee2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,8 @@ jobs: command: make test generate_data: - machine: true + machine: + image: ubuntu-1604:202004-01 # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images with Docker 19.03.8 steps: - checkout - add_ssh_keys: From e577b77df0a267ac12fb00493989d85f6000d500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 00:10:45 +0200 Subject: [PATCH 20/44] play: try to use quay image --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 6061efc6c..601de680c 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -28,7 +28,7 @@ run_updater() { git fetch git checkout test docker build -t pokeapi-updater . - docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" pokeapi-updater + docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" quay.io/pokeapi/updater cd .. || exit } From af88ee02aa5fa5d3dfb142381892e0524be82527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 00:16:35 +0200 Subject: [PATCH 21/44] play: decrease Docker version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 10eaacee2..1ded6f554 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: generate_data: machine: - image: ubuntu-1604:202004-01 # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images with Docker 19.03.8 + image: ubuntu-1604:201903-01 # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images with Docker 18.09.3 steps: - checkout - add_ssh_keys: From 11715de64b97b3882151d4f2d2a407629b8f723f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 12:33:20 +0200 Subject: [PATCH 22/44] play: use Docker 18.09.3 --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 601de680c..6061efc6c 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -28,7 +28,7 @@ run_updater() { git fetch git checkout test docker build -t pokeapi-updater . - docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" quay.io/pokeapi/updater + docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" pokeapi-updater cd .. || exit } From ab18921455b65af15fd87e4ca9562eb563784983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 15:36:24 +0200 Subject: [PATCH 23/44] play: use ssh/copy .ssh --- Resources/scripts/updater.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 6061efc6c..ffc4792ab 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -18,8 +18,6 @@ clone() { configure_git() { git config --global user.name "pokeapi-machine-user" git config --global user.email pokeapi.co@gmail.com - # chown "$USER" ~/.ssh/config - # chmod 644 ~/.ssh/config } run_updater() { @@ -28,7 +26,13 @@ run_updater() { git fetch git checkout test docker build -t pokeapi-updater . - docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://github.com/PokeAPI/api-data.git" pokeapi-updater + docker run --name pokeapi-api-data-updater --privileged -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" pokeapi-updater bash + docker exec pokeapi-api-data-updater mkdir -p /root/.ssh + docker cp ~/.ssh pokeapi-api-data-updater:/root/.ssh + docker exec pokeapi-api-data-updater chown -R root /root/.ssh + docker exec pokeapi-api-data-updater chmod -R 600 /root/.ssh + docker exec pokeapi-api-data-updater bash cmd.bash + docker stop pokeapi-api-data-updater cd .. || exit } From 9dcabc048d6d375f2741ca59d407deb0b1eab870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 15:40:08 +0200 Subject: [PATCH 24/44] play: run container in background --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index ffc4792ab..e452bca86 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -26,7 +26,7 @@ run_updater() { git fetch git checkout test docker build -t pokeapi-updater . - docker run --name pokeapi-api-data-updater --privileged -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" pokeapi-updater bash + docker run -itd --name pokeapi-api-data-updater --privileged -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" pokeapi-updater bash docker exec pokeapi-api-data-updater mkdir -p /root/.ssh docker cp ~/.ssh pokeapi-api-data-updater:/root/.ssh docker exec pokeapi-api-data-updater chown -R root /root/.ssh From 7541adb44b0e70e5f56e3e3c61547bb30aa5ecc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 16:12:00 +0200 Subject: [PATCH 25/44] refactor: remove ssh keys/use https and API token --- Resources/scripts/updater.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index e452bca86..8116539b1 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -26,13 +26,7 @@ run_updater() { git fetch git checkout test docker build -t pokeapi-updater . - docker run -itd --name pokeapi-api-data-updater --privileged -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" pokeapi-updater bash - docker exec pokeapi-api-data-updater mkdir -p /root/.ssh - docker cp ~/.ssh pokeapi-api-data-updater:/root/.ssh - docker exec pokeapi-api-data-updater chown -R root /root/.ssh - docker exec pokeapi-api-data-updater chmod -R 600 /root/.ssh - docker exec pokeapi-api-data-updater bash cmd.bash - docker stop pokeapi-api-data-updater + docker run --privileged -e COMMIT_EMAIL=pokeapi.co@gmail.com -e COMMIT_NAME="pokeapi-machine-user" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/PokeAPI/pokeapi.git" -e REPO_DATA="https://${MACHINE_USER_GITHUB_API_TOKEN}@github.com/PokeAPI/api-data.git" pokeapi-updater cd .. || exit } From 41cf8109e39951374c5a9ff53481c946dd2ef8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 21:15:41 +0200 Subject: [PATCH 26/44] multi: refactor/notify updater start --- Resources/scripts/updater.sh | 86 +++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 8116539b1..76d009fce 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -4,45 +4,90 @@ set -o pipefail org='PokeAPI' data_repo='api-data' +engine_repo='pokeapi' branch_name='testbranch' +username='pokeapi-machine-user' +email='pokeapi.co@gmail.com' + +function cleanexit { + echo "Exiting" + echo "$2" + exit "$1" +} prepare() { mkdir -p ./repositories - cd repositories || exit + cd repositories || cleanexit 1 "Failed to cd" +} + +get_invokator_pr_number() { + if [ -z "$CIRCLE_PULL_REQUEST" ]; then + echo "${CIRCLE_PULL_REQUEST##*/}" + fi } clone() { - git clone "https://github.com/$org/$data_repo.git" "$data_repo" + git clone "https://github.com/${org}/${data_repo}.git" "$data_repo" } configure_git() { - git config --global user.name "pokeapi-machine-user" - git config --global user.email pokeapi.co@gmail.com + git config --global user.name "$username" + git config --global user.email "$email" +} + +notification_comment_content() { + cat < Date: Sat, 25 Apr 2020 21:23:58 +0200 Subject: [PATCH 27/44] fix: add not operator --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 76d009fce..6566c1fcf 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -44,7 +44,7 @@ EOF } notify_updater_start() { - if [ -z "$CIRCLE_PULL_REQUEST" ]; then + if ! [ -z "$CIRCLE_PULL_REQUEST" ]; then pr_number="${CIRCLE_PULL_REQUEST##*/}" curl -f -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(notification_comment_content)" "https://api.github.com/repos/$org/$data_repo/issues/${pr_number}/comments" fi From 5e06cb22aa3aa7efe7e871452aaab9da2d3b3abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 21:28:28 +0200 Subject: [PATCH 28/44] fix: send notification to pokeapi --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 6566c1fcf..bb11a3c64 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -46,7 +46,7 @@ EOF notify_updater_start() { if ! [ -z "$CIRCLE_PULL_REQUEST" ]; then pr_number="${CIRCLE_PULL_REQUEST##*/}" - curl -f -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(notification_comment_content)" "https://api.github.com/repos/$org/$data_repo/issues/${pr_number}/comments" + curl -f -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(notification_comment_content)" "https://api.github.com/repos/$org/$engine_repo/issues/${pr_number}/comments" fi } From 56a01dddbf456181c2cb7a6fe8a5d5eda07e2ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 25 Apr 2020 22:40:21 +0200 Subject: [PATCH 29/44] refactor: comments/cleanup/better branch name --- Resources/scripts/updater.sh | 71 ++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index bb11a3c64..8d1b80d0e 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -5,7 +5,7 @@ set -o pipefail org='PokeAPI' data_repo='api-data' engine_repo='pokeapi' -branch_name='testbranch' +branch_name='updated-data' username='pokeapi-machine-user' email='pokeapi.co@gmail.com' @@ -15,27 +15,32 @@ function cleanexit { exit "$1" } +# Create and use a personal folder prepare() { mkdir -p ./repositories cd repositories || cleanexit 1 "Failed to cd" } +# Not used +# Check and return the number of the Pull Request that started this job get_invokator_pr_number() { if [ -z "$CIRCLE_PULL_REQUEST" ]; then echo "${CIRCLE_PULL_REQUEST##*/}" fi } +# Clone the repository containing the static JSON files clone() { git clone "https://github.com/${org}/${data_repo}.git" "$data_repo" } +# Configure git to use the supplied user when committing configure_git() { git config --global user.name "$username" git config --global user.email "$email" } -notification_comment_content() { +pr_input_notification_comment() { cat < Date: Sat, 25 Apr 2020 22:50:08 +0200 Subject: [PATCH 30/44] refactor: add core team/comments --- Resources/scripts/updater.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 8d1b80d0e..d543169bc 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -1,4 +1,7 @@ #!/bin/bash +# Executed when the master branch of PokeAPI/pokeapi gets updated +# Runs in CircleCI +# Generates new data using the latest changes of PokeAPI/pokeapi in order to open a Pull Request towards PokeAPI/api-data set -o pipefail @@ -12,7 +15,8 @@ email='pokeapi.co@gmail.com' function cleanexit { echo "Exiting" echo "$2" - exit "$1" + # TODO: Notify users on Github + exit $1 } # Create and use a personal folder @@ -76,7 +80,7 @@ run_updater() { # Run the updater docker run --privileged -e COMMIT_EMAIL="$email" -e COMMIT_NAME="$username" -e BRANCH_NAME="$branch_name" -e REPO_POKEAPI="https://github.com/${org}/${engine_repo}.git" -e REPO_DATA="https://${MACHINE_USER_GITHUB_API_TOKEN}@github.com/${org}/${data_repo}.git" pokeapi-updater if [ $? -ne 0 ]; then - cleanexit 2 "Failed to run the pokeapi-updater container" + cleanexit 1 "Failed to run the pokeapi-updater container" fi cd .. || cleanexit 1 "Failed to cd" @@ -144,11 +148,14 @@ customize_pr() { fi } -pr_input_reviewers() { # TODO: Add core team +pr_input_reviewers() { cat < Date: Wed, 29 Apr 2020 17:14:37 +0200 Subject: [PATCH 31/44] refactor: don't use api-data's test branch --- Resources/scripts/updater.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index d543169bc..20a51864e 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -66,10 +66,6 @@ run_updater() { # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content sleep 10 - - # Switch to test branch # TODO: don't checkout - git fetch - git checkout test # Build the updater image docker build -t pokeapi-updater . From 7d8ea5c082ebbdec13badbdc9452853cd05ddb4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 30 Apr 2020 14:51:52 +0200 Subject: [PATCH 32/44] feat: end notification/refactor --- Resources/scripts/updater.sh | 58 +++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 20a51864e..e1295d679 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -15,7 +15,11 @@ email='pokeapi.co@gmail.com' function cleanexit { echo "Exiting" echo "$2" - # TODO: Notify users on Github + if [ "$1" -gt "0" ]; then + notify_engine_pr "end_failed" + else + notify_engine_pr "end_success" + fi exit $1 } @@ -44,7 +48,7 @@ configure_git() { git config --global user.email "$email" } -pr_input_notification_comment() { +pr_input_updater_start() { cat < Date: Thu, 30 Apr 2020 19:42:56 +0200 Subject: [PATCH 33/44] fix: allow fail and success messages --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index e1295d679..74b73ff90 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -74,7 +74,7 @@ EOF # If the job was started by a Pull Request add a comment to notify the users notify_engine_pr() { - if [[ $1 == "start" || $1 == "end" ]]; then + if [[ $1 == "start" || $1 == "end_failed" || $1 == "end_success" ]]; then if ! [ -z "$CIRCLE_PULL_REQUEST" ]; then engine_repo_pr_number="${CIRCLE_PULL_REQUEST##*/}" curl -f -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(pr_input_updater_$1)" "https://api.github.com/repos/$org/$engine_repo/issues/${engine_repo_pr_number}/comments" From 6842c66cdfb8b74a317a2f4921bd1478733323d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Thu, 30 Apr 2020 23:53:46 +0200 Subject: [PATCH 34/44] refactor: clean/better messages --- Resources/scripts/updater.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 74b73ff90..ce13f1506 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -12,7 +12,8 @@ branch_name='updated-data' username='pokeapi-machine-user' email='pokeapi.co@gmail.com' -function cleanexit { +# Exit the script notifying the user about its success +cleanexit() { echo "Exiting" echo "$2" if [ "$1" -gt "0" ]; then @@ -51,7 +52,7 @@ configure_git() { pr_input_updater_start() { cat < Date: Thu, 30 Apr 2020 23:57:28 +0200 Subject: [PATCH 35/44] feat: run the updater only on master --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ded6f554..126fd7003 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: fingerprints: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" - run: - name: TEST - Create test branch on api-data and relative PR # TODO: move to the deploy phase TODO: rename + name: Generate new data and open a Pull Request towards PokeAPI/api-data command: bash -x Resources/scripts/updater.sh workflows: @@ -36,3 +36,6 @@ workflows: - generate_data: requires: - test + filters: + branches: + only: master From 283959504484f19fd5322f6f8bda5e7a2975301b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 1 May 2020 13:30:00 +0200 Subject: [PATCH 36/44] refactor: differentiate workflow from job name --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 126fd7003..25a3f8676 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: - test: + test-and-generate_data: docker: - image: circleci/python:3.7 steps: @@ -30,7 +30,7 @@ jobs: workflows: version: 2 - test: + test-and-generate_data: jobs: - test - generate_data: From 686ed9ce34024ec8d5951223c9bffac73f56b50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Fri, 1 May 2020 13:30:53 +0200 Subject: [PATCH 37/44] fix: change name only for the workflow --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 25a3f8676..32a7d3c2b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: - test-and-generate_data: + test: docker: - image: circleci/python:3.7 steps: From cad6c2e7e3ce7437939ecfc206709b125d93ba81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 2 May 2020 19:59:09 +0200 Subject: [PATCH 38/44] feat: push to staging/better messages/wait deploy --- Resources/scripts/updater.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index ce13f1506..091645295 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -8,7 +8,7 @@ set -o pipefail org='PokeAPI' data_repo='api-data' engine_repo='pokeapi' -branch_name='updated-data' +branch_name='staging' username='pokeapi-machine-user' email='pokeapi.co@gmail.com' @@ -30,11 +30,12 @@ prepare() { cd repositories || cleanexit 1 "Failed to cd" } -# Not used # Check and return the number of the Pull Request that started this job get_invokator_pr_number() { - if [ -z "$CIRCLE_PULL_REQUEST" ]; then + if ! [ -z "$CIRCLE_PULL_REQUEST" ]; then echo "${CIRCLE_PULL_REQUEST##*/}" + else + echo 'null' fi } @@ -52,7 +53,7 @@ configure_git() { pr_input_updater_start() { cat <
The staging branch will be deployed in our [staging environment](https://pokeapi-test-b6137.firebaseapp.com/api/v2/) and you will be able to review the entire API.

A Pull Request ([master](https://github.com/PokeAPI/api-data/tree/master)<-[staging](https://github.com/PokeAPI/api-data/tree/staging)) will be also created at [PokeAPI/api-data](https://github.com/PokeAPI/api-data/pulls) and assigned to the PokeAPI Core team to be reviewed. If approved and merged new data will soon be available worldwide at [pokeapi.co](https:/pokeapi.co)" } EOF } @@ -60,7 +61,7 @@ EOF pr_input_updater_end_success() { cat <
You can see the Pull Request deployed at our [staging environment](https://pokeapi-test-b6137.firebaseapp.com/api/v2/) when [CircleCI deploy](https://app.circleci.com/pipelines/github/PokeAPI/deploy) will be finished (_check the started time of the last build_)" } EOF } @@ -68,7 +69,7 @@ EOF pr_input_updater_end_failed() { cat <
The new data was generated using a copy of [${org}/${engine_repo}](https://github.com/${org}/${engine_repo}/commits/master) repository when the \`HEAD\` was pointing to \`${CIRCLE_SHA1}\`.

This Pull Request should have been deployed in our [staging environment](https://pokeapi-test-b6137.firebaseapp.com/api/v2/), check [here](https://app.circleci.com/pipelines/github/PokeAPI/deploy) for the lastest build timestamp and status code.", "head": "$branch_name", "base": "master", "assignees": [ @@ -135,7 +136,8 @@ EOF # Create a Pull Request to merge the branch recently pushed by the updater with the master branch create_pr() { - data_repo_pr_number=$(curl -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(pr_input_content)" "https://api.github.com/repos/$org/$data_repo/pulls" | jq '.number') + engine_repo_pr_number=$(get_invokator_pr_number) + data_repo_pr_number=$(curl -H "Authorization: token $MACHINE_USER_GITHUB_API_TOKEN" -X POST --data "$(pr_input_content "$engine_repo_pr_number")" "https://api.github.com/repos/$org/$data_repo/pulls" | jq '.number') if [[ "$data_repo_pr_number" = "null" ]]; then cleanexit 1 "Couldn't create the Pull Request" fi @@ -195,6 +197,8 @@ clone notify_engine_pr "start" run_updater check_remote_branch "$branch_name" +sleep 300 # 5 minutes, the time it takes for CircleCI's api-data script to generate the data and for CircleCI's deploy script to deploy it to the staging environment +check_remote_branch "$branch_name" data_repo_pr_number=$(create_pr) customize_pr "$data_repo_pr_number" add_reviewers_to_pr "$data_repo_pr_number" From 60813dbd5b6c42ad3bd9998f3b0e1b9b3efb7488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 2 May 2020 20:03:25 +0200 Subject: [PATCH 39/44] refactor: update config to 2.1/test run --- .circleci/config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32a7d3c2b..1dd38833a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 jobs: test: @@ -16,7 +16,7 @@ jobs: name: Run tests command: make test - generate_data: + generate_data_and_trigger_pullrequest_deploy: machine: image: ubuntu-1604:201903-01 # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images with Docker 18.09.3 steps: @@ -25,17 +25,17 @@ jobs: fingerprints: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" - run: - name: Generate new data and open a Pull Request towards PokeAPI/api-data + name: Generate new data, push to PokeAPI/api-data's staging branch and open a Pull Request towards PokeAPI/api-data's master branch command: bash -x Resources/scripts/updater.sh workflows: version: 2 - test-and-generate_data: + test-and-generate_data_and_trigger_pullrequest_deploy: jobs: - - test - - generate_data: - requires: - - test - filters: - branches: - only: master + - test + - generate_data_and_trigger_pullrequest_deploy: + requires: + - test + # filters: + # branches: + # only: master From b2ccfb86c798bf39005254dfc93008eaf1e07eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sat, 2 May 2020 21:09:35 +0200 Subject: [PATCH 40/44] play: checkout staging --- Resources/scripts/updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 091645295..05a8df681 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -87,7 +87,7 @@ notify_engine_pr() { # Run the updater script (https://github.com/PokeAPI/api-data/blob/master/updater/cmd.bash) which will generate the new pokeapi data and push it to the api-data repository under a new branch run_updater() { cd "${data_repo}/updater" || cleanexit 1 "Failed to cd" - + git checkout staging # TODO: remove # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content sleep 10 From b7902ee3d090e437de2e406f1f6bbcc62d5c5ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sun, 3 May 2020 00:16:51 +0200 Subject: [PATCH 41/44] refactor: run staging deploy only on master --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1dd38833a..76f7d8deb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,6 +36,6 @@ workflows: - generate_data_and_trigger_pullrequest_deploy: requires: - test - # filters: - # branches: - # only: master + filters: + branches: + only: master From 44acf889e674e672e347ac092b06b49ef843ee38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Sun, 3 May 2020 17:16:02 +0200 Subject: [PATCH 42/44] feat: deploy staging branch to staging env --- .circleci/config.yml | 12 +++++++----- Resources/scripts/updater.sh | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 76f7d8deb..955007a11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ jobs: name: Run tests command: make test - generate_data_and_trigger_pullrequest_deploy: + deploy: machine: image: ubuntu-1604:201903-01 # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images with Docker 18.09.3 steps: @@ -25,17 +25,19 @@ jobs: fingerprints: - "d3:5f:9d:68:ea:12:9a:42:d3:d3:e1:db:b0:6a:fc:d2" - run: - name: Generate new data, push to PokeAPI/api-data's staging branch and open a Pull Request towards PokeAPI/api-data's master branch + name: Generate new data, push to PokeAPI/api-data's staging branch, if master open a Pull Request towards PokeAPI/api-data's master branch command: bash -x Resources/scripts/updater.sh workflows: version: 2 - test-and-generate_data_and_trigger_pullrequest_deploy: + test-and-deploy: jobs: - test - - generate_data_and_trigger_pullrequest_deploy: + - deploy: requires: - test filters: branches: - only: master + only: + - master + - circleci-hook # TODO: change to staging diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 05a8df681..0824a4890 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -197,9 +197,11 @@ clone notify_engine_pr "start" run_updater check_remote_branch "$branch_name" -sleep 300 # 5 minutes, the time it takes for CircleCI's api-data script to generate the data and for CircleCI's deploy script to deploy it to the staging environment -check_remote_branch "$branch_name" -data_repo_pr_number=$(create_pr) -customize_pr "$data_repo_pr_number" -add_reviewers_to_pr "$data_repo_pr_number" +if [ "$CIRCLE_BRANCH" = 'master' ]; then + sleep 300 # 5 minutes, the time it takes for CircleCI's api-data script to generate the data and for CircleCI's deploy script to deploy it to the staging environment + check_remote_branch "$branch_name" + data_repo_pr_number=$(create_pr) + customize_pr "$data_repo_pr_number" + add_reviewers_to_pr "$data_repo_pr_number" +fi cleanexit 0 'Done' From bada6cd076a1ac6b589a0892f8d1ef520be7e31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Wed, 13 May 2020 16:01:54 +0200 Subject: [PATCH 43/44] refactor: address todos --- .circleci/config.yml | 2 +- Resources/scripts/updater.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 955007a11..32e43f043 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,4 +40,4 @@ workflows: branches: only: - master - - circleci-hook # TODO: change to staging + - staging diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index 0824a4890..ff09d2ffd 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -87,8 +87,7 @@ notify_engine_pr() { # Run the updater script (https://github.com/PokeAPI/api-data/blob/master/updater/cmd.bash) which will generate the new pokeapi data and push it to the api-data repository under a new branch run_updater() { cd "${data_repo}/updater" || cleanexit 1 "Failed to cd" - git checkout staging # TODO: remove - # Wait to be sure PokeAPI/pokeapi:origin/master has been updated on Github with the lastest merged PR content + # Wait to be sure PokeAPI/pokeapi's master branch has been updated on Github with the lastest merged PR content sleep 10 # Build the updater image From 79ebc3e04a6be35d1ef2fdc62d8a5f2a6dfdfc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alessandro=20Pezz=C3=A8?= Date: Wed, 13 May 2020 16:14:42 +0200 Subject: [PATCH 44/44] refactor: use some variables in PR comments --- Resources/scripts/updater.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Resources/scripts/updater.sh b/Resources/scripts/updater.sh index ff09d2ffd..e2d077f62 100644 --- a/Resources/scripts/updater.sh +++ b/Resources/scripts/updater.sh @@ -11,6 +11,11 @@ engine_repo='pokeapi' branch_name='staging' username='pokeapi-machine-user' email='pokeapi.co@gmail.com' +staging_environment_url='https://pokeapi-test-b6137.firebaseapp.com/api/v2/' +production_environment_url='https://pokeapi.co/api/v2/' +data_repo_url='https://github.com/PokeAPI/api-data' +engine_circleci_status_url='https://app.circleci.com/pipelines/github/PokeAPI/pokeapi' +deploy_circleci_status_url='https://app.circleci.com/pipelines/github/PokeAPI/deploy' # Exit the script notifying the user about its success cleanexit() { @@ -53,7 +58,7 @@ configure_git() { pr_input_updater_start() { cat <
The staging branch will be deployed in our [staging environment](https://pokeapi-test-b6137.firebaseapp.com/api/v2/) and you will be able to review the entire API.

A Pull Request ([master](https://github.com/PokeAPI/api-data/tree/master)<-[staging](https://github.com/PokeAPI/api-data/tree/staging)) will be also created at [PokeAPI/api-data](https://github.com/PokeAPI/api-data/pulls) and assigned to the PokeAPI Core team to be reviewed. If approved and merged new data will soon be available worldwide at [pokeapi.co](https:/pokeapi.co)" + "body": "A [PokeAPI/api-data](${data_repo_url}) refresh has started. In 45 minutes the staging branch of [PokeAPI/api-data](${data_repo_url}/tree/staging) will be pushed with the new generated data.

The staging branch will be deployed in our [staging environment]($staging_environment_url) and you will be able to review the entire API.

A Pull Request ([master](${data_repo_url}/tree/master)<-[staging](${data_repo_url}/tree/staging)) will be also created at [PokeAPI/api-data](${data_repo_url}/pulls) and assigned to the PokeAPI Core team to be reviewed. If approved and merged new data will soon be available worldwide at [pokeapi.co]($production_environment_url)" } EOF } @@ -61,7 +66,7 @@ EOF pr_input_updater_end_success() { cat <
You can see the Pull Request deployed at our [staging environment](https://pokeapi-test-b6137.firebaseapp.com/api/v2/) when [CircleCI deploy](https://app.circleci.com/pipelines/github/PokeAPI/deploy) will be finished (_check the started time of the last build_)" + "body": "The updater script has finished its job and has now opened a Pull Request towards [PokeAPI/api-data](${data_repo_url}/pulls) with the updated data.

You can see the Pull Request deployed at our [staging environment]($staging_environment_url) when [CircleCI deploy]($deploy_circleci_status_url) will be finished (_check the started time of the last build_)" } EOF } @@ -69,7 +74,7 @@ EOF pr_input_updater_end_failed() { cat <
The new data was generated using a copy of [${org}/${engine_repo}](https://github.com/${org}/${engine_repo}/commits/master) repository when the \`HEAD\` was pointing to \`${CIRCLE_SHA1}\`.

This Pull Request should have been deployed in our [staging environment](https://pokeapi-test-b6137.firebaseapp.com/api/v2/), check [here](https://app.circleci.com/pipelines/github/PokeAPI/deploy) for the lastest build timestamp and status code.", + "body": "Incoming data generated by [${org}/${engine_repo}](https://github.com/${org}/${engine_repo})'s CircleCI worker since Pull Request [#${1}](https://github.com/${org}/${engine_repo}/pull/${1}) was merged into the \`master\` branch.

The new data was generated using a copy of [${org}/${engine_repo}](https://github.com/${org}/${engine_repo}/commits/master) repository when the \`HEAD\` was pointing to \`${CIRCLE_SHA1}\`.

This Pull Request should have been deployed in our [staging environment]($staging_environment_url), check [here]($deploy_circleci_status_url) for the lastest build timestamp and status code.", "head": "$branch_name", "base": "master", "assignees": [