From f83b72a9b072519731a03443860a61f662b23410 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 13:23:54 -0600 Subject: [PATCH 01/58] initial attempt at integration testing --- .../actions/to-integration-tests/Dockerfile | 25 +++ .../actions/to-integration-tests/README.md | 48 ++++++ .../actions/to-integration-tests/action.yml | 22 +++ .../to-integration-tests/entrypoint.sh | 145 ++++++++++++++++++ .github/workflows/traffic ops.yml | 66 ++++++++ 5 files changed, 306 insertions(+) create mode 100644 .github/actions/to-integration-tests/Dockerfile create mode 100644 .github/actions/to-integration-tests/README.md create mode 100644 .github/actions/to-integration-tests/action.yml create mode 100644 .github/actions/to-integration-tests/entrypoint.sh create mode 100644 .github/workflows/traffic ops.yml diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile new file mode 100644 index 0000000000..17f4800193 --- /dev/null +++ b/.github/actions/to-integration-tests/Dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM golang:1.14.2 + +RUN apk add --no-cache openssl + +COPY entrypoint.sh /entrypoint.sh +RUN chmod a+x /entrypoint.sh + +ENTRYPOINT /entrypoint.sh diff --git a/.github/actions/to-integration-tests/README.md b/.github/actions/to-integration-tests/README.md new file mode 100644 index 0000000000..b468bdc9e4 --- /dev/null +++ b/.github/actions/to-integration-tests/README.md @@ -0,0 +1,48 @@ + + +# to-integration-tests Docker action +This action runs the Traffic Ops Go client integration tests with the Traffic +Ops API. + +### `exit-code` +1 if the Go program(s) could be built successfully. + +## Example usage +```yaml +jobs: + tests: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:11.9 + env: + POSTGRES_USER: traffic_ops + POSTGRES_PASSWORD: twelve + POSTGRES_DB: traffic_ops + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - name: Checkout + uses: actions/checkout@master + - name: ./.github/actions/to-integration-tests +``` diff --git a/.github/actions/to-integration-tests/action.yml b/.github/actions/to-integration-tests/action.yml new file mode 100644 index 0000000000..4afee44811 --- /dev/null +++ b/.github/actions/to-integration-tests/action.yml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'to-integration-tests' +description: 'Runs Traffic Ops Go client/API integration tests' +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh new file mode 100644 index 0000000000..28c48bd482 --- /dev/null +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -0,0 +1,145 @@ +#!/bin/sh -l +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +GOPATH="$(mktemp -d)" +SRCDIR="$GOPATH/src/github.com/apache" +mkdir -p "$SRCDIR" +ln -s "$PWD" "$SRCDIR/trafficcontrol" +cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" + +/usr/local/go/bin/go get ./... +/usr/local/go/bin/go build . + +openssl req -x509 -out localhost.crt -keyout localhost.key \ + -newkey rsa:2048 -nodes -sha256 \ + -subj '/CN=localhost' -extensions EXT -config <( \ + printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") + +cat <<-EOF >cdn.conf +{ + "hypnotoad" : { + "listen" : [ + "$TO_PERL_SCHEME://$TO_PERL_FQDN:$TO_PERL_PORT?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" + ], + "user" : "trafops", + "group" : "trafops", + "heartbeat_timeout" : 20, + "pid_file" : "/var/run/traffic_ops.pid", + "workers" : 12 + }, + "use_ims": true, + "traffic_ops_golang" : { + "insecure": true, + "port" : "6443", + "log_location_error": "stderr", + "log_location_warning": "stderr", + "log_location_info": "stderr", + "log_location_debug": "stderr", + "log_location_event": "stderr", + "max_db_connections": 20, + "db_conn_max_lifetime_seconds": 60, + "db_query_timeout_seconds": 20, + "supported_ds_metrics": [ "kbps", "tps_total", "tps_2xx", "tps_3xx", "tps_4xx", "tps_5xx" ] + }, + "cors" : { + "access_control_allow_origin" : "*" + }, + "to" : { + "base_url" : "https://localhost", + "email_from" : "no-reply@traffic-ops.test", + "no_account_found_msg" : "A Traffic Ops user account is required for access. Please contact your Traffic Ops user administrator." + }, + "portal" : { + "base_url" : "https://not-a-real-host.test/#!/", + "email_from" : "no-reply@traffic-portal.test", + "pass_reset_path" : "user", + "user_register_path" : "user" + }, + "secrets" : [ + "blahblah" + ], + "geniso" : { + "iso_root_path" : "/opt/traffic_ops/app/public" + }, + "inactivity_timeout" : 60, + "smtp" : { + "enabled" : false + }, + "InfluxEnabled": false +} +EOF + +cat <<-EOF >database.conf +{ + "description": "Local PostgreSQL database on port 5432", + "dbname": "traffic_ops", + "hostname": "localhost", + "user": "traffic_ops", + "password": "twelve", + "port": "5432", + "ssl": false, + "type": "Pg" +} +EOF + +./traffic_ops_golang --cfg ./cdn.conf --dbcfg ./database.conf >out.log 2>err.log & + +cd ../testing/api/v1 + +cat <<-EOF >traffic-ops-test.conf +{ + "default": { + "logLocations": { + "debug": "stdout", + "error": "stdout", + "event": "stdout", + "info": "stdout", + "warning": "stdout" + }, + "session": { + "timeoutInSecs": 60 + } + }, + "trafficOps": { + "URL": "https://localhost:6443", + "password": "twelve", + "users": { + "disallowed": "disallowed", + "operations": "operations", + "admin": "admin", + "federation": "federation", + "portal": "portal", + "readOnly": "readOnly", + "extension": "extension" + } + }, + "trafficOpsDB": { + "dbname": "traffic_ops", + "description": "Test database to_test", + "hostname": "localhost", + "password": "twelve", + "port": "5432", + "type": "Pg", + "user": "traffic_ops" + } +} +EOF + +exit $? diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml new file mode 100644 index 0000000000..00d7bf9755 --- /dev/null +++ b/.github/workflows/traffic ops.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Traffic Ops Push CI + +on: + push: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + unit tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@master + - name: Run Go unit tests + uses: ./.github/actions/go-test + with: + dir: ./traffic_ops/traffic_ops_golang/... + + go fmt: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@master + - name: Go fmt Traffic Ops + uses: ./.github/actions/go-fmt/ + with: + dir: ./traffic_ops/traffic_ops_golang/... + + API tests: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:11.9 + env: + POSTGRES_USER: traffic_ops + POSTGRES_PASSWORD: twelve + POSTGRES_DB: traffic_ops + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - name: Checkout + uses: actions/checkout@master + - name: Run API tests + uses: ./.github/actions/to-integration-tests From cbac849f418035e66cf801743b108a6be4674c05 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 13:56:41 -0600 Subject: [PATCH 02/58] Added db upgrades and actual test running --- .github/actions/to-integration-tests/Dockerfile | 1 + .github/actions/to-integration-tests/entrypoint.sh | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 17f4800193..0761e2afda 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -18,6 +18,7 @@ FROM golang:1.14.2 RUN apk add --no-cache openssl +RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh RUN chmod a+x /entrypoint.sh diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 28c48bd482..8fff4fcd70 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -22,6 +22,14 @@ GOPATH="$(mktemp -d)" SRCDIR="$GOPATH/src/github.com/apache" mkdir -p "$SRCDIR" ln -s "$PWD" "$SRCDIR/trafficcontrol" + +cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" + +/usr/local/go/bin/go get ./... +/usr/local/go/bin/go build . + +./admin --env="test" upgrade + cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" /usr/local/go/bin/go get ./... @@ -36,7 +44,7 @@ cat <<-EOF >cdn.conf { "hypnotoad" : { "listen" : [ - "$TO_PERL_SCHEME://$TO_PERL_FQDN:$TO_PERL_PORT?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" + "https://not-a-real-host.test:1?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" ], "user" : "trafops", "group" : "trafops", @@ -99,6 +107,8 @@ cat <<-EOF >database.conf } EOF + + ./traffic_ops_golang --cfg ./cdn.conf --dbcfg ./database.conf >out.log 2>err.log & cd ../testing/api/v1 @@ -142,4 +152,6 @@ cat <<-EOF >traffic-ops-test.conf } EOF +go test -v --cfg ./traffic-ops-test + exit $? From ec929c6185d8fe0e07deb8b5cb4d7a4d34f05c06 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 13:58:14 -0600 Subject: [PATCH 03/58] Fix bad job names --- .github/workflows/traffic ops.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 00d7bf9755..123df94daf 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -23,7 +23,7 @@ on: types: [opened, reopened, edited, synchronize] jobs: - unit tests: + unit_tests: runs-on: ubuntu-latest steps: @@ -34,7 +34,7 @@ jobs: with: dir: ./traffic_ops/traffic_ops_golang/... - go fmt: + go_fmt: runs-on: ubuntu-latest steps: @@ -45,7 +45,7 @@ jobs: with: dir: ./traffic_ops/traffic_ops_golang/... - API tests: + API_tests: runs-on: ubuntu-latest services: From 2cff83e621402f6554b4337f9d6eef2031ef1ea0 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:00:24 -0600 Subject: [PATCH 04/58] Fix postgres service version --- .github/workflows/traffic ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 123df94daf..0e041987c1 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -50,7 +50,7 @@ jobs: services: postgres: - image: postgres:11.9 + image: postgres:11 env: POSTGRES_USER: traffic_ops POSTGRES_PASSWORD: twelve From d64a78334de27a4d7d42b7aaa90e8bd78cda4d63 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:05:10 -0600 Subject: [PATCH 05/58] Fix install command --- .github/actions/to-integration-tests/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 0761e2afda..48da5b01fd 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2 +FROM golang:1.14.2-alpine -RUN apk add --no-cache openssl +RUN apt install openssl RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From 4247cdba89ce15c0cafad7d77e52240cb2c19d98 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:06:44 -0600 Subject: [PATCH 06/58] Fix install command --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 48da5b01fd..b58e524b13 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2-alpine -RUN apt install openssl +RUN apt-get install openssl RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From 2dbc48a9a9e9b01dd0bbc38b353ff248f2606168 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:08:03 -0600 Subject: [PATCH 07/58] Fix install command --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index b58e524b13..41c1bbf033 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2-alpine -RUN apt-get install openssl +RUN apk add --no-cache openssl RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From cce4a3e8009d78dbb1f83e1b472bb2ff24925861 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:09:39 -0600 Subject: [PATCH 08/58] Add 'git' --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 41c1bbf033..77174f1aea 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2-alpine -RUN apk add --no-cache openssl +RUN apk add --no-cache openssl git RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From 5d04957a92ad7592feda28cb33940d7ab205d6d3 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:11:38 -0600 Subject: [PATCH 09/58] Add 'gcc' --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 77174f1aea..efe7c4d220 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2-alpine -RUN apk add --no-cache openssl git +RUN apk add --no-cache openssl git gcc RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From cdf613b25e4978d26bf0243b10c27966839d937b Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:14:29 -0600 Subject: [PATCH 10/58] Switch to ubuntu --- .github/actions/to-integration-tests/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index efe7c4d220..401e9ec4f9 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-alpine +FROM golang:1.14.2 -RUN apk add --no-cache openssl git gcc +RUN apt-get openssl git gcc RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From 1c7b48f5126186a95edd5d5c4272524723fb72bc Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:16:02 -0600 Subject: [PATCH 11/58] Forgot 'install' --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 401e9ec4f9..91a7a0119c 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2 -RUN apt-get openssl git gcc +RUN apt-get install openssl git gcc RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq COPY entrypoint.sh /entrypoint.sh From 78c90bfd89928ac71b2cd3cfd57a8f4464035280 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:18:45 -0600 Subject: [PATCH 12/58] Add echoing steps to entrypoint --- .github/actions/to-integration-tests/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 8fff4fcd70..87dd55460a 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex GOPATH="$(mktemp -d)" SRCDIR="$GOPATH/src/github.com/apache" @@ -26,7 +26,7 @@ ln -s "$PWD" "$SRCDIR/trafficcontrol" cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" /usr/local/go/bin/go get ./... -/usr/local/go/bin/go build . +/usr/local/go/bin/go build ./admin.go ./admin --env="test" upgrade From e08b8c2f64940593fec13f0a1e985aeaca7c214f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:22:30 -0600 Subject: [PATCH 13/58] Fix db execution path --- .github/actions/to-integration-tests/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 87dd55460a..e96f5ea6a2 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -28,7 +28,9 @@ cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" /usr/local/go/bin/go get ./... /usr/local/go/bin/go build ./admin.go -./admin --env="test" upgrade +cd .. + +./db/admin --env="test" upgrade cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" From 8fc90adeb4cdcd202d29a9d417c4a41471712556 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:25:42 -0600 Subject: [PATCH 14/58] Move goose into PATH --- .github/actions/to-integration-tests/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 91a7a0119c..308d9ce946 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -19,6 +19,7 @@ FROM golang:1.14.2 RUN apt-get install openssl git gcc RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq +RUN mv $GOPATH/bin/goose /bin/ COPY entrypoint.sh /entrypoint.sh RUN chmod a+x /entrypoint.sh From 7a98504a8797e74b05f7e3c092b240fd7ddd7480 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:30:44 -0600 Subject: [PATCH 15/58] use localhost instead of IP --- .github/actions/to-integration-tests/entrypoint.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index e96f5ea6a2..148242ae69 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -28,6 +28,15 @@ cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" /usr/local/go/bin/go get ./... /usr/local/go/bin/go build ./admin.go +echo 'version: "1.0" +name: dbconf.yml + +test: + driver: postgres + open: host=localhost port=5432 user=traffic_ops password=twelve dbname=to_test sslmode=disable + +' > dbconf.yml + cd .. ./db/admin --env="test" upgrade From 177e4d534f17ad599de265eda96397a75dbd033d Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 14:36:22 -0600 Subject: [PATCH 16/58] I'm gonna try something weird --- .github/actions/to-integration-tests/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 148242ae69..84179bcd19 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -33,7 +33,7 @@ name: dbconf.yml test: driver: postgres - open: host=localhost port=5432 user=traffic_ops password=twelve dbname=to_test sslmode=disable + open: host=postgres port=5432 user=traffic_ops password=twelve dbname=to_test sslmode=disable ' > dbconf.yml @@ -109,7 +109,7 @@ cat <<-EOF >database.conf { "description": "Local PostgreSQL database on port 5432", "dbname": "traffic_ops", - "hostname": "localhost", + "hostname": "postgres", "user": "traffic_ops", "password": "twelve", "port": "5432", @@ -154,7 +154,7 @@ cat <<-EOF >traffic-ops-test.conf "trafficOpsDB": { "dbname": "traffic_ops", "description": "Test database to_test", - "hostname": "localhost", + "hostname": "postgres", "password": "twelve", "port": "5432", "type": "Pg", From 466c2e08ea48ba21da6873d9c5e6d97b1fbe6f27 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 15:25:18 -0600 Subject: [PATCH 17/58] Fix dbname --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 84179bcd19..a3e059075e 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -33,7 +33,7 @@ name: dbconf.yml test: driver: postgres - open: host=postgres port=5432 user=traffic_ops password=twelve dbname=to_test sslmode=disable + open: host=postgres port=5432 user=traffic_ops password=twelve dbname=traffic_ops sslmode=disable ' > dbconf.yml From 5a651050a937606bd950dada31d7452ee7246891 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 15:32:42 -0600 Subject: [PATCH 18/58] Change command from upgrade to reset --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index a3e059075e..9bfeaa3780 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -39,7 +39,7 @@ test: cd .. -./db/admin --env="test" upgrade +./db/admin --env="test" reset cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" From 1d6b9c3023848e9554e9f1894b1c1aaa2ee488f7 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 15:36:27 -0600 Subject: [PATCH 19/58] Install postgresql client --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 308d9ce946..8890b4b410 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2 -RUN apt-get install openssl git gcc +RUN apt-get install openssl git gcc postgresql RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 65f6de3d1d1a627e2e0c8445e034d7e8af4a9c20 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 15:44:25 -0600 Subject: [PATCH 20/58] Install postgresql client --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 8890b4b410..258e377c12 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2 -RUN apt-get install openssl git gcc postgresql +RUN apt-get install openssl git gcc postgresql-client RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 646e39a4fac8e072ef7e65f523e9a6c845197b42 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 16:00:05 -0600 Subject: [PATCH 21/58] wat --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 258e377c12..19d5790486 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2 +FROM golang:1.14.2-ubuntu RUN apt-get install openssl git gcc postgresql-client RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq From 25ee0c6604bf8382a98a61363ddecce38e6fe801 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 16:13:40 -0600 Subject: [PATCH 22/58] Why can't I install postgresql-client??? --- .github/actions/to-integration-tests/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 19d5790486..3ba5502ea6 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-ubuntu +FROM golang:1.14.2 +RUN lsb_release RUN apt-get install openssl git gcc postgresql-client RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From e4059152e3ff7a1aa523cd7e7cf1799c473c948d Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 16:20:16 -0600 Subject: [PATCH 23/58] Refine package name --- .github/actions/to-integration-tests/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 3ba5502ea6..475ecc8867 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,10 +15,11 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2 +FROM golang:1.14.2-buster -RUN lsb_release -RUN apt-get install openssl git gcc postgresql-client +RUN apt-get update &&\ + apt-get install-y --no-install-recommends openssl git gcc postgresql-client-11 &&\ + rm -rf /var/lib/apt/lists/* RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 9449faca02d78dd2b1a8da82b638ca496080beec Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 16:22:38 -0600 Subject: [PATCH 24/58] Forgot a space --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 475ecc8867..2eaac1329d 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -18,7 +18,7 @@ FROM golang:1.14.2-buster RUN apt-get update &&\ - apt-get install-y --no-install-recommends openssl git gcc postgresql-client-11 &&\ + apt-get install -y --no-install-recommends openssl git gcc postgresql-client-11 &&\ rm -rf /var/lib/apt/lists/* RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 302f1fd6739cd9428d120e9e3f289a57ab514852 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 16:36:07 -0600 Subject: [PATCH 25/58] Calling goose manually --- .../actions/to-integration-tests/entrypoint.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 9bfeaa3780..cb995ae6db 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -25,8 +25,8 @@ ln -s "$PWD" "$SRCDIR/trafficcontrol" cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" -/usr/local/go/bin/go get ./... -/usr/local/go/bin/go build ./admin.go +# /usr/local/go/bin/go get ./... +# /usr/local/go/bin/go build ./admin.go echo 'version: "1.0" name: dbconf.yml @@ -37,12 +37,19 @@ test: ' > dbconf.yml -cd .. +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql + +goose --env=test --path="$PWD" up -./db/admin --env="test" reset +# cd .. + +# ./db/admin --env="test" reset cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" + /usr/local/go/bin/go get ./... /usr/local/go/bin/go build . From d9e8814f9ed05ced2230e64771148a6111b93044 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 16:43:11 -0600 Subject: [PATCH 26/58] Fix db order of operations --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index cb995ae6db..af93d5ab8d 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -38,10 +38,10 @@ test: ' > dbconf.yml psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql +goose --env=test --path="$PWD" up psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql -goose --env=test --path="$PWD" up # cd .. From 5d2f902ab5a5ef8d25b230dbf99e2a57db036cf5 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:05:38 -0600 Subject: [PATCH 27/58] Add hand-made cert and key --- .../to-integration-tests/entrypoint.sh | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index af93d5ab8d..b281a9f12a 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -53,10 +53,64 @@ cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" /usr/local/go/bin/go get ./... /usr/local/go/bin/go build . -openssl req -x509 -out localhost.crt -keyout localhost.key \ - -newkey rsa:2048 -nodes -sha256 \ - -subj '/CN=localhost' -extensions EXT -config <( \ - printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") +# openssl req -x509 -out localhost.crt -keyout localhost.key \ +# -newkey rsa:2048 -nodes -sha256 \ +# -subj '/CN=localhost' -extensions EXT -config <( \ +# printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") + +echo " +-----BEGIN CERTIFICATE----- +MIIDtTCCAp2gAwIBAgIJAJgQuE9T48+gMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTcwNTA5MDIyNTI0WhcNMTgwNTA5MDIyNTI0WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA1OsuQ71qaSZ4ivGnh4YryeQEHMn5wZLX7kWYB637ssyOJnkU1ZGs93JM +XJADzsjmssP6icSDhV2JPgDDYzx1eBJt6y3vHI7L3AdGfQJj+4FFABKR8moqpc1J +WIMGnPzO6DeEc8irf0qxSh+yvuFX0j6oS8oCqiRxz5+HL2wEGWmrgr37JY4/bs7o +4CMY19Ru1dP2Fr292HEIqCEnLTOuaHSWAEWx1Tm93kT9sXbw/SG2JTLQSX80biFL +7foJeoGWLls2reTCYTprzWFaMu3x9I8HLtf4VIN44rtvo5N20KYgjGqvPjFGPljL +yrgB8rXSCpH3M4AbazxD8fZKbdORawIDAQABo4GnMIGkMB0GA1UdDgQWBBT6zEpf +DYbYCI3Bu82+Q5SmI+/7ojB1BgNVHSMEbjBsgBT6zEpfDYbYCI3Bu82+Q5SmI+/7 +oqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNV +BAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAJgQuE9T48+gMAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAGLs1NcYNtUgN6FuMb6/UskEWLTKwfno +NBtNdIbcZP3HmJHwruLWCeqj6HIWJC87EqmPTIYPdem3SAN1L20fWpzm7AB7av+2 +wTCAPVP0punF/IouSb6fyo8fdG1a104Mge4iy/Sf2uf09NEv08sfVdB4P0tKRRlg +5KChhmspdPP7fmPXyghm4IC0Seknmh6IlVOnALXLU5OoCLHTie5Hjv4Tm8Xu0oBA +dIH/cPu2/w5SAIVq9CtcsdglS0ZsCAv4W2YieuSLPf5xuI0q/5lFZGNoDpIWJldx +Y2IpnoNCrHEAxijP5ctPawsxkSt2PmQ5uNNL7TbMudc3hZzOpTPkGoo= +-----END CERTIFICATE----- +" > localhost.crt + +echo "-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA1OsuQ71qaSZ4ivGnh4YryeQEHMn5wZLX7kWYB637ssyOJnkU +1ZGs93JMXJADzsjmssP6icSDhV2JPgDDYzx1eBJt6y3vHI7L3AdGfQJj+4FFABKR +8moqpc1JWIMGnPzO6DeEc8irf0qxSh+yvuFX0j6oS8oCqiRxz5+HL2wEGWmrgr37 +JY4/bs7o4CMY19Ru1dP2Fr292HEIqCEnLTOuaHSWAEWx1Tm93kT9sXbw/SG2JTLQ +SX80biFL7foJeoGWLls2reTCYTprzWFaMu3x9I8HLtf4VIN44rtvo5N20KYgjGqv +PjFGPljLyrgB8rXSCpH3M4AbazxD8fZKbdORawIDAQABAoIBAQCYOINc/qCDCHQJ +sfa511ya/B9MjcG3eMpTmQG2C9b033WJX+tbPMjSJ68cRgHS5qK4j5AgypPU1yh1 +YYpO+jxpWZOoHbDjU9u/NJxaZ0kf2C2CfcRF8U0IOJoFY7doqP0r2/Uf6glh+f6C +JeNewDBPKWictpHtHh0X+M9nQew0VZ7slXnV+IwUxiWYEtiIjwMyzSmfDEnN3ix5 +fuVQLvVaq+bbqXj2rMpJWFj7zMsG5HRePQl2kQGtMYLCIalnJIQs5jQn2YsliNyy +fQiwWnU0wkrLlmkhlivlISRDtP35WQgF8ObsoQ3LZXRflB0C7U7zEl7Dj3Vi7WXr +jsRZC4dxAoGBAPwuPdtc9gSNKjn8RnqfEJjdSo1qdLbGvRcSJNy4/kEEFECJXkeO +mV/aklCi39cxAaIjVdTQ1XN67RMxgdekCI2Eg8h4RdvwgB/tAO+C3ExzMSOA1IcZ +tWuwIA2YnaFF9Sla9iJqxgtoGlaqm4VTUM/IdZqlzsP08pfNq7bXPsr9AoGBANgk +tkovf1Y0O4lBHX3eVLnHXForxEZh8bGHwuJJWWzb0ZFcXrrSd8FSycZrR28v4sdQ +WSSVPz3Op95HoTVXVL9EJcZ+MTnHaoCHbYBkrGTlGviu5Fl2V5EbrN7R7CdxJeem +HOU4shTy1acMPgf8sT17ykkXhVeUhSK2Jg6fZn6HAoGAI4/51SeE4htuKwMyhTRN +SOFcFBlBIE1ieRBr9lx4Ln7+xCMbEog/hM7z9z8gxd35Vv4YqoxQrZpWOHCw2NIf +CqX3V5vubhe6WcY4bY5MttM/yLvwPKUZeng57PDqucV9zzkuoKfiCdXCcRpaGDEp +okOooghj4ip204WDg6NTDZkCgYEAwZTfzsGLgmF1kRBIoZqmt1zeUcQxHfhKx32Y +BaM7/EtD/rSEAz7NEtBa9uLOL77rlSdZL3KcGXck0efFckitFkCqtIQBAoaf1E12 +vS9tV0/6QBAjZByhgM0Qnt/Uad7k2/vilUmZ9TkoMVy9kdm3xCFCowP14OKb+uK4 +YxBQc7ECgYEAm7eVtNlPHYmC54FU2bLucryNMLmu9I8O6zvbK5sxiMdtlh7OjaUB +RQS5iVc0iTacDZTGh7eqNzgGplj76pWGHeZUy0xIj/ZIRu2qOy0v+ffqfX1wCz7p +A22D22wvfs7CE3cUz/8UnvLM3kbTTu1WbbBbrHjAV47sAHjW/ckTqeo= +-----END RSA PRIVATE KEY----- +" > localhost.key cat <<-EOF >cdn.conf { From ab35d454392085c830b93a46f2855bd11755902b Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:10:46 -0600 Subject: [PATCH 28/58] fix path to go binary --- .github/actions/to-integration-tests/entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index b281a9f12a..f60a47f2e2 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -224,6 +224,9 @@ cat <<-EOF >traffic-ops-test.conf } EOF -go test -v --cfg ./traffic-ops-test +/usr/local/bin/go test -v --cfg ./traffic-ops-test + +cat ../traffic_ops_golang/out.log +cat ../traffic_ops_golang/err.log exit $? From 168d91291aa02f4b5c7ec5c31eba1405cf80b6de Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:14:22 -0600 Subject: [PATCH 29/58] fix path to go binary - but actually --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index f60a47f2e2..d82fd25fb0 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -224,7 +224,7 @@ cat <<-EOF >traffic-ops-test.conf } EOF -/usr/local/bin/go test -v --cfg ./traffic-ops-test +/usr/local/go/bin/go test -v --cfg ./traffic-ops-test cat ../traffic_ops_golang/out.log cat ../traffic_ops_golang/err.log From dce08a6ec1c0c0a34a503e57ae4e559af0273c10 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:18:02 -0600 Subject: [PATCH 30/58] fix configuration file name --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index d82fd25fb0..3bec98e800 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -224,7 +224,7 @@ cat <<-EOF >traffic-ops-test.conf } EOF -/usr/local/go/bin/go test -v --cfg ./traffic-ops-test +/usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf cat ../traffic_ops_golang/out.log cat ../traffic_ops_golang/err.log From 9630642443efc1d41677504cd814dc208bd66735 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:23:49 -0600 Subject: [PATCH 31/58] remove unused code --- .../actions/to-integration-tests/entrypoint.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 3bec98e800..5a4bcb93f4 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -25,9 +25,6 @@ ln -s "$PWD" "$SRCDIR/trafficcontrol" cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" -# /usr/local/go/bin/go get ./... -# /usr/local/go/bin/go build ./admin.go - echo 'version: "1.0" name: dbconf.yml @@ -42,22 +39,12 @@ goose --env=test --path="$PWD" up psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql - -# cd .. - -# ./db/admin --env="test" reset - cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" /usr/local/go/bin/go get ./... /usr/local/go/bin/go build . -# openssl req -x509 -out localhost.crt -keyout localhost.key \ -# -newkey rsa:2048 -nodes -sha256 \ -# -subj '/CN=localhost' -extensions EXT -config <( \ -# printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") - echo " -----BEGIN CERTIFICATE----- MIIDtTCCAp2gAwIBAgIJAJgQuE9T48+gMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV @@ -179,8 +166,6 @@ cat <<-EOF >database.conf } EOF - - ./traffic_ops_golang --cfg ./cdn.conf --dbcfg ./database.conf >out.log 2>err.log & cd ../testing/api/v1 From 94eacb17eac2caf30642839d040137ef1633000f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:32:28 -0600 Subject: [PATCH 32/58] Add running v2 and v3 tests --- .../to-integration-tests/entrypoint.sh | 220 ++++++++++-------- 1 file changed, 121 insertions(+), 99 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 5a4bcb93f4..a5eb83ba66 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -34,10 +34,10 @@ test: ' > dbconf.yml -psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql >/dev/null goose --env=test --path="$PWD" up -psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql -psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql >/dev/null +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql >/dev/null cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" @@ -101,68 +101,68 @@ A22D22wvfs7CE3cUz/8UnvLM3kbTTu1WbbBbrHjAV47sAHjW/ckTqeo= cat <<-EOF >cdn.conf { - "hypnotoad" : { - "listen" : [ - "https://not-a-real-host.test:1?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" - ], - "user" : "trafops", - "group" : "trafops", - "heartbeat_timeout" : 20, - "pid_file" : "/var/run/traffic_ops.pid", - "workers" : 12 - }, - "use_ims": true, - "traffic_ops_golang" : { - "insecure": true, - "port" : "6443", - "log_location_error": "stderr", - "log_location_warning": "stderr", - "log_location_info": "stderr", - "log_location_debug": "stderr", - "log_location_event": "stderr", - "max_db_connections": 20, - "db_conn_max_lifetime_seconds": 60, - "db_query_timeout_seconds": 20, - "supported_ds_metrics": [ "kbps", "tps_total", "tps_2xx", "tps_3xx", "tps_4xx", "tps_5xx" ] - }, - "cors" : { - "access_control_allow_origin" : "*" - }, - "to" : { - "base_url" : "https://localhost", - "email_from" : "no-reply@traffic-ops.test", - "no_account_found_msg" : "A Traffic Ops user account is required for access. Please contact your Traffic Ops user administrator." - }, - "portal" : { - "base_url" : "https://not-a-real-host.test/#!/", - "email_from" : "no-reply@traffic-portal.test", - "pass_reset_path" : "user", - "user_register_path" : "user" - }, - "secrets" : [ - "blahblah" - ], - "geniso" : { - "iso_root_path" : "/opt/traffic_ops/app/public" - }, - "inactivity_timeout" : 60, - "smtp" : { - "enabled" : false - }, - "InfluxEnabled": false + "hypnotoad" : { + "listen" : [ + "https://not-a-real-host.test:1?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" + ], + "user" : "trafops", + "group" : "trafops", + "heartbeat_timeout" : 20, + "pid_file" : "/var/run/traffic_ops.pid", + "workers" : 12 + }, + "use_ims": true, + "traffic_ops_golang" : { + "insecure": true, + "port" : "6443", + "log_location_error": "stderr", + "log_location_warning": "stderr", + "log_location_info": "stderr", + "log_location_debug": "stderr", + "log_location_event": "stderr", + "max_db_connections": 20, + "db_conn_max_lifetime_seconds": 60, + "db_query_timeout_seconds": 20, + "supported_ds_metrics": [ "kbps", "tps_total", "tps_2xx", "tps_3xx", "tps_4xx", "tps_5xx" ] + }, + "cors" : { + "access_control_allow_origin" : "*" + }, + "to" : { + "base_url" : "https://localhost", + "email_from" : "no-reply@traffic-ops.test", + "no_account_found_msg" : "A Traffic Ops user account is required for access. Please contact your Traffic Ops user administrator." + }, + "portal" : { + "base_url" : "https://not-a-real-host.test/#!/", + "email_from" : "no-reply@traffic-portal.test", + "pass_reset_path" : "user", + "user_register_path" : "user" + }, + "secrets" : [ + "blahblah" + ], + "geniso" : { + "iso_root_path" : "/opt/traffic_ops/app/public" + }, + "inactivity_timeout" : 60, + "smtp" : { + "enabled" : false + }, + "InfluxEnabled": false } EOF cat <<-EOF >database.conf { - "description": "Local PostgreSQL database on port 5432", - "dbname": "traffic_ops", - "hostname": "postgres", - "user": "traffic_ops", - "password": "twelve", - "port": "5432", - "ssl": false, - "type": "Pg" + "description": "Local PostgreSQL database on port 5432", + "dbname": "traffic_ops", + "hostname": "postgres", + "user": "traffic_ops", + "password": "twelve", + "port": "5432", + "ssl": false, + "type": "Pg" } EOF @@ -170,48 +170,70 @@ EOF cd ../testing/api/v1 -cat <<-EOF >traffic-ops-test.conf -{ - "default": { - "logLocations": { - "debug": "stdout", - "error": "stdout", - "event": "stdout", - "info": "stdout", - "warning": "stdout" - }, - "session": { - "timeoutInSecs": 60 - } - }, - "trafficOps": { - "URL": "https://localhost:6443", - "password": "twelve", - "users": { - "disallowed": "disallowed", - "operations": "operations", - "admin": "admin", - "federation": "federation", - "portal": "portal", - "readOnly": "readOnly", - "extension": "extension" - } - }, - "trafficOpsDB": { - "dbname": "traffic_ops", - "description": "Test database to_test", - "hostname": "postgres", - "password": "twelve", - "port": "5432", - "type": "Pg", - "user": "traffic_ops" - } +makeCFG() { + cat <<-EOF >traffic-ops-test.conf + { + "default": { + "logLocations": { + "debug": "stdout", + "error": "stdout", + "event": "stdout", + "info": "stdout", + "warning": "stdout" + }, + "session": { + "timeoutInSecs": 60 + } + }, + "trafficOps": { + "URL": "https://localhost:6443", + "password": "twelve", + "users": { + "disallowed": "disallowed", + "operations": "operations", + "admin": "admin", + "federation": "federation", + "portal": "portal", + "readOnly": "readOnly", + "extension": "extension" + } + }, + "trafficOpsDB": { + "dbname": "traffic_ops", + "description": "Test database to_test", + "hostname": "postgres", + "password": "twelve", + "port": "5432", + "type": "Pg", + "user": "traffic_ops" + } + } + EOF } -EOF -/usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf +CODE="0" + +makeCFG +if [ ! /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf ]; then + CODE="1" +fi +rm traffic-ops-test.conf + +cd ../v2 +makeCFG +if [ ! /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf ]; then + CODE="1" +fi +rm traffic-ops-test.conf + +cd ../v3 +makeCFG +if [ ! /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf ]; then + CODE="1" +fi +rm traffic-ops-test cat ../traffic_ops_golang/out.log cat ../traffic_ops_golang/err.log -exit $? +exit "$CODE" From 0df31d8f08f4fe3c1af229259ec3597e5aabb74d Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 17:37:07 -0600 Subject: [PATCH 33/58] Fix if/then syntax --- .github/actions/to-integration-tests/entrypoint.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index a5eb83ba66..098372693f 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -214,21 +214,24 @@ makeCFG() { CODE="0" makeCFG -if [ ! /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf ]; then +/usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf +if [ "$?" -gt "0" ]; then CODE="1" fi rm traffic-ops-test.conf cd ../v2 makeCFG -if [ ! /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf ]; then + /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf +if [ "$?" -gt "0" ]; then CODE="1" fi rm traffic-ops-test.conf cd ../v3 makeCFG -if [ ! /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf ]; then + /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf +if [ "$?" -gt "0" ]; then CODE="1" fi rm traffic-ops-test From 6a63efddef881b8529cf75f7a6a042161cedda68 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 18:04:01 -0600 Subject: [PATCH 34/58] Remove echoing and errexit --- .github/actions/to-integration-tests/entrypoint.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 098372693f..4ecf34cea5 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -16,8 +16,6 @@ # specific language governing permissions and limitations # under the License. -set -ex - GOPATH="$(mktemp -d)" SRCDIR="$GOPATH/src/github.com/apache" mkdir -p "$SRCDIR" From 0b033ab701b023db67b17b59910985edfbc2b4af Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 18:17:45 -0600 Subject: [PATCH 35/58] fix path to logfiles --- .github/actions/to-integration-tests/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 4ecf34cea5..aab97a9765 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -234,7 +234,7 @@ if [ "$?" -gt "0" ]; then fi rm traffic-ops-test -cat ../traffic_ops_golang/out.log -cat ../traffic_ops_golang/err.log +cat ../../../traffic_ops_golang/out.log +cat ../../../traffic_ops_golang/err.log exit "$CODE" From e9b369fb271b41411b17eb1dd87028989b307c13 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 18:31:57 -0600 Subject: [PATCH 36/58] shorten logs --- .../actions/to-integration-tests/entrypoint.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index aab97a9765..e2f22092a0 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -113,11 +113,11 @@ cat <<-EOF >cdn.conf "traffic_ops_golang" : { "insecure": true, "port" : "6443", - "log_location_error": "stderr", + "log_location_error": "stderr, "log_location_warning": "stderr", - "log_location_info": "stderr", - "log_location_debug": "stderr", - "log_location_event": "stderr", + "log_location_info": null, + "log_location_debug": null, + "log_location_event": null, "max_db_connections": 20, "db_conn_max_lifetime_seconds": 60, "db_query_timeout_seconds": 20, @@ -173,11 +173,11 @@ makeCFG() { { "default": { "logLocations": { - "debug": "stdout", - "error": "stdout", - "event": "stdout", - "info": "stdout", - "warning": "stdout" + "debug": "stderr", + "error": "stderr", + "event": "stderr", + "info": "stderr", + "warning": "stderr" }, "session": { "timeoutInSecs": 60 From 91effda35303e81e59e294ff60f3380fcc91bee6 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 22:55:36 -0600 Subject: [PATCH 37/58] Add noPerl and noISO to testing configuration --- .github/actions/to-integration-tests/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index e2f22092a0..dcb0c9b532 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -204,7 +204,9 @@ makeCFG() { "port": "5432", "type": "Pg", "user": "traffic_ops" - } + }, + "noPerl": true, + "noISO": true } EOF } From c2083b89fdccb2c7e500622c6dc28cc8e2d19a39 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 23:03:41 -0600 Subject: [PATCH 38/58] Fix no closing " --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index dcb0c9b532..eb3451efc7 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -113,7 +113,7 @@ cat <<-EOF >cdn.conf "traffic_ops_golang" : { "insecure": true, "port" : "6443", - "log_location_error": "stderr, + "log_location_error": "stderr", "log_location_warning": "stderr", "log_location_info": null, "log_location_debug": null, From 5d686570174b2bd696095832a0047a1eeca97795 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Thu, 13 Aug 2020 23:40:02 -0600 Subject: [PATCH 39/58] Add missed Perl-dependent tests --- traffic_ops/testing/api/v1/federation_users_test.go | 3 +++ traffic_ops/testing/api/v1/federations_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/traffic_ops/testing/api/v1/federation_users_test.go b/traffic_ops/testing/api/v1/federation_users_test.go index 4473c356ef..2ec7417bb1 100644 --- a/traffic_ops/testing/api/v1/federation_users_test.go +++ b/traffic_ops/testing/api/v1/federation_users_test.go @@ -20,6 +20,9 @@ import ( ) func TestFederationUsers(t *testing.T) { + if Config.NoPerl { + t.Skip("No Perl instance for proxying") + } WithObjs(t, []TCObj{CDNs, Types, Tenants, Users, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, DeliveryServices, UsersDeliveryServices, CDNFederations, FederationUsers}, func() { CreateTestInvalidFederationUsers(t) GetTestInvalidFederationIDUsers(t) diff --git a/traffic_ops/testing/api/v1/federations_test.go b/traffic_ops/testing/api/v1/federations_test.go index 0b62080134..28c608ba9a 100644 --- a/traffic_ops/testing/api/v1/federations_test.go +++ b/traffic_ops/testing/api/v1/federations_test.go @@ -24,6 +24,9 @@ import ( ) func TestFederations(t *testing.T) { + if Config.NoPerl { + t.Skip("No Perl instance for proxying") + } WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, DeliveryServices, UsersDeliveryServices, CDNFederations}, func() { PostDeleteTestFederationsDeliveryServices(t) GetTestFederations(t) From 919564af4cf0d6ab576c2b0155bc58e5b4dedfb2 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 14 Aug 2020 10:53:43 -0600 Subject: [PATCH 40/58] Remove extraneous tests --- .github/workflows/traffic ops.yml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 0e041987c1..3bf6b025b5 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -15,35 +15,15 @@ # specific language governing permissions and limitations # under the License. -name: Traffic Ops Push CI +name: Traffic Ops Go client/API integration tests on: push: + create: pull_request: types: [opened, reopened, edited, synchronize] jobs: - unit_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@master - - name: Run Go unit tests - uses: ./.github/actions/go-test - with: - dir: ./traffic_ops/traffic_ops_golang/... - - go_fmt: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@master - - name: Go fmt Traffic Ops - uses: ./.github/actions/go-fmt/ - with: - dir: ./traffic_ops/traffic_ops_golang/... API_tests: runs-on: ubuntu-latest From eee25b867143d29f1e2070dabceeb488b2280488 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 14 Aug 2020 12:38:48 -0600 Subject: [PATCH 41/58] Remove unused dependency, add path matching --- .github/actions/to-integration-tests/Dockerfile | 2 +- .github/workflows/traffic ops.yml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 2eaac1329d..e304598da1 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -18,7 +18,7 @@ FROM golang:1.14.2-buster RUN apt-get update &&\ - apt-get install -y --no-install-recommends openssl git gcc postgresql-client-11 &&\ + apt-get install -y --no-install-recommends git gcc postgresql-client-11 &&\ rm -rf /var/lib/apt/lists/* RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 3bf6b025b5..4795913e62 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -19,8 +19,16 @@ name: Traffic Ops Go client/API integration tests on: push: + paths: + - traffic_ops/traffic_ops_golang/**.go + - traffic_ops/testing/api/**.go + - traffic_ops/*client/**.go create: pull_request: + paths: + - traffic_ops/traffic_ops_golang/**.go + - traffic_ops/testing/api/**.go + - traffic_ops/*client/**.go types: [opened, reopened, edited, synchronize] jobs: From 6a3aa3f7300ecbcde9f35b5a67e1ad0363ac165f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 14 Aug 2020 12:43:17 -0600 Subject: [PATCH 42/58] Fix removing non-existent file --- .github/actions/to-integration-tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index eb3451efc7..0ae64fbcdd 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -234,7 +234,7 @@ makeCFG if [ "$?" -gt "0" ]; then CODE="1" fi -rm traffic-ops-test +rm traffic-ops-test.conf cat ../../../traffic_ops_golang/out.log cat ../../../traffic_ops_golang/err.log From 5dd3e8f4589cdf7576acec8a7ed3673f0e72dfe4 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 08:32:03 -0600 Subject: [PATCH 43/58] Move configuration files out into actual files --- .../actions/to-integration-tests/Dockerfile | 4 +- .github/actions/to-integration-tests/cdn.json | 51 +++++++ .../to-integration-tests/database.json | 10 ++ .../actions/to-integration-tests/dbconf.yml | 23 ++++ .../to-integration-tests/entrypoint.sh | 126 +----------------- .../traffic-ops-test.json | 38 ++++++ 6 files changed, 130 insertions(+), 122 deletions(-) create mode 100644 .github/actions/to-integration-tests/cdn.json create mode 100644 .github/actions/to-integration-tests/database.json create mode 100644 .github/actions/to-integration-tests/dbconf.yml create mode 100644 .github/actions/to-integration-tests/traffic-ops-test.json diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index e304598da1..8294dd09d9 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -18,12 +18,12 @@ FROM golang:1.14.2-buster RUN apt-get update &&\ - apt-get install -y --no-install-recommends git gcc postgresql-client-11 &&\ + apt-get install -y --no-install-recommends git gcc postgresql-client-11 envsubst &&\ rm -rf /var/lib/apt/lists/* RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ -COPY entrypoint.sh /entrypoint.sh +COPY entrypoint.sh database.json traffic-ops-test.json dbconf.yml cdn.json / RUN chmod a+x /entrypoint.sh ENTRYPOINT /entrypoint.sh diff --git a/.github/actions/to-integration-tests/cdn.json b/.github/actions/to-integration-tests/cdn.json new file mode 100644 index 0000000000..efda921f66 --- /dev/null +++ b/.github/actions/to-integration-tests/cdn.json @@ -0,0 +1,51 @@ +{ + "hypnotoad" : { + "listen" : [ + "https://not-a-real-host.test:1?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" + ], + "user" : "trafops", + "group" : "trafops", + "heartbeat_timeout" : 20, + "pid_file" : "/var/run/traffic_ops.pid", + "workers" : 12 + }, + "use_ims": true, + "traffic_ops_golang" : { + "insecure": true, + "port" : "6443", + "log_location_error": "stderr", + "log_location_warning": "stderr", + "log_location_info": null, + "log_location_debug": null, + "log_location_event": null, + "max_db_connections": 20, + "db_conn_max_lifetime_seconds": 60, + "db_query_timeout_seconds": 20, + "supported_ds_metrics": [ "kbps", "tps_total", "tps_2xx", "tps_3xx", "tps_4xx", "tps_5xx" ] + }, + "cors" : { + "access_control_allow_origin" : "*" + }, + "to" : { + "base_url" : "https://localhost", + "email_from" : "no-reply@traffic-ops.test", + "no_account_found_msg" : "A Traffic Ops user account is required for access. Please contact your Traffic Ops user administrator." + }, + "portal" : { + "base_url" : "https://not-a-real-host.test/#!/", + "email_from" : "no-reply@traffic-portal.test", + "pass_reset_path" : "user", + "user_register_path" : "user" + }, + "secrets" : [ + "blahblah" + ], + "geniso" : { + "iso_root_path" : "/opt/traffic_ops/app/public" + }, + "inactivity_timeout" : 60, + "smtp" : { + "enabled" : false + }, + "InfluxEnabled": false +} diff --git a/.github/actions/to-integration-tests/database.json b/.github/actions/to-integration-tests/database.json new file mode 100644 index 0000000000..8d08eb40d4 --- /dev/null +++ b/.github/actions/to-integration-tests/database.json @@ -0,0 +1,10 @@ +{ + "description": "Local PostgreSQL database on port 5432", + "dbname": "traffic_ops", + "hostname": "postgres", + "user": "traffic_ops", + "password": "twelve", + "port": "5432", + "ssl": false, + "type": "Pg" +} diff --git a/.github/actions/to-integration-tests/dbconf.yml b/.github/actions/to-integration-tests/dbconf.yml new file mode 100644 index 0000000000..f565670b38 --- /dev/null +++ b/.github/actions/to-integration-tests/dbconf.yml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "1.0" +name: dbconf.yml + +test: + driver: postgres + open: host=postgres port=5432 user=traffic_ops password=twelve dbname=traffic_ops sslmode=disable + diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index eb3451efc7..44a56b38df 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -23,14 +23,7 @@ ln -s "$PWD" "$SRCDIR/trafficcontrol" cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" -echo 'version: "1.0" -name: dbconf.yml - -test: - driver: postgres - open: host=postgres port=5432 user=traffic_ops password=twelve dbname=traffic_ops sslmode=disable - -' > dbconf.yml +mv /dbconf.yml ./ psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql >/dev/null goose --env=test --path="$PWD" up @@ -97,123 +90,16 @@ A22D22wvfs7CE3cUz/8UnvLM3kbTTu1WbbBbrHjAV47sAHjW/ckTqeo= -----END RSA PRIVATE KEY----- " > localhost.key -cat <<-EOF >cdn.conf -{ - "hypnotoad" : { - "listen" : [ - "https://not-a-real-host.test:1?cert=$PWD/localhost.crt&key=$PWD/localhost.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED" - ], - "user" : "trafops", - "group" : "trafops", - "heartbeat_timeout" : 20, - "pid_file" : "/var/run/traffic_ops.pid", - "workers" : 12 - }, - "use_ims": true, - "traffic_ops_golang" : { - "insecure": true, - "port" : "6443", - "log_location_error": "stderr", - "log_location_warning": "stderr", - "log_location_info": null, - "log_location_debug": null, - "log_location_event": null, - "max_db_connections": 20, - "db_conn_max_lifetime_seconds": 60, - "db_query_timeout_seconds": 20, - "supported_ds_metrics": [ "kbps", "tps_total", "tps_2xx", "tps_3xx", "tps_4xx", "tps_5xx" ] - }, - "cors" : { - "access_control_allow_origin" : "*" - }, - "to" : { - "base_url" : "https://localhost", - "email_from" : "no-reply@traffic-ops.test", - "no_account_found_msg" : "A Traffic Ops user account is required for access. Please contact your Traffic Ops user administrator." - }, - "portal" : { - "base_url" : "https://not-a-real-host.test/#!/", - "email_from" : "no-reply@traffic-portal.test", - "pass_reset_path" : "user", - "user_register_path" : "user" - }, - "secrets" : [ - "blahblah" - ], - "geniso" : { - "iso_root_path" : "/opt/traffic_ops/app/public" - }, - "inactivity_timeout" : 60, - "smtp" : { - "enabled" : false - }, - "InfluxEnabled": false -} -EOF - -cat <<-EOF >database.conf -{ - "description": "Local PostgreSQL database on port 5432", - "dbname": "traffic_ops", - "hostname": "postgres", - "user": "traffic_ops", - "password": "twelve", - "port": "5432", - "ssl": false, - "type": "Pg" -} -EOF +envsubst cdn.conf +mv /database.json ./database.conf ./traffic_ops_golang --cfg ./cdn.conf --dbcfg ./database.conf >out.log 2>err.log & cd ../testing/api/v1 -makeCFG() { - cat <<-EOF >traffic-ops-test.conf - { - "default": { - "logLocations": { - "debug": "stderr", - "error": "stderr", - "event": "stderr", - "info": "stderr", - "warning": "stderr" - }, - "session": { - "timeoutInSecs": 60 - } - }, - "trafficOps": { - "URL": "https://localhost:6443", - "password": "twelve", - "users": { - "disallowed": "disallowed", - "operations": "operations", - "admin": "admin", - "federation": "federation", - "portal": "portal", - "readOnly": "readOnly", - "extension": "extension" - } - }, - "trafficOpsDB": { - "dbname": "traffic_ops", - "description": "Test database to_test", - "hostname": "postgres", - "password": "twelve", - "port": "5432", - "type": "Pg", - "user": "traffic_ops" - }, - "noPerl": true, - "noISO": true - } - EOF -} - CODE="0" -makeCFG +cp /traffic-ops-test.json ./traffic-ops-test.conf /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf if [ "$?" -gt "0" ]; then CODE="1" @@ -221,7 +107,7 @@ fi rm traffic-ops-test.conf cd ../v2 -makeCFG +cp /traffic-ops-test.json ./traffic-ops-test.conf /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf if [ "$?" -gt "0" ]; then CODE="1" @@ -229,7 +115,7 @@ fi rm traffic-ops-test.conf cd ../v3 -makeCFG +cp /traffic-ops-test.json ./traffic-ops-test.conf /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf if [ "$?" -gt "0" ]; then CODE="1" diff --git a/.github/actions/to-integration-tests/traffic-ops-test.json b/.github/actions/to-integration-tests/traffic-ops-test.json new file mode 100644 index 0000000000..78c96eac95 --- /dev/null +++ b/.github/actions/to-integration-tests/traffic-ops-test.json @@ -0,0 +1,38 @@ +{ + "default": { + "logLocations": { + "debug": "stderr", + "error": "stderr", + "event": "stderr", + "info": "stderr", + "warning": "stderr" + }, + "session": { + "timeoutInSecs": 60 + } + }, + "trafficOps": { + "URL": "https://localhost:6443", + "password": "twelve", + "users": { + "disallowed": "disallowed", + "operations": "operations", + "admin": "admin", + "federation": "federation", + "portal": "portal", + "readOnly": "readOnly", + "extension": "extension" + } + }, + "trafficOpsDB": { + "dbname": "traffic_ops", + "description": "Test database to_test", + "hostname": "postgres", + "password": "twelve", + "port": "5432", + "type": "Pg", + "user": "traffic_ops" + }, + "noPerl": true, + "noISO": true +} From 69c12bb44760002ea1fc641bfa3158edbd7ecda3 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 08:35:30 -0600 Subject: [PATCH 44/58] Fix package name --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 8294dd09d9..52e73a5220 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -18,7 +18,7 @@ FROM golang:1.14.2-buster RUN apt-get update &&\ - apt-get install -y --no-install-recommends git gcc postgresql-client-11 envsubst &&\ + apt-get install -y --no-install-recommends git gcc postgresql-client-11 gettext-base &&\ rm -rf /var/lib/apt/lists/* RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 89a5c18cf3465cd7424164060ad5f98f92292553 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:32:47 -0600 Subject: [PATCH 45/58] Move different versions into different steps --- .../actions/to-integration-tests/README.md | 7 ++++ .../actions/to-integration-tests/action.yml | 2 ++ .../to-integration-tests/entrypoint.sh | 32 +++++++------------ .github/workflows/traffic ops.yml | 12 ++++++- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.github/actions/to-integration-tests/README.md b/.github/actions/to-integration-tests/README.md index b468bdc9e4..f06b5bb546 100644 --- a/.github/actions/to-integration-tests/README.md +++ b/.github/actions/to-integration-tests/README.md @@ -21,6 +21,13 @@ This action runs the Traffic Ops Go client integration tests with the Traffic Ops API. +## Inputs + +### `version` +**Required** Major API version to test e.g. 1, 2, 3 etc. + +## Outputs + ### `exit-code` 1 if the Go program(s) could be built successfully. diff --git a/.github/actions/to-integration-tests/action.yml b/.github/actions/to-integration-tests/action.yml index 4afee44811..acd70a16c1 100644 --- a/.github/actions/to-integration-tests/action.yml +++ b/.github/actions/to-integration-tests/action.yml @@ -20,3 +20,5 @@ description: 'Runs Traffic Ops Go client/API integration tests' runs: using: 'docker' image: 'Dockerfile' + args: + - ${{ inputs.version }} diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index e4aaed95f3..5aaad56057 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -95,34 +95,26 @@ mv /database.json ./database.conf ./traffic_ops_golang --cfg ./cdn.conf --dbcfg ./database.conf >out.log 2>err.log & -cd ../testing/api/v1 +if [ -z "$INPUT_VERSION" ]; then + INPUT_VERSION="3"; +fi -CODE="0" +cd "../testing/api/v$INPUT_VERSION" cp /traffic-ops-test.json ./traffic-ops-test.conf /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf -if [ "$?" -gt "0" ]; then - CODE="1" -fi +CODE="$?" rm traffic-ops-test.conf -cd ../v2 -cp /traffic-ops-test.json ./traffic-ops-test.conf - /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf -if [ "$?" -gt "0" ]; then - CODE="1" +# TODO - make these build artifacts +if [ -f ../../../traffic_ops_golang/out.log ]; then + cat ../../../traffic_ops_golang/out.log + rm ../.../../traffic_ops_golang/out.log fi -rm traffic-ops-test.conf -cd ../v3 -cp /traffic-ops-test.json ./traffic-ops-test.conf - /usr/local/go/bin/go test -v --cfg ./traffic-ops-test.conf -if [ "$?" -gt "0" ]; then - CODE="1" +if [ -f ../../../traffic_ops_golang/err.log ]; then + cat ../../../traffic_ops_golang/err.log >&2 + rm ../../../traffic_ops_golang/err.log fi -rm traffic-ops-test.conf - -cat ../../../traffic_ops_golang/out.log -cat ../../../traffic_ops_golang/err.log exit "$CODE" diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 4795913e62..0bb54e4499 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -50,5 +50,15 @@ jobs: steps: - name: Checkout uses: actions/checkout@master - - name: Run API tests + - name: Run API v1 tests uses: ./.github/actions/to-integration-tests + with: + version: 1 + - name: Run API v2 tests + uses: ./.github/actions/to-integration-tests + with: + version: 2 + - name: Run API v3 tests + uses: ./.github/actions/to-integration-tests + with: + version: 3 From 42270c43f63b1aab0583001cb37d3dfb3ec47099 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:33:49 -0600 Subject: [PATCH 46/58] Make API v2 and v3 tests always run even if earlier versions failed --- .github/workflows/traffic ops.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 0bb54e4499..0a8ef4b22e 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -55,10 +55,12 @@ jobs: with: version: 1 - name: Run API v2 tests + if: {{ $always }} uses: ./.github/actions/to-integration-tests with: version: 2 - name: Run API v3 tests + if: {{ $always }} uses: ./.github/actions/to-integration-tests with: version: 3 From e797d49b98a5faa127e0ac969998d575654a4f09 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:35:46 -0600 Subject: [PATCH 47/58] Fix 'always' syntax --- .github/workflows/traffic ops.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 0a8ef4b22e..2485491a71 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -55,12 +55,12 @@ jobs: with: version: 1 - name: Run API v2 tests - if: {{ $always }} + if: ${{ always() }} uses: ./.github/actions/to-integration-tests with: version: 2 - name: Run API v3 tests - if: {{ $always }} + if: ${{ always() }} uses: ./.github/actions/to-integration-tests with: version: 3 From 951d1710773d37001760c5f10de47f958ea569c5 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:39:50 -0600 Subject: [PATCH 48/58] Add input to action YML --- .github/actions/to-integration-tests/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/to-integration-tests/action.yml b/.github/actions/to-integration-tests/action.yml index acd70a16c1..8c9d57f130 100644 --- a/.github/actions/to-integration-tests/action.yml +++ b/.github/actions/to-integration-tests/action.yml @@ -17,6 +17,10 @@ name: 'to-integration-tests' description: 'Runs Traffic Ops Go client/API integration tests' +inputs: + version: + description: 'Version of Traffic Ops API against which to run tests, e.g. 1, 2, 3, etc.' + required: true runs: using: 'docker' image: 'Dockerfile' From 90b12bc458cc1a17a8110d331932f37af516573a Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:54:50 -0600 Subject: [PATCH 49/58] Move todb initialization into its own action --- .../actions/to-integration-tests/Dockerfile | 10 ++-- .../actions/to-integration-tests/README.md | 7 ++- .../to-integration-tests/entrypoint.sh | 9 ---- .github/actions/todb-init/Dockerfile | 27 ++++++++++ .github/actions/todb-init/README.md | 50 +++++++++++++++++++ .github/actions/todb-init/action.yml | 22 ++++++++ .github/actions/todb-init/dbconf.yml | 23 +++++++++ .github/actions/todb-init/entrypoint.sh | 28 +++++++++++ .github/workflows/traffic ops.yml | 2 + 9 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 .github/actions/todb-init/Dockerfile create mode 100644 .github/actions/todb-init/README.md create mode 100644 .github/actions/todb-init/action.yml create mode 100644 .github/actions/todb-init/dbconf.yml create mode 100644 .github/actions/todb-init/entrypoint.sh diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 52e73a5220..10509ef864 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,15 +15,11 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-buster +FROM golang:1.14.2-alpine -RUN apt-get update &&\ - apt-get install -y --no-install-recommends git gcc postgresql-client-11 gettext-base &&\ - rm -rf /var/lib/apt/lists/* -RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq -RUN mv $GOPATH/bin/goose /bin/ +RUN apk add --no-cache gettext -COPY entrypoint.sh database.json traffic-ops-test.json dbconf.yml cdn.json / +COPY entrypoint.sh database.json traffic-ops-test.json cdn.json / RUN chmod a+x /entrypoint.sh ENTRYPOINT /entrypoint.sh diff --git a/.github/actions/to-integration-tests/README.md b/.github/actions/to-integration-tests/README.md index f06b5bb546..6f81959326 100644 --- a/.github/actions/to-integration-tests/README.md +++ b/.github/actions/to-integration-tests/README.md @@ -51,5 +51,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@master - - name: ./.github/actions/to-integration-tests + - name: initialize database + uses: ./.github/actions/todb-init + - name: Run API v5 tests + uses: ./.github/actions/to-integration-tests + with: + version: 5 ``` diff --git a/.github/actions/to-integration-tests/entrypoint.sh b/.github/actions/to-integration-tests/entrypoint.sh index 5aaad56057..cc99f4a039 100644 --- a/.github/actions/to-integration-tests/entrypoint.sh +++ b/.github/actions/to-integration-tests/entrypoint.sh @@ -21,15 +21,6 @@ SRCDIR="$GOPATH/src/github.com/apache" mkdir -p "$SRCDIR" ln -s "$PWD" "$SRCDIR/trafficcontrol" -cd "$SRCDIR/trafficcontrol/traffic_ops/app/db" - -mv /dbconf.yml ./ - -psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql >/dev/null -goose --env=test --path="$PWD" up -psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql >/dev/null -psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql >/dev/null - cd "$SRCDIR/trafficcontrol/traffic_ops/traffic_ops_golang" diff --git a/.github/actions/todb-init/Dockerfile b/.github/actions/todb-init/Dockerfile new file mode 100644 index 0000000000..15fa80e316 --- /dev/null +++ b/.github/actions/todb-init/Dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM golang:1.14.2-alpine + +RUN apk add --no-cache git gcc gettext-base postgresql-client +RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq +RUN mv $GOPATH/bin/goose /bin/ + +COPY entrypoint.sh dbconf.yml / +RUN chmod a+x /entrypoint.sh + +ENTRYPOINT /entrypoint.sh diff --git a/.github/actions/todb-init/README.md b/.github/actions/todb-init/README.md new file mode 100644 index 0000000000..c341d44321 --- /dev/null +++ b/.github/actions/todb-init/README.md @@ -0,0 +1,50 @@ + + +# todb-init Docker action +This action initializes a Traffic Ops database in a locally running PostgresQL +instance. + +## Outputs + +### `exit-code` +1 if the database initialized successfully. + +## Example usage +```yaml +jobs: + tests: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:11.9 + env: + POSTGRES_USER: traffic_ops + POSTGRES_PASSWORD: twelve + POSTGRES_DB: traffic_ops + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - name: Checkout + uses: actions/checkout@master + - name: ./.github/actions/todb-init +``` diff --git a/.github/actions/todb-init/action.yml b/.github/actions/todb-init/action.yml new file mode 100644 index 0000000000..394fa964af --- /dev/null +++ b/.github/actions/todb-init/action.yml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'todb-init' +description: 'Initializes a Traffic Ops database using a local PostgreSQL instance' +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/.github/actions/todb-init/dbconf.yml b/.github/actions/todb-init/dbconf.yml new file mode 100644 index 0000000000..f565670b38 --- /dev/null +++ b/.github/actions/todb-init/dbconf.yml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "1.0" +name: dbconf.yml + +test: + driver: postgres + open: host=postgres port=5432 user=traffic_ops password=twelve dbname=traffic_ops sslmode=disable + diff --git a/.github/actions/todb-init/entrypoint.sh b/.github/actions/todb-init/entrypoint.sh new file mode 100644 index 0000000000..a951ce4f6e --- /dev/null +++ b/.github/actions/todb-init/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh -l +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +cd "traffic_ops/app/db" + +mv /dbconf.yml ./ + +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./create_tables.sql >/dev/null +goose --env=test --path="$PWD" up +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./seeds.sql >/dev/null +psql -d postgresql://traffic_ops:twelve@postgres:5432/traffic_ops < ./patches.sql >/dev/null diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 2485491a71..b2dbbf6756 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -50,6 +50,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@master + - name: Initialize Traffic Ops Database + uses: ./.github/actions/todb-init - name: Run API v1 tests uses: ./.github/actions/to-integration-tests with: From 3375e3e2da29c55c9f4154e73eb2b787f0658096 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:57:25 -0600 Subject: [PATCH 50/58] Fix package name --- .github/actions/todb-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/todb-init/Dockerfile b/.github/actions/todb-init/Dockerfile index 15fa80e316..000d554d58 100644 --- a/.github/actions/todb-init/Dockerfile +++ b/.github/actions/todb-init/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2-alpine -RUN apk add --no-cache git gcc gettext-base postgresql-client +RUN apk add --no-cache git gcc gettext postgresql-client RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 5a07100646c0a9fd078277a7869af5cc3edf7503 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 14:58:13 -0600 Subject: [PATCH 51/58] Add missing packages --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 10509ef864..5ba9e988fa 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -17,7 +17,7 @@ FROM golang:1.14.2-alpine -RUN apk add --no-cache gettext +RUN apk add --no-cache gettext git gcc COPY entrypoint.sh database.json traffic-ops-test.json cdn.json / RUN chmod a+x /entrypoint.sh From 97c191601a463e6fcfd8477e74dd7b0fda0799b0 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 15:03:20 -0600 Subject: [PATCH 52/58] Run v2 and v3 only if db initialization succeeds --- .github/workflows/traffic ops.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index b2dbbf6756..1db286ff0c 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -51,18 +51,19 @@ jobs: - name: Checkout uses: actions/checkout@master - name: Initialize Traffic Ops Database + id: todb uses: ./.github/actions/todb-init - name: Run API v1 tests uses: ./.github/actions/to-integration-tests with: version: 1 - name: Run API v2 tests - if: ${{ always() }} + if: ${{ steps.todb.outcome == 'success' }} uses: ./.github/actions/to-integration-tests with: version: 2 - name: Run API v3 tests - if: ${{ always() }} + if: ${{ steps.todb.outcome == 'success' }} uses: ./.github/actions/to-integration-tests with: version: 3 From 20ae173e986de9d05e622d0cf8bd14a6f646d61d Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 15:06:49 -0600 Subject: [PATCH 53/58] Alpine doesn't have stdio, so I'm using buster --- .github/actions/todb-init/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/todb-init/Dockerfile b/.github/actions/todb-init/Dockerfile index 000d554d58..30c17bd7aa 100644 --- a/.github/actions/todb-init/Dockerfile +++ b/.github/actions/todb-init/Dockerfile @@ -15,9 +15,11 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-alpine +FROM golang:1.14.2-buster -RUN apk add --no-cache git gcc gettext postgresql-client +RUN apt-get update &&\ + apt-get install -y --no-install-recommends git gcc gettext-base postgresql-client-11 &&\ + rm -rf /var/lib/apt/lists/* RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq RUN mv $GOPATH/bin/goose /bin/ From 2add9e5ef2e7f54eeb949492146bf529928244eb Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 15:11:23 -0600 Subject: [PATCH 54/58] Alpine doesn't have stdio, so I'm using buster --- .github/actions/to-integration-tests/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 5ba9e988fa..008ab889a0 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,9 +15,11 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-alpine +FROM golang:1.14.2-buster -RUN apk add --no-cache gettext git gcc +RUN apt-get update &&\ + apt-get install -y --no-install-recommends gettext-base git gcc &&\ + rm -rf /var/lib/apt/lists/* COPY entrypoint.sh database.json traffic-ops-test.json cdn.json / RUN chmod a+x /entrypoint.sh From 071ec2478146daf3457a0208860f7e5e9bdc62a4 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 17 Aug 2020 15:11:53 -0600 Subject: [PATCH 55/58] Remove unnecessary 'go get' --- .github/actions/todb-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/todb-init/Dockerfile b/.github/actions/todb-init/Dockerfile index 30c17bd7aa..6c854c0023 100644 --- a/.github/actions/todb-init/Dockerfile +++ b/.github/actions/todb-init/Dockerfile @@ -20,7 +20,7 @@ FROM golang:1.14.2-buster RUN apt-get update &&\ apt-get install -y --no-install-recommends git gcc gettext-base postgresql-client-11 &&\ rm -rf /var/lib/apt/lists/* -RUN go get -v bitbucket.org/liamstask/goose/cmd/goose github.com/lib/pq +RUN go get -v bitbucket.org/liamstask/goose/cmd/goose RUN mv $GOPATH/bin/goose /bin/ COPY entrypoint.sh dbconf.yml / From 98f10b757ccb0bdc5aae317bab42c3a2cf615f44 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 18 Aug 2020 11:04:11 -0600 Subject: [PATCH 56/58] Add always() to later API version test conditionals --- .github/workflows/traffic ops.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/traffic ops.yml b/.github/workflows/traffic ops.yml index 1db286ff0c..0db639fa0d 100644 --- a/.github/workflows/traffic ops.yml +++ b/.github/workflows/traffic ops.yml @@ -58,12 +58,12 @@ jobs: with: version: 1 - name: Run API v2 tests - if: ${{ steps.todb.outcome == 'success' }} + if: ${{ steps.todb.outcome == 'success' && always() }} uses: ./.github/actions/to-integration-tests with: version: 2 - name: Run API v3 tests - if: ${{ steps.todb.outcome == 'success' }} + if: ${{ steps.todb.outcome == 'success' && always() }} uses: ./.github/actions/to-integration-tests with: version: 3 From c72acee44aaecbc2314cee89a95dc60216fd58ac Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 18 Aug 2020 11:09:40 -0600 Subject: [PATCH 57/58] Try to switch back to Alpine --- .github/actions/to-integration-tests/Dockerfile | 7 +++---- .github/actions/todb-init/Dockerfile | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index 008ab889a0..ff908a630e 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -15,11 +15,10 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-buster +FROM golang:1.14.2-alpine -RUN apt-get update &&\ - apt-get install -y --no-install-recommends gettext-base git gcc &&\ - rm -rf /var/lib/apt/lists/* +ENV CGO_ENABLED=0 +RUN apk-add gettext git COPY entrypoint.sh database.json traffic-ops-test.json cdn.json / RUN chmod a+x /entrypoint.sh diff --git a/.github/actions/todb-init/Dockerfile b/.github/actions/todb-init/Dockerfile index 6c854c0023..d5836abc81 100644 --- a/.github/actions/todb-init/Dockerfile +++ b/.github/actions/todb-init/Dockerfile @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. -FROM golang:1.14.2-buster +FROM golang:1.14.2-alpine + +RUN apk add --no-cache git gcc gettext postgresql-client musl-dev &&\ + go get -v bitbucket.org/liamstask/goose/cmd/goose &&\ + mv $GOPATH/bin/goose /bin/ &&\ + apk del git gcc musl-dev -RUN apt-get update &&\ - apt-get install -y --no-install-recommends git gcc gettext-base postgresql-client-11 &&\ - rm -rf /var/lib/apt/lists/* -RUN go get -v bitbucket.org/liamstask/goose/cmd/goose -RUN mv $GOPATH/bin/goose /bin/ COPY entrypoint.sh dbconf.yml / RUN chmod a+x /entrypoint.sh From a3d93f21a981f987afe79f6af6a94527f04348bd Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 18 Aug 2020 11:13:53 -0600 Subject: [PATCH 58/58] Fix apk command --- .github/actions/to-integration-tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/to-integration-tests/Dockerfile b/.github/actions/to-integration-tests/Dockerfile index ff908a630e..3457f8dee9 100644 --- a/.github/actions/to-integration-tests/Dockerfile +++ b/.github/actions/to-integration-tests/Dockerfile @@ -18,7 +18,7 @@ FROM golang:1.14.2-alpine ENV CGO_ENABLED=0 -RUN apk-add gettext git +RUN apk add --no-cache gettext git COPY entrypoint.sh database.json traffic-ops-test.json cdn.json / RUN chmod a+x /entrypoint.sh