diff --git a/.github/actions/run-ciab/run-ciab.sh b/.github/actions/run-ciab/run-ciab.sh index ba7f1d07b2..80c0e6631c 100755 --- a/.github/actions/run-ciab/run-ciab.sh +++ b/.github/actions/run-ciab/run-ciab.sh @@ -28,7 +28,7 @@ store_ciab_logs() { cd infrastructure/cdn-in-a-box; logged_services='trafficrouter readiness'; -other_services='dns edge enroller mid-01 mid-02 origin static trafficmonitor trafficops trafficstats trafficvault'; +other_services='dns edge enroller mid-01 mid-02 origin static trafficmonitor trafficops trafficstats'; docker_compose='docker-compose -f ./docker-compose.yml -f ./docker-compose.readiness.yml'; $docker_compose up -d $logged_services $other_services; $docker_compose logs -f $logged_services & diff --git a/.github/workflows/tp.integration.tests.yml b/.github/workflows/tp.integration.tests.yml index 4958a180a8..03caaaf282 100644 --- a/.github/workflows/tp.integration.tests.yml +++ b/.github/workflows/tp.integration.tests.yml @@ -29,7 +29,7 @@ on: - .github/actions/tp-integration-tests/** - .github/workflows/tp.integration.tests.yml - GO_VERSION - - infrastructure/cdn-in-a-box/traffic_vault/** + - infrastructure/cdn-in-a-box/optional/traffic_vault/** - traffic_ops/*client/**.go - traffic_ops/testing/api/**.go - traffic_ops/traffic_ops_golang/**.go @@ -42,7 +42,7 @@ on: - .github/actions/tp-integration-tests/** - .github/workflows/tp.integration.tests.yml - GO_VERSION - - infrastructure/cdn-in-a-box/traffic_vault/** + - infrastructure/cdn-in-a-box/optional/traffic_vault/** - traffic_ops/*client/**.go - traffic_ops/testing/api/**.go - traffic_ops/traffic_ops_golang/**.go diff --git a/.github/workflows/traffic-ops.yml b/.github/workflows/traffic-ops.yml index 6114aebc41..43cb6f5a62 100644 --- a/.github/workflows/traffic-ops.yml +++ b/.github/workflows/traffic-ops.yml @@ -31,7 +31,7 @@ on: - go.mod - go.sum - GO_VERSION - - infrastructure/cdn-in-a-box/traffic_vault/** + - infrastructure/cdn-in-a-box/optional/traffic_vault/** - traffic_ops/*client/**.go - traffic_ops/testing/api/**.go - traffic_ops/traffic_ops_golang/**.go @@ -47,7 +47,7 @@ on: - go.mod - go.sum - GO_VERSION - - infrastructure/cdn-in-a-box/traffic_vault/** + - infrastructure/cdn-in-a-box/optional/traffic_vault/** - traffic_ops/*client/**.go - traffic_ops/testing/api/**.go - traffic_ops/traffic_ops_golang/**.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c11cc526fe..978af74f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed - Updated `t3c` to request less unnecessary deliveryservice-server assignment and invalidation jobs data via new query params supported by Traffic Ops - [#6179](https://github.com/apache/trafficcontrol/issues/6179) Updated the Traffic Ops rpm to include the `ToDnssecRefresh` binary and make the `trafops_dnssec_refresh` cron job use it +- [#5927](https://github.com/apache/trafficcontrol/issues/5927) Updated CDN-in-a-Box to not run a Riak container by default but instead only run it if the optional flag is provided. ### Deprecated diff --git a/infrastructure/cdn-in-a-box/.gitignore b/infrastructure/cdn-in-a-box/.gitignore index b903ca7f1e..61a7c54a75 100644 --- a/infrastructure/cdn-in-a-box/.gitignore +++ b/infrastructure/cdn-in-a-box/.gitignore @@ -19,4 +19,4 @@ /cache/ATS_VERSION /optional/vpn/vpnca /traffic_ops/ca -/traffic_vault/prestart.d/00-0-standalone-config.sh +/optional/traffic_vault/prestart.d/00-0-standalone-config.sh diff --git a/infrastructure/cdn-in-a-box/README.md b/infrastructure/cdn-in-a-box/README.md index 9797c1b615..1f5a9ad803 100644 --- a/infrastructure/cdn-in-a-box/README.md +++ b/infrastructure/cdn-in-a-box/README.md @@ -159,13 +159,6 @@ show you the default UI for interacting with the CDN - Traffic Portal. > Web interfaces on ports 3080 (HTTP) and 3443 (HTTPS), with a DNS service on 53 and an API on 3333 > N/A > N/A -> -> -> Traffic Vault -> Riak key-value store on port 8010 -> TV_ADMIN_USER in variables.env -> TV_ADMIN_PASSWORD in variables.env -> > > > diff --git a/infrastructure/cdn-in-a-box/docker-compose.expose-ports.yml b/infrastructure/cdn-in-a-box/docker-compose.expose-ports.yml index b45b52de85..b827ff5f9c 100644 --- a/infrastructure/cdn-in-a-box/docker-compose.expose-ports.yml +++ b/infrastructure/cdn-in-a-box/docker-compose.expose-ports.yml @@ -47,11 +47,6 @@ services: influxdb: ports: - "8086:8086" - trafficvault: - ports: - - "8087:8087" - - "8088:8088" - - "8098:8098" edge: ports: - "9000:80" diff --git a/infrastructure/cdn-in-a-box/docker-compose.yml b/infrastructure/cdn-in-a-box/docker-compose.yml index 6533fefc08..1bdb3c5470 100644 --- a/infrastructure/cdn-in-a-box/docker-compose.yml +++ b/infrastructure/cdn-in-a-box/docker-compose.yml @@ -163,25 +163,6 @@ services: volumes: - shared:/shared - # trafficvault runs a riak container to store private keys - trafficvault: - build: - context: . - dockerfile: traffic_vault/Dockerfile - depends_on: - - enroller - environment: - - CLUSTER_NAME=trafficvault - labels: - - "com.basho.riak.cluster.name=trafficvault" - volumes: - - schemas:/etc/riak/schemas - - shared:/shared - domainname: infra.ciab.test - env_file: - - variables.env - hostname: trafficvault - # edge cache edge: build: diff --git a/infrastructure/cdn-in-a-box/optional/docker-compose.traffic-vault.expose-ports.yml b/infrastructure/cdn-in-a-box/optional/docker-compose.traffic-vault.expose-ports.yml new file mode 100644 index 0000000000..aaa8fb196d --- /dev/null +++ b/infrastructure/cdn-in-a-box/optional/docker-compose.traffic-vault.expose-ports.yml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Expose the trafficvault container on the host on ports 8087, 8088, and 8098 + +--- +version: '2.1' + +services: + trafficvault: + ports: + - "8087:8087" + - "8088:8088" + - "8098:8098" diff --git a/infrastructure/cdn-in-a-box/optional/docker-compose.traffic-vault.yml b/infrastructure/cdn-in-a-box/optional/docker-compose.traffic-vault.yml new file mode 100644 index 0000000000..c6ad2efc6a --- /dev/null +++ b/infrastructure/cdn-in-a-box/optional/docker-compose.traffic-vault.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# This compose file will runs a riak container to store private keys. +# +# Below is an example of how to run the main compose with this file: +# +# docker-compose -f docker-compose.yml -f optional/docker-compose.traffic-vault.yml up +# + +--- +version: '2.1' + +services: + trafficvault: + build: + context: "" + dockerfile: optional/traffic_vault/Dockerfile + environment: + - CLUSTER_NAME=trafficvault + labels: + - "com.basho.riak.cluster.name=trafficvault" + volumes: + - schemas:/etc/riak/schemas + - shared:/shared + domainname: infra.ciab.test + env_file: + - variables.env + hostname: trafficvault + +volumes: + schemas: + external: false + shared: + external: false diff --git a/infrastructure/cdn-in-a-box/traffic_vault/Dockerfile b/infrastructure/cdn-in-a-box/optional/traffic_vault/Dockerfile similarity index 87% rename from infrastructure/cdn-in-a-box/traffic_vault/Dockerfile rename to infrastructure/cdn-in-a-box/optional/traffic_vault/Dockerfile index ed08908b51..7583e75a48 100644 --- a/infrastructure/cdn-in-a-box/traffic_vault/Dockerfile +++ b/infrastructure/cdn-in-a-box/optional/traffic_vault/Dockerfile @@ -27,11 +27,11 @@ RUN echo 'APT::Install-Recommends 0;' >> /etc/apt/apt.conf.d/01norecommends \ && DEBIAN_FRONTEND=noninteractive apt-get install -y net-tools ca-certificates dnsutils gettext-base \ && rm -rf /var/lib/apt/lists/* && rm -rf /etc/apt/apt.conf.d/docker-gzip-indexes -ADD traffic_vault/prestart.d/* /etc/riak/prestart.d/ -ADD traffic_vault/poststart.d/* /etc/riak/poststart.d/ +ADD optional/traffic_vault/prestart.d/* /etc/riak/prestart.d/ +ADD optional/traffic_vault/poststart.d/* /etc/riak/poststart.d/ ADD enroller/server_template.json \ - traffic_vault/run.sh \ - traffic_vault/sslkeys.xml \ + optional/traffic_vault/run.sh \ + optional/traffic_vault/sslkeys.xml \ traffic_ops/to-access.sh \ / diff --git a/infrastructure/cdn-in-a-box/traffic_vault/poststart.d/00-enable-security.sh b/infrastructure/cdn-in-a-box/optional/traffic_vault/poststart.d/00-enable-security.sh similarity index 100% rename from infrastructure/cdn-in-a-box/traffic_vault/poststart.d/00-enable-security.sh rename to infrastructure/cdn-in-a-box/optional/traffic_vault/poststart.d/00-enable-security.sh diff --git a/infrastructure/cdn-in-a-box/traffic_vault/poststart.d/01-add-search-group.sh b/infrastructure/cdn-in-a-box/optional/traffic_vault/poststart.d/01-add-search-group.sh similarity index 100% rename from infrastructure/cdn-in-a-box/traffic_vault/poststart.d/01-add-search-group.sh rename to infrastructure/cdn-in-a-box/optional/traffic_vault/poststart.d/01-add-search-group.sh diff --git a/infrastructure/cdn-in-a-box/traffic_vault/poststart.d/02-add-search-schema.sh b/infrastructure/cdn-in-a-box/optional/traffic_vault/poststart.d/02-add-search-schema.sh similarity index 100% rename from infrastructure/cdn-in-a-box/traffic_vault/poststart.d/02-add-search-schema.sh rename to infrastructure/cdn-in-a-box/optional/traffic_vault/poststart.d/02-add-search-schema.sh diff --git a/infrastructure/cdn-in-a-box/traffic_vault/prestart.d/00-config.sh b/infrastructure/cdn-in-a-box/optional/traffic_vault/prestart.d/00-config.sh similarity index 100% rename from infrastructure/cdn-in-a-box/traffic_vault/prestart.d/00-config.sh rename to infrastructure/cdn-in-a-box/optional/traffic_vault/prestart.d/00-config.sh diff --git a/infrastructure/cdn-in-a-box/traffic_vault/run.sh b/infrastructure/cdn-in-a-box/optional/traffic_vault/run.sh similarity index 100% rename from infrastructure/cdn-in-a-box/traffic_vault/run.sh rename to infrastructure/cdn-in-a-box/optional/traffic_vault/run.sh diff --git a/infrastructure/cdn-in-a-box/traffic_vault/sslkeys.xml b/infrastructure/cdn-in-a-box/optional/traffic_vault/sslkeys.xml similarity index 100% rename from infrastructure/cdn-in-a-box/traffic_vault/sslkeys.xml rename to infrastructure/cdn-in-a-box/optional/traffic_vault/sslkeys.xml diff --git a/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh b/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh index c70e3ca40a..0eab594fc0 100755 --- a/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh +++ b/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh @@ -90,6 +90,11 @@ CREDS fi } +tv-ping() { + to-auth && \ + curl $CURLAUTH $CURLOPTS --cookie "$COOKIEJAR" -X GET "$TO_URL/api/$TO_API_VERSION/vault/ping" +} + to-ping() { # ping endpoint does not require authentication curl $CURLAUTH $CURLOPTS -X GET "$TO_URL/api/$TO_API_VERSION/ping" diff --git a/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh b/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh index 7ab3b35b0b..851be011c5 100755 --- a/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh +++ b/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh @@ -108,6 +108,14 @@ load_data_from() { touch "$ENROLLER_DIR/initial-load-done" sync fi + if [[ "$d" = 'deliveryservices' ]]; then + # Traffic Vault must be accepting connections before enroller can start + until tv-ping; do + echo "Waiting for Traffic Vault to accept connections" + sleep 5 + done + fi + [[ -d $d ]] || continue for f in $(find "$d" -name "*.json" -type f); do echo "Loading $f" diff --git a/infrastructure/cdn-in-a-box/variables.env b/infrastructure/cdn-in-a-box/variables.env index 5d241f485f..eda91da214 100644 --- a/infrastructure/cdn-in-a-box/variables.env +++ b/infrastructure/cdn-in-a-box/variables.env @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. TV_AES_KEY_LOCATION=/opt/traffic_ops/app/conf/aes.key -# Unset TV_BACKEND to use riak as the traffic_vault backend +# Unset TV_BACKEND to use riak as the traffic_vault backend and run the traffic_vault image from the optional directory TV_BACKEND=postgres TLD_DOMAIN=ciab.test INFRA_SUBDOMAIN=infra @@ -131,6 +131,6 @@ TV_HTTP_PORT=8098 TV_HTTPS_PORT=8088 ENROLLER_DIR=/shared/enroller AUTO_SNAPQUEUE_ENABLED=true -AUTO_SNAPQUEUE_SERVERS=trafficops,trafficmonitor,trafficrouter,trafficvault,edge,mid-01,mid-02 +AUTO_SNAPQUEUE_SERVERS=trafficops,trafficmonitor,trafficrouter,edge,mid-01,mid-02 AUTO_SNAPQUEUE_POLL_INTERVAL=2 AUTO_SNAPQUEUE_ACTION_WAIT=2