diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-convert b/traffic_ops/app/db/pg-migration/Dockerfile-convert new file mode 100644 index 0000000000..b1a7dd56d6 --- /dev/null +++ b/traffic_ops/app/db/pg-migration/Dockerfile-convert @@ -0,0 +1,9 @@ +FROM postgres:9.6 + +MAINTAINER Dan Kirkwood + +RUN apt-get update && apt-get -y install netcat +ADD pg-migration/runconvert.sh . +ADD convert_bools.sql . + +CMD ./runconvert.sh diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-pgloader b/traffic_ops/app/db/pg-migration/Dockerfile-pgloader index 5bd4aab1e7..5acc979427 100644 --- a/traffic_ops/app/db/pg-migration/Dockerfile-pgloader +++ b/traffic_ops/app/db/pg-migration/Dockerfile-pgloader @@ -4,5 +4,5 @@ MAINTAINER Dan Kirkwood # install nc to check that postgres and mysql are ready.. RUN apt-get -y install netcat -ADD runpgloader.sh . +ADD pg-migration/runpgloader.sh . CMD ./runpgloader.sh diff --git a/traffic_ops/app/db/pg-migration/docker-compose.yml b/traffic_ops/app/db/pg-migration/docker-compose.yml index c8ac83da63..74b5bd8c74 100644 --- a/traffic_ops/app/db/pg-migration/docker-compose.yml +++ b/traffic_ops/app/db/pg-migration/docker-compose.yml @@ -1,10 +1,14 @@ version: '2' +volumes: + sync: + services: + # dataimport reads data from an existing traffic_ops server running mysql thru the API dataimport: build: - dockerfile: Dockerfile-traffic_ops-client context: . + dockerfile: Dockerfile-traffic_ops-client restart: "no" environment: - TO_USER @@ -14,6 +18,7 @@ services: volumes: - ./mysql/initdb.d:/docker-entrypoint-initdb.d + # mysql_host loads mysql data locally and provides direct access for pgloader mysql_host: image: mysql:5.6 restart: "no" @@ -28,24 +33,26 @@ services: - ./mysql/conf.d:/etc/mysql/conf.d - ./mysql/initdb.d:/docker-entrypoint-initdb.d + # postgres_host provides a postgres data to import data to postgres_host: build: - dockerfile: pg-migration/Dockerfile-postgres context: .. + dockerfile: pg-migration/Dockerfile-postgres restart: "no" environment: - POSTGRES_DB=traffic_ops - POSTGRES_PASSWORD=twelve - POSTGRES_USER=traffic_ops volumes: - - /var/lib/postgresql/data:/var/lib/postgresql/data + - /opt/traffic_ops-db:/var/lib/postgresql/data ports: - 5432 + # pgloader converts the data from mysql to postgres pgloader: build: - dockerfile: Dockerfile-pgloader - context: . + context: .. + dockerfile: pg-migration/Dockerfile-pgloader restart: "no" depends_on: - mysql_host @@ -59,3 +66,21 @@ services: - POSTGRES_DB=traffic_ops - POSTGRES_PASSWORD=twelve - POSTGRES_USER=traffic_ops + volumes: + - sync:/sync + + # convert runs necessary post-import conversion(s) in postgres + convert: + build: + context: .. + dockerfile: pg-migration/Dockerfile-convert + restart: "no" + depends_on: + - postgres_host + environment: + - POSTGRES_HOST=postgres_host + - POSTGRES_DB=traffic_ops + - POSTGRES_PASSWORD=twelve + - POSTGRES_USER=traffic_ops + volumes: + - sync:/sync diff --git a/traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh b/traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh deleted file mode 100755 index 69681abcaf..0000000000 --- a/traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -x - -# wait for traffic_ops.sql file to appear -while [[ ! -f /docker-entrypoint-initdb.d/traffic_ops.sql ]]; do - sleep 1 -done diff --git a/traffic_ops/app/db/pg-migration/mysql/run.sh b/traffic_ops/app/db/pg-migration/mysql/run.sh deleted file mode 100644 index 50c149d78b..0000000000 --- a/traffic_ops/app/db/pg-migration/mysql/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -docker run \ - --name mysql \ - -p 3306:3306 \ - -v $(pwd)/mysql/conf.d:/etc/mysql/conf.d \ - -v $(pwd)/mysql/initdb.d:/docker-entrypoint-initdb.d \ - -d mysql - - diff --git a/traffic_ops/app/db/pg-migration/runconvert.sh b/traffic_ops/app/db/pg-migration/runconvert.sh new file mode 100755 index 0000000000..6b27f97fc8 --- /dev/null +++ b/traffic_ops/app/db/pg-migration/runconvert.sh @@ -0,0 +1,17 @@ +#!/bin/bash -x + +set -x + +waiting=/sync/waiting-for-pgloader +touch $waiting + +# Wait for pgloader to finish +while [[ -f $waiting ]]; do + ls -l $waiting + sleep 3 +done + +echo "Looks like pgloader is finished.. Converting.." + +# Load required conversion of booleans +psql postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB < ./convert_bools.sql diff --git a/traffic_ops/app/db/pg-migration/runpgloader.sh b/traffic_ops/app/db/pg-migration/runpgloader.sh index 9f698f373b..cbbf8e345a 100755 --- a/traffic_ops/app/db/pg-migration/runpgloader.sh +++ b/traffic_ops/app/db/pg-migration/runpgloader.sh @@ -16,3 +16,11 @@ pgloader -v \ --cast 'type double to numeric drop typemod' \ mysql://$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST/traffic_ops_db \ postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB + +waiting=/sync/waiting-for-pgloader +while [[ ! -f $waiting ]]; do + echo "Nothing waiting for pgloader?" + sleep 3 +done + +rm $waiting