From 140ca3e9470758a7fe77e0d2851cc38481b44a8e Mon Sep 17 00:00:00 2001 From: Dan Kirkwood Date: Thu, 1 Dec 2016 20:51:06 +0000 Subject: [PATCH 1/2] add convert service for docker-compose for psql conversion --- .../app/db/pg-migration/Dockerfile-convert | 7 +++++++ .../app/db/pg-migration/Dockerfile-pgloader | 2 +- .../app/db/pg-migration/docker-compose.yml | 19 +++++++++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 traffic_ops/app/db/pg-migration/Dockerfile-convert diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-convert b/traffic_ops/app/db/pg-migration/Dockerfile-convert new file mode 100644 index 0000000000..7f96523dc3 --- /dev/null +++ b/traffic_ops/app/db/pg-migration/Dockerfile-convert @@ -0,0 +1,7 @@ +FROM postgres:9.6 + +MAINTAINER Dan Kirkwood + +ADD convert_bools.sql . + +CMD psql postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB < ./convert_bools.sql diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-pgloader b/traffic_ops/app/db/pg-migration/Dockerfile-pgloader index 5bd4aab1e7..5acc979427 100644 --- a/traffic_ops/app/db/pg-migration/Dockerfile-pgloader +++ b/traffic_ops/app/db/pg-migration/Dockerfile-pgloader @@ -4,5 +4,5 @@ MAINTAINER Dan Kirkwood # install nc to check that postgres and mysql are ready.. RUN apt-get -y install netcat -ADD runpgloader.sh . +ADD pg-migration/runpgloader.sh . CMD ./runpgloader.sh diff --git a/traffic_ops/app/db/pg-migration/docker-compose.yml b/traffic_ops/app/db/pg-migration/docker-compose.yml index c8ac83da63..fc327fa036 100644 --- a/traffic_ops/app/db/pg-migration/docker-compose.yml +++ b/traffic_ops/app/db/pg-migration/docker-compose.yml @@ -1,10 +1,11 @@ version: '2' services: + # dataimport reads data from an existing traffic_ops server running mysql thru the API dataimport: build: - dockerfile: Dockerfile-traffic_ops-client context: . + dockerfile: Dockerfile-traffic_ops-client restart: "no" environment: - TO_USER @@ -14,6 +15,7 @@ services: volumes: - ./mysql/initdb.d:/docker-entrypoint-initdb.d + # mysql_host loads mysql data locally and provides direct access for pgloader mysql_host: image: mysql:5.6 restart: "no" @@ -28,10 +30,11 @@ services: - ./mysql/conf.d:/etc/mysql/conf.d - ./mysql/initdb.d:/docker-entrypoint-initdb.d + # postgres_host provides a postgres data to import data to postgres_host: build: - dockerfile: pg-migration/Dockerfile-postgres context: .. + dockerfile: pg-migration/Dockerfile-postgres restart: "no" environment: - POSTGRES_DB=traffic_ops @@ -42,10 +45,11 @@ services: ports: - 5432 + # pgloader converts the data from mysql to postgres pgloader: build: - dockerfile: Dockerfile-pgloader - context: . + context: .. + dockerfile: pg-migration/Dockerfile-pgloader restart: "no" depends_on: - mysql_host @@ -59,3 +63,10 @@ services: - POSTGRES_DB=traffic_ops - POSTGRES_PASSWORD=twelve - POSTGRES_USER=traffic_ops + + # convert runs necessary post-import conversion(s) in postgres + convert: + build: + context: .. + dockerfile: pg-migration/Dockerfile-convert + restart: "no" From 624f1ba16d99bb0fc3d00dee4e5e363e2e8d4fdd Mon Sep 17 00:00:00 2001 From: Dan Kirkwood Date: Sun, 4 Dec 2016 01:44:03 +0000 Subject: [PATCH 2/2] add another step to convert bools --- .../app/db/pg-migration/Dockerfile-convert | 4 +++- .../app/db/pg-migration/docker-compose.yml | 16 +++++++++++++++- .../mysql/initdb.d/AAA-wait-for-data.sh | 6 ------ traffic_ops/app/db/pg-migration/mysql/run.sh | 8 -------- traffic_ops/app/db/pg-migration/runconvert.sh | 17 +++++++++++++++++ traffic_ops/app/db/pg-migration/runpgloader.sh | 8 ++++++++ 6 files changed, 43 insertions(+), 16 deletions(-) delete mode 100755 traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh delete mode 100644 traffic_ops/app/db/pg-migration/mysql/run.sh create mode 100755 traffic_ops/app/db/pg-migration/runconvert.sh diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-convert b/traffic_ops/app/db/pg-migration/Dockerfile-convert index 7f96523dc3..b1a7dd56d6 100644 --- a/traffic_ops/app/db/pg-migration/Dockerfile-convert +++ b/traffic_ops/app/db/pg-migration/Dockerfile-convert @@ -2,6 +2,8 @@ FROM postgres:9.6 MAINTAINER Dan Kirkwood +RUN apt-get update && apt-get -y install netcat +ADD pg-migration/runconvert.sh . ADD convert_bools.sql . -CMD psql postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB < ./convert_bools.sql +CMD ./runconvert.sh diff --git a/traffic_ops/app/db/pg-migration/docker-compose.yml b/traffic_ops/app/db/pg-migration/docker-compose.yml index fc327fa036..74b5bd8c74 100644 --- a/traffic_ops/app/db/pg-migration/docker-compose.yml +++ b/traffic_ops/app/db/pg-migration/docker-compose.yml @@ -1,5 +1,8 @@ version: '2' +volumes: + sync: + services: # dataimport reads data from an existing traffic_ops server running mysql thru the API dataimport: @@ -41,7 +44,7 @@ services: - POSTGRES_PASSWORD=twelve - POSTGRES_USER=traffic_ops volumes: - - /var/lib/postgresql/data:/var/lib/postgresql/data + - /opt/traffic_ops-db:/var/lib/postgresql/data ports: - 5432 @@ -63,6 +66,8 @@ services: - POSTGRES_DB=traffic_ops - POSTGRES_PASSWORD=twelve - POSTGRES_USER=traffic_ops + volumes: + - sync:/sync # convert runs necessary post-import conversion(s) in postgres convert: @@ -70,3 +75,12 @@ services: context: .. dockerfile: pg-migration/Dockerfile-convert restart: "no" + depends_on: + - postgres_host + environment: + - POSTGRES_HOST=postgres_host + - POSTGRES_DB=traffic_ops + - POSTGRES_PASSWORD=twelve + - POSTGRES_USER=traffic_ops + volumes: + - sync:/sync diff --git a/traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh b/traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh deleted file mode 100755 index 69681abcaf..0000000000 --- a/traffic_ops/app/db/pg-migration/mysql/initdb.d/AAA-wait-for-data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -x - -# wait for traffic_ops.sql file to appear -while [[ ! -f /docker-entrypoint-initdb.d/traffic_ops.sql ]]; do - sleep 1 -done diff --git a/traffic_ops/app/db/pg-migration/mysql/run.sh b/traffic_ops/app/db/pg-migration/mysql/run.sh deleted file mode 100644 index 50c149d78b..0000000000 --- a/traffic_ops/app/db/pg-migration/mysql/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -docker run \ - --name mysql \ - -p 3306:3306 \ - -v $(pwd)/mysql/conf.d:/etc/mysql/conf.d \ - -v $(pwd)/mysql/initdb.d:/docker-entrypoint-initdb.d \ - -d mysql - - diff --git a/traffic_ops/app/db/pg-migration/runconvert.sh b/traffic_ops/app/db/pg-migration/runconvert.sh new file mode 100755 index 0000000000..6b27f97fc8 --- /dev/null +++ b/traffic_ops/app/db/pg-migration/runconvert.sh @@ -0,0 +1,17 @@ +#!/bin/bash -x + +set -x + +waiting=/sync/waiting-for-pgloader +touch $waiting + +# Wait for pgloader to finish +while [[ -f $waiting ]]; do + ls -l $waiting + sleep 3 +done + +echo "Looks like pgloader is finished.. Converting.." + +# Load required conversion of booleans +psql postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB < ./convert_bools.sql diff --git a/traffic_ops/app/db/pg-migration/runpgloader.sh b/traffic_ops/app/db/pg-migration/runpgloader.sh index 9f698f373b..cbbf8e345a 100755 --- a/traffic_ops/app/db/pg-migration/runpgloader.sh +++ b/traffic_ops/app/db/pg-migration/runpgloader.sh @@ -16,3 +16,11 @@ pgloader -v \ --cast 'type double to numeric drop typemod' \ mysql://$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST/traffic_ops_db \ postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB + +waiting=/sync/waiting-for-pgloader +while [[ ! -f $waiting ]]; do + echo "Nothing waiting for pgloader?" + sleep 3 +done + +rm $waiting