From 51cecfc60496edd976af8797f130c2e274d9f9a6 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Thu, 8 Mar 2018 13:54:06 -0500 Subject: [PATCH 1/2] Add Postgresql statefulsets with master/slave replication on OpenShift. Add Postgresql statefulsets with master/slave replication on OpoenShift. Simple test. Non functional. Adding missing semicolon. Glassfish starts but no dataverse. Glassfish can connect to postgres but does not save in application. Fixing service problem with postgres stateful set. going to try to break command over multiple lines. WIP extending image with S2I Replication is working. Just needs to be cleaned up. Cleaned up. Removing print tests. Resetting install to upstream. Resetting build.sh to upstream. Fixing image pull policy. Fixing default config. Fixing formatting issues. More formatting. --- .../docker/dataverse-glassfish/default.config | 2 +- conf/openshift/openshift.json | 49 ++++++++++++++++--- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/conf/docker/dataverse-glassfish/default.config b/conf/docker/dataverse-glassfish/default.config index c9a7279742d..7af10336f30 100644 --- a/conf/docker/dataverse-glassfish/default.config +++ b/conf/docker/dataverse-glassfish/default.config @@ -3,7 +3,7 @@ GLASSFISH_DIRECTORY /usr/local/glassfish4 ADMIN_EMAIL MAIL_SERVER mail.hmdc.harvard.edu POSTGRES_ADMIN_PASSWORD secret -POSTGRES_SERVER dataverse-postgresql-service +POSTGRES_SERVER dataverse-postgresql-0.dataverse-postgresql-service POSTGRES_PORT 5432 POSTGRES_DATABASE dvndb POSTGRES_USER dvnapp diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index 330fc8914ae..bf74631cfb1 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -42,6 +42,7 @@ "selector": { "name": "iqss-dataverse-postgresql" }, + "clusterIP": "None", "ports": [ { "name": "database", @@ -150,6 +151,10 @@ } ], "env": [ + { + "name": "POSTGRES_SERVER", + "value": "dataverse-postgresql-0" + }, { "name": "POSTGRES_SERVICE_HOST", "value": "dataverse-postgresql-service" @@ -222,8 +227,8 @@ } }, { - "kind": "DeploymentConfig", - "apiVersion": "v1", + "kind": "StatefulSet", + "apiVersion": "apps/v1beta1", "metadata": { "name": "dataverse-postgresql", "annotations": { @@ -231,6 +236,8 @@ } }, "spec": { + "serviceName" : "dataverse-postgresql-service", + "replicas" : 1, "template": { "metadata": { "labels": { @@ -241,7 +248,10 @@ "containers": [ { "name": "centos-postgresql-94-centos7", - "image": "centos-postgresql-94-centos7", + "image": "centos/postgresql-94-centos7", + "command": [ + "sh", "-c", "echo 'Setting up Postgres Master/Slave replication...'; [[ `hostname` =~ -([0-9]+)$ ]] || exit 1; ordinal=${BASH_REMATCH[1]}; if [[ $ordinal -eq 0 ]]; then run-postgresql-master; else run-postgresql-slave; fi;" + ], "ports": [ { "containerPort": 5432, @@ -253,10 +263,34 @@ "name": "POSTGRESQL_USER", "value": "dvnapp" }, + { + "name": "POSTGRESQL_MASTER_USER", + "value": "master" + }, { "name": "POSTGRESQL_PASSWORD", "value": "secret" }, + { + "name": "POSTGRESQL_MASTER_PASSWORD", + "value": "master" + }, + { + "name": "POSTGRESQL_MASTER_SERVICE_NAME", + "value": "dataverse-postgresql-service" + }, + { + "name": "POSTGRESQL_MASTER_IP", + "value": "dataverse-postgresql-0.dataverse-postgresql-service" + }, + { + "name": "postgresql_master_addr", + "value": "dataverse-postgresql-0.dataverse-postgresql-service" + }, + { + "name": "master_fqdn", + "value": "dataverse-postgresql-0.dataverse-postgresql-service" + }, { "name": "POSTGRESQL_DATABASE", "value": "dvndb" @@ -281,6 +315,7 @@ ] } }, + "strategy": { "type": "Rolling", "rollingParams": { @@ -300,7 +335,7 @@ ], "from": { "kind": "ImageStreamTag", - "name": "centos-postgresql-94-centos7:latest" + "name": "centos/postgresql-94-centos7:latest" } } }, @@ -308,9 +343,11 @@ "type": "ConfigChange" } ], - "replicas": 1, "selector": { - "name": "iqss-dataverse-postgresql" + "name": "iqss-dataverse-postgresql", + "matchLabels" : { + "name" : "iqss-dataverse-postgresql" + } } } }, From e7a56c7ee07ef3ff931c18927a2f737eeae33dba Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Apr 2018 18:16:22 -0400 Subject: [PATCH 2/2] add `huborg ` for non-iqss Docker Hub #4599 --- conf/docker/build.sh | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 95ff41bab66..9ed27c06306 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -1,10 +1,15 @@ #!/bin/sh # Creates images and pushes them to Docker Hub. -# The "latest" tag should be relatively stable. Don't push breaking changes there. +# The "latest" tag under "iqss" should be relatively stable. Don't push breaking changes there. # None of the tags are suitable for production use. See https://github.com/IQSS/dataverse/issues/4040 -# Push to custom tags or tags based on branch names to iterate on the images. +# To interate on images, push to custom tags or tags based on branch names or a non-iqss Docker Hub org/username. +# Docker Hub organization or username +HUBORG=iqss +# The most stable tag we have. +STABLE=latest +#FIXME: Use a real flag/argument parser. download-files.sh uses "getopts" for example. if [ -z "$1" ]; then - echo "No argument supplied. Please specify \"branch\" or \"custom my-custom-tag\" for experiments or \"stable\" if your change won't break anything." + echo "No argument supplied. For experiments, specify \"branch\" or \"custom my-custom-tag\" or \"huborg \". Specify \"stable\" to push to the \"$STABLE\" tag under \"$HUBORG\" if your change won't break anything." exit 1 fi @@ -14,23 +19,32 @@ if [ "$1" == 'branch' ]; then TAG=$GIT_BRANCH elif [ "$1" == 'stable' ]; then echo "We'll push a tag to the most stable tag (which isn't saying much!)." - TAG=kick-the-tires + TAG=$STABLE elif [ "$1" == 'custom' ]; then - if [ -z "$1" ]; then - echo "You must provide a custom tag as the second argument." + if [ -z "$2" ]; then + echo "You must provide a custom tag as the second argument. Something other than \"$STABLE\"." exit 1 else echo "We'll push a custom tag." TAG=$2 fi +elif [ "$1" == 'huborg' ]; then + if [ -z "$2" ]; then + echo "You must provide your Docker Hub organization or username as the second argument. \"$USER\" or whatever." + exit 1 + else + HUBORG=$2 + TAG=$STABLE + echo "We'll push to the Docker Hub organization or username you specified: $HUBORG." + fi else echo "Unexpected argument: $1. Exiting. Run with no arguments for help." exit 1 fi -echo Images will be pushed to Docker Hub with the tag \"$TAG\". +echo Images will be pushed to Docker Hub org/username \"$HUBORG\" with the tag \"$TAG\". # Use "conf" directory as context so we can copy schema.xml into Solr image. -docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf -docker push iqss/dataverse-solr:$TAG +docker build -t $HUBORG/dataverse-solr:$TAG -f solr/Dockerfile ../../conf +docker push $HUBORG/dataverse-solr:$TAG # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. cd ../.. mvn clean @@ -58,6 +72,6 @@ if [[ "$?" -ne 0 ]]; then fi # We'll assume at this point that the download script has been run. cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish -docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish +docker build -t $HUBORG/dataverse-glassfish:$TAG dataverse-glassfish # FIXME: Check the output of `docker build` and only push on success. -docker push iqss/dataverse-glassfish:$TAG +docker push $HUBORG/dataverse-glassfish:$TAG