From d28777268f993b6e628666a48428398f1e78d8c1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 14 Sep 2017 16:00:48 -0400 Subject: [PATCH 01/25] stub out Docker and OpenShift files #4040 --- Dockerfile | 1 + conf/docker/build.sh | 2 + conf/docker/dataverse-glassfish/Dockerfile | 5 +++ conf/docker/postgresql/Dockerfile | 3 ++ conf/docker/solr/Dockerfile | 8 ++++ conf/openshift/openshift.json | 40 +++++++++++++++++++ .../source/installation/index.rst | 1 + .../source/installation/openshift.rst | 24 +++++++++++ .../source/installation/prep.rst | 5 +++ 9 files changed, 89 insertions(+) create mode 100644 Dockerfile create mode 100644 conf/docker/build.sh create mode 100644 conf/docker/dataverse-glassfish/Dockerfile create mode 100644 conf/docker/postgresql/Dockerfile create mode 100644 conf/docker/solr/Dockerfile create mode 100644 conf/openshift/openshift.json create mode 100644 doc/sphinx-guides/source/installation/openshift.rst diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..5f492ea0594 --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +# See `conf/docker` for Docker images diff --git a/conf/docker/build.sh b/conf/docker/build.sh new file mode 100644 index 00000000000..95a3a15979a --- /dev/null +++ b/conf/docker/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "TODO: Build docker images and push to DockerHub." diff --git a/conf/docker/dataverse-glassfish/Dockerfile b/conf/docker/dataverse-glassfish/Dockerfile new file mode 100644 index 00000000000..603660b5264 --- /dev/null +++ b/conf/docker/dataverse-glassfish/Dockerfile @@ -0,0 +1,5 @@ +# Glassfish and Dataverse in one Docker image, as suggested by danmcp. +# +# TODO: Start with the Dataverse 4.2.3 version at https://github.com/pdurbin/ndslabs-dataverse/blob/v4.2.3.1/dockerfiles/dataverse/Dockerfile +# +# TODO: Once we get Dataverse 4.2.3 working, we can try Dataverse 4.7 from https://github.com/pdurbin/ndslabs-dataverse/blob/upgrade-4.7/dockerfiles/dataverse/Dockerfile diff --git a/conf/docker/postgresql/Dockerfile b/conf/docker/postgresql/Dockerfile new file mode 100644 index 00000000000..81ecf0fdeb8 --- /dev/null +++ b/conf/docker/postgresql/Dockerfile @@ -0,0 +1,3 @@ +# PostgreSQL for Dataverse (but consider switching to the image from CentOS) +# +# See also conf/docker/dataverse-glassfish/Dockerfile diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile new file mode 100644 index 00000000000..7d4bf9292e2 --- /dev/null +++ b/conf/docker/solr/Dockerfile @@ -0,0 +1,8 @@ +# Solr for Dataverse +# +# See the note about Dataverse 4.2.3 vs. 4.7 at conf/docker/dataverse-glassfish/Dockerfile +# +# Note that there is a separate Dockerfile we can start with: +# +# Dataverse 4.2.3: https://github.com/pdurbin/ndslabs-dataverse/blob/v4.2.3.1/dockerfiles/solr/Dockerfile +# Dataverse 4.7: https://github.com/pdurbin/ndslabs-dataverse/blob/upgrade-4.7/dockerfiles/solr/Dockerfile diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json new file mode 100644 index 00000000000..e1f92c0fbdb --- /dev/null +++ b/conf/openshift/openshift.json @@ -0,0 +1,40 @@ +{ + "kind": "Project", + "apiVersion": "v1", + "metadata": { + "name": "dataverse", + "creationTimestamp": null, + "labels": { + "name": "dataverse" + }, + "annotations": { + "openshift.io/description": "Dataverse is open source research data repository software: https://dataverse.org", + "openshift.io/display-name": "Dataverse" + } + }, + "spec": { + "containers": [ + { + "name": "dataverse-glassfish", + "image": "ndslabs/dataverse", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ] + }, + { + "name": "dataverse-solr", + "image": "ndslabs/dataverse-solr", + "ports": [ + { + "containerPort": 8983, + "protocol": "TCP" + } + ] + } + ] + }, + "status": {} +} diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index b8423e77ae5..185fb94b4e3 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -20,3 +20,4 @@ Installation Guide geoconnect shibboleth oauth2 + openshift diff --git a/doc/sphinx-guides/source/installation/openshift.rst b/doc/sphinx-guides/source/installation/openshift.rst new file mode 100644 index 00000000000..71758cd23f9 --- /dev/null +++ b/doc/sphinx-guides/source/installation/openshift.rst @@ -0,0 +1,24 @@ +================================= +Installing Dataverse on OpenShift +================================= + +.. contents:: |toctitle| + :local: + +Familiarize Yourself with OpenShift +----------------------------------- + +https://docs.openshift.com/online/getting_started/index.html + +Create OpenShift Account +------------------------ + +Create Project +-------------- + +Import YAML/JSON +---------------- + +Upload ``conf/openshift/openshift.json``. + +FIXME: Getting "cannot create projects at the cluster scope" diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index 9662b5c40b6..ba8bf08cd4a 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -14,6 +14,11 @@ We'll try to get you up and running as quickly as possible, but we thought you m Choose Your Own Installation Adventure -------------------------------------- +OpenShift (for Testing Only) +++++++++++++++++++++++++++++ + +If you would like to kick the tires on your own installation of Dataverse, you can install it for free on OpenShift Online, a platform as a service (PaaS) offering from Red Hat. For details, please see the :doc:`openshift` section. + Vagrant (for Testing Only) ++++++++++++++++++++++++++ From 77b3f67376f6ba340f49a4dcd515bf38c438dfbc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 15 Sep 2017 14:24:25 -0400 Subject: [PATCH 02/25] switch from Project to Template, add dev docs #4040 --- conf/openshift/openshift.json | 39 ++++++----- .../source/developers/dev-environment.rst | 66 +++++++++++++++++++ 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index e1f92c0fbdb..ea09a272347 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -1,9 +1,8 @@ { - "kind": "Project", + "kind": "Template", "apiVersion": "v1", "metadata": { "name": "dataverse", - "creationTimestamp": null, "labels": { "name": "dataverse" }, @@ -12,29 +11,29 @@ "openshift.io/display-name": "Dataverse" } }, - "spec": { - "containers": [ - { - "name": "dataverse-glassfish", - "image": "ndslabs/dataverse", + "objects": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "dataverse-glassfish-service" + }, + "spec": { "ports": [ { - "containerPort": 8080, - "protocol": "TCP" + "name": "web", + "protocol": "TCP", + "port": 8080, + "targetPort": 8080 } - ] - }, - { - "name": "dataverse-solr", - "image": "ndslabs/dataverse-solr", - "ports": [ + ], + "containers": [ { - "containerPort": 8983, - "protocol": "TCP" + "name": "dataverse-glassfish", + "image": "ndslabs/dataverse" } ] } - ] - }, - "status": {} + } + ] } diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 5eda6b1e308..798604cc621 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -328,6 +328,72 @@ If you've reconfigured from EZID to DataCite and are seeing ``Response code: 400 ``asadmin create-jvm-options '-Ddataverse.siteUrl=http\://demo.dataverse.org'`` +OpenShift +--------- + +From the Dataverse perspective, we are in the business of providing a "template" for OpenShift that describes how the various components we build our application on (Glassfish, PostgreSQL, Solr, the Dataverse war file itself, etc.) work together. We also plan to publish Docker images to DockerHub but for now we are trying to use the Dataverse images published to https://hub.docker.com/r/ndslabs/ + +The OpenShift template for Dataverse can be found at ``conf/openshift/openshift.json`` and if you need to hack on the template or related files under ``conf/docker`` it is recommended that you iterate on them using Minishift. + +Install Minishift +~~~~~~~~~~~~~~~~~ + +Minishift requires a hypervisor and since we already use VirtualBox for Vagrant, you should install VirtualBox from http://virtualbox.org . + +Download the Minishift tarball from https://docs.openshift.org/latest/minishift/getting-started/installing.html and put the ``minishift`` binary in ``/usr/local/bin`` or somewhere in your ``$PATH``. This assumes Mac or Linux. + +At this point, you might want to consider going through the Minishift quickstart to get oriented: https://docs.openshift.org/latest/minishift/getting-started/quickstart.html + +Start Minishift +~~~~~~~~~~~~~~~ + +``minishift start --vm-driver=virtualbox`` + +Make the oc Command Executable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``eval $(minishift oc-env)`` + +Create a Minishift Project +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``oc new-project project1`` + +Create a Dataverse App within the Minishift Project +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``oc new-app conf/openshift/openshift.json`` + +Make the Dataverse App Available to Your Browser +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``oc expose svc/dataverse-glassfish-service`` + +Log into Minishift and Visit Dataverse in your Browser +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +FIXME: This doesn't work yet. Rather than Dataverse running, for now you are expected to see "Application is not available". + +- https://192.168.99.100:8443 +- username: developer +- password: developer + +Visit https://192.168.99.100:8443/console/project/project1/browse/routes and click http://dataverse-glassfish-service-project1.192.168.99.100.nip.io/ or whatever is shows. This assumes you named your project ``project1``. + +Cleaning up +~~~~~~~~~~~ + +``oc delete project project1`` + +Minishift Resources +~~~~~~~~~~~~~~~~~~~ + +The following resources might be helpful. + +- https://blog.openshift.com/part-1-from-app-to-openshift-runtimes-and-templates/ +- https://blog.openshift.com/part-2-creating-a-template-a-technical-walkthrough/ +- https://docs.openshift.com/enterprise/3.0/architecture/core_concepts/templates.html + ---- Previous: :doc:`intro` | Next: :doc:`version-control` From 4702e0af6e0db773e35b89a804e5eef743e29b65 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 15 Sep 2017 19:37:18 -0400 Subject: [PATCH 03/25] use config from @danmcp #4040 --- conf/openshift/openshift.json | 83 ++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index ea09a272347..6af0d0531ab 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -26,13 +26,86 @@ "port": 8080, "targetPort": 8080 } - ], - "containers": [ + ] + } + }, + { + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "ndslabs-dataverse" + }, + "spec": { + "dockerImageRepository": "ndslabs/dataverse" + } + }, + { + "kind": "DeploymentConfig", + "apiVersion": "v1", + "metadata": { + "name": "dataverse-glassfish", + "annotations": { + "template.alpha.openshift.io/wait-for-ready": "true" + } + }, + "spec": { + "template": { + "metadata": { + "labels": { + "name": "ndslabs-dataverse" + } + }, + "spec": { + "containers": [ + { + "name": "ndslabs-dataverse", + "image": "ndslabs-dataverse", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": {}, + "privileged": false + } + } + ] + } + }, + "strategy": { + "type": "Rolling", + "rollingParams": { + "updatePeriodSeconds": 1, + "intervalSeconds": 1, + "timeoutSeconds": 120 + }, + "resources": {} + }, + "triggers": [ + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "ndslabs-dataverse" + ], + "from": { + "kind": "ImageStreamTag", + "name": "ndslabs-dataverse:latest" + } + } + }, { - "name": "dataverse-glassfish", - "image": "ndslabs/dataverse" + "type": "ConfigChange" } - ] + ], + "replicas": 1, + "selector": { + "name": "ndslabs-dataverse" + } } } ] From f41d7533098b256c42a8d7effda645f267f89242 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 18 Sep 2017 11:41:29 -0400 Subject: [PATCH 04/25] add centos/postgresql-94-centos7 #4040 --- conf/openshift/openshift.json | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index 6af0d0531ab..f34ae9e95d3 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -40,6 +40,16 @@ } }, { + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "centos-postgresql-94-centos7" + }, + "spec": { + "dockerImageRepository": "centos/postgresql-94-centos7" + } + }, + { "kind": "DeploymentConfig", "apiVersion": "v1", "metadata": { @@ -71,6 +81,35 @@ "capabilities": {}, "privileged": false } + }, + { + "name": "centos-postgresql-94-centos7", + "image": "centos-postgresql-94-centos7", + "ports": [ + { + "containerPort": 5432, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "POSTGRESQL_USER", + "value": "pgUserValue" + }, + { + "name": "POSTGRESQL_PASSWORD", + "value": "pgPasswordValue" + }, + { + "name": "POSTGRESQL_DATABASE", + "value": "pgDatabaseValue" + } + ], + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": {}, + "privileged": false + } } ] } @@ -98,6 +137,19 @@ } } }, + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "centos-postgresql-94-centos7" + ], + "from": { + "kind": "ImageStreamTag", + "name": "centos-postgresql-94-centos7:latest" + } + } + }, { "type": "ConfigChange" } From c20dd3904e523504d2e1965cb52b0f2a30d9f274 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 18 Sep 2017 12:31:14 -0400 Subject: [PATCH 05/25] add ndslabs/dataverse-solr image #4040 --- conf/openshift/openshift.json | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index f34ae9e95d3..c3adbbcfc57 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -49,6 +49,16 @@ "dockerImageRepository": "centos/postgresql-94-centos7" } }, + { + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "ndslabs-dataverse-solr" + }, + "spec": { + "dockerImageRepository": "ndslabs/dataverse-solr" + } + }, { "kind": "DeploymentConfig", "apiVersion": "v1", @@ -110,6 +120,21 @@ "capabilities": {}, "privileged": false } + }, + { + "name": "ndslabs-dataverse-solr", + "image": "ndslabs-dataverse-solr", + "ports": [ + { + "containerPort": 8983, + "protocol": "TCP" + } + ], + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": {}, + "privileged": false + } } ] } @@ -150,6 +175,19 @@ } } }, + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "ndslabs-dataverse-solr" + ], + "from": { + "kind": "ImageStreamTag", + "name": "ndslabs-dataverse-solr:latest" + } + } + }, { "type": "ConfigChange" } From 3ab921c69c08668d1d3f8160c8de6003b5216731 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Sep 2017 13:36:09 -0400 Subject: [PATCH 06/25] add more handy `oc` commands to docs #4040 --- .../source/developers/dev-environment.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 798604cc621..fd27b734ec6 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -364,6 +364,21 @@ Create a Dataverse App within the Minishift Project ``oc new-app conf/openshift/openshift.json`` +Check Status of Dataverse Deployment to Minishift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``oc get all`` + +Review Logs of Dataverse Deployment to Minishift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``oc logs -c ndslabs-dataverse $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep -v dataverse-glassfish-1-deploy)`` + +Get a Shell (ssh/rsh) on Glassfish Server Deployed to Minishift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``oc rsh $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep -v dataverse-glassfish-1-deploy)`` + Make the Dataverse App Available to Your Browser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 21983673171d6483ab9acdb9065312a953429d12 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Sep 2017 17:30:03 -0400 Subject: [PATCH 07/25] expose postgres env vars needed by init-glassfish #4040 --- conf/openshift/openshift.json | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index c3adbbcfc57..3289b1db453 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -59,7 +59,7 @@ "dockerImageRepository": "ndslabs/dataverse-solr" } }, - { + { "kind": "DeploymentConfig", "apiVersion": "v1", "metadata": { @@ -86,6 +86,24 @@ "protocol": "TCP" } ], + "env": [ + { + "name": "SMTP_HOST", + "value": "localhost" + }, + { + "name": "POSTGRES_USER", + "value": "dvnapp" + }, + { + "name": "POSTGRES_PASSWORD", + "value": "dvnappPassword" + }, + { + "name": "POSTGRES_DATABASE", + "value": "dvndb" + } + ], "imagePullPolicy": "IfNotPresent", "securityContext": { "capabilities": {}, From 6ec54c2b7f1518a31331788db2d613e12fbfee3c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Sep 2017 17:43:08 -0400 Subject: [PATCH 08/25] must have selector for expose route to work #4040 --- conf/openshift/openshift.json | 3 +++ doc/sphinx-guides/source/developers/dev-environment.rst | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index 3289b1db453..b9c3ad8f80e 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -19,6 +19,9 @@ "name": "dataverse-glassfish-service" }, "spec": { + "selector": { + "name": "ndslabs-dataverse" + }, "ports": [ { "name": "web", diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index fd27b734ec6..fbd0866c786 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -379,6 +379,11 @@ Get a Shell (ssh/rsh) on Glassfish Server Deployed to Minishift ``oc rsh $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep -v dataverse-glassfish-1-deploy)`` +From the ``rsh`` prompt you could run something like the following to build confidence that Dataverse is running on port 8080: + +``curl -L localhost:8080`` + + Make the Dataverse App Available to Your Browser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -387,8 +392,6 @@ Make the Dataverse App Available to Your Browser Log into Minishift and Visit Dataverse in your Browser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -FIXME: This doesn't work yet. Rather than Dataverse running, for now you are expected to see "Application is not available". - - https://192.168.99.100:8443 - username: developer - password: developer From e90f7714c352aa6c43ee061384cb9b4abfa0e280 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Sep 2017 17:54:38 -0400 Subject: [PATCH 09/25] set dataverseAdmin password to "admin" #4040 --- conf/openshift/openshift.json | 4 ++++ doc/sphinx-guides/source/developers/dev-environment.rst | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index b9c3ad8f80e..c2f033c8912 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -90,6 +90,10 @@ } ], "env": [ + { + "name": "ADMIN_PASSWORD", + "value": "admin" + }, { "name": "SMTP_HOST", "value": "localhost" diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index fbd0866c786..5685751aaf9 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -354,6 +354,13 @@ Make the oc Command Executable ``eval $(minishift oc-env)`` +Log in to Minishift +~~~~~~~~~~~~~~~~~~~ + +``oc login`` + +Use "developer" as the username and a couple characters as the password. + Create a Minishift Project ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -398,6 +405,8 @@ Log into Minishift and Visit Dataverse in your Browser Visit https://192.168.99.100:8443/console/project/project1/browse/routes and click http://dataverse-glassfish-service-project1.192.168.99.100.nip.io/ or whatever is shows. This assumes you named your project ``project1``. +You should be able to log in with username "dataverseAdmin" and password "admin". + Cleaning up ~~~~~~~~~~~ From 975d38e9a5eaa115db3c25bb17492f19de955096 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Sep 2017 19:02:53 -0400 Subject: [PATCH 10/25] explain how to run containers as root #4040 --- doc/sphinx-guides/source/developers/dev-environment.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 5685751aaf9..03b239e6337 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -361,6 +361,13 @@ Log in to Minishift Use "developer" as the username and a couple characters as the password. +Allow Containers to Run as Root in Minishift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This isn't ideal, but for now we're allowing containers to run as root. FIXME: Eventually, we should create containers that don't require root. + +``oc adm policy add-scc-to-user anyuid -z default --as system:admin`` + Create a Minishift Project ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 0a444105a2fdb5924e9598f0d2bf5f98b4dff700 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 20 Sep 2017 06:54:18 -0400 Subject: [PATCH 11/25] build and switch to iqss/dataverse-solr on Docker Hub #4040 --- conf/docker/build.sh | 6 +++++- conf/docker/solr/Dockerfile | 36 ++++++++++++++++++++++++++-------- conf/docker/solr/entrypoint.sh | 10 ++++++++++ conf/openshift/openshift.json | 12 ++++++------ 4 files changed, 49 insertions(+), 15 deletions(-) mode change 100644 => 100755 conf/docker/build.sh create mode 100755 conf/docker/solr/entrypoint.sh diff --git a/conf/docker/build.sh b/conf/docker/build.sh old mode 100644 new mode 100755 index 95a3a15979a..44100c99aff --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -1,2 +1,6 @@ #!/bin/sh -echo "TODO: Build docker images and push to DockerHub." +# Creates images and pushes them to Docker Hub. +GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +# Use "conf" directory as context so we can copy schema.xml into Solr image. +docker build -t iqss/dataverse-solr:$GIT_BRANCH -f solr/Dockerfile ../../conf +docker push iqss/dataverse-solr:$GIT_BRANCH diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile index 7d4bf9292e2..99114ce6a6d 100644 --- a/conf/docker/solr/Dockerfile +++ b/conf/docker/solr/Dockerfile @@ -1,8 +1,28 @@ -# Solr for Dataverse -# -# See the note about Dataverse 4.2.3 vs. 4.7 at conf/docker/dataverse-glassfish/Dockerfile -# -# Note that there is a separate Dockerfile we can start with: -# -# Dataverse 4.2.3: https://github.com/pdurbin/ndslabs-dataverse/blob/v4.2.3.1/dockerfiles/solr/Dockerfile -# Dataverse 4.7: https://github.com/pdurbin/ndslabs-dataverse/blob/upgrade-4.7/dockerfiles/solr/Dockerfile +FROM centos:7.2.1511 +MAINTAINER Dataverse (support@dataverse.org) + +RUN yum install -y wget unzip perl git java-1.8.0-openjdk-devel postgresql.x86_64 + +# Install Solr 4.6.0 +# The context of the build is the "conf" directory. +COPY solr/4.6.0/schema.xml /tmp + +RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz && \ + tar xvzf solr-4.6.0.tgz && \ + mv solr-4.6.0 /usr/local/ && \ + cd /usr/local/solr-4.6.0/example/solr/collection1/conf/ && \ + mv schema.xml schema.xml.backup && \ + cp /tmp/schema.xml . && \ + rm /tmp/solr-4.6.0.tgz + +RUN ln -s /usr/local/solr-4.6.0/example/logs /var/log/solr + +VOLUME /usr/local/solr-4.6.0/example/solr/collection1/data + +EXPOSE 8983 + +COPY docker/solr/Dockerfile /Dockerfile +COPY docker/solr/entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["solr"] diff --git a/conf/docker/solr/entrypoint.sh b/conf/docker/solr/entrypoint.sh new file mode 100755 index 00000000000..7fd8d6380c2 --- /dev/null +++ b/conf/docker/solr/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ "$1" = 'solr' ]; then + cd /usr/local/solr-4.6.0/example/ + java -jar start.jar +elif [ "$1" = 'usage' ]; then + echo 'docker run -d iqss/dataverse-solr solr' +else + exec "$@" +fi diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index c2f033c8912..36a29a87cbc 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -56,10 +56,10 @@ "kind": "ImageStream", "apiVersion": "v1", "metadata": { - "name": "ndslabs-dataverse-solr" + "name": "iqss-dataverse-solr" }, "spec": { - "dockerImageRepository": "ndslabs/dataverse-solr" + "dockerImageRepository": "iqss/dataverse-solr" } }, { @@ -147,8 +147,8 @@ } }, { - "name": "ndslabs-dataverse-solr", - "image": "ndslabs-dataverse-solr", + "name": "iqss-dataverse-solr", + "image": "iqss-dataverse-solr", "ports": [ { "containerPort": 8983, @@ -205,11 +205,11 @@ "imageChangeParams": { "automatic": true, "containerNames": [ - "ndslabs-dataverse-solr" + "iqss-dataverse-solr" ], "from": { "kind": "ImageStreamTag", - "name": "ndslabs-dataverse-solr:latest" + "name": "iqss-dataverse-solr:4040-docker-openshift" } } }, From 7c81b4e73a3570513dd372b489a66b603f6d594a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 21 Sep 2017 11:30:50 -0400 Subject: [PATCH 12/25] switch from ndslabs to iqss Dataverse/Glassfish image #4040 --- conf/docker/build.sh | 8 ++ conf/docker/dataverse-glassfish/.gitignore | 5 + conf/docker/dataverse-glassfish/Dockerfile | 102 ++++++++++++++- conf/docker/dataverse-glassfish/entrypoint.sh | 122 ++++++++++++++++++ conf/openshift/openshift.json | 22 ++-- .../source/developers/dev-environment.rst | 24 +++- 6 files changed, 266 insertions(+), 17 deletions(-) create mode 100644 conf/docker/dataverse-glassfish/.gitignore create mode 100755 conf/docker/dataverse-glassfish/entrypoint.sh diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 44100c99aff..9eb20f9a857 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -4,3 +4,11 @@ GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) # Use "conf" directory as context so we can copy schema.xml into Solr image. docker build -t iqss/dataverse-solr:$GIT_BRANCH -f solr/Dockerfile ../../conf docker push iqss/dataverse-solr:$GIT_BRANCH +# TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. +cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war +cp ../../scripts/installer/dvinstall.zip dataverse-glassfish +cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish +cp ../../downloads/glassfish-4.1.zip dataverse-glassfish +cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish +docker build -t iqss/dataverse-glassfish:$GIT_BRANCH dataverse-glassfish +docker push iqss/dataverse-glassfish:$GIT_BRANCH diff --git a/conf/docker/dataverse-glassfish/.gitignore b/conf/docker/dataverse-glassfish/.gitignore new file mode 100644 index 00000000000..b0e6e38894f --- /dev/null +++ b/conf/docker/dataverse-glassfish/.gitignore @@ -0,0 +1,5 @@ +glassfish-4.1.zip +weld-osgi-bundle-2.2.10.Final-glassfish4.jar +dvinstall.zip +dataverse.war +default.config diff --git a/conf/docker/dataverse-glassfish/Dockerfile b/conf/docker/dataverse-glassfish/Dockerfile index 603660b5264..a71aabd6ee1 100644 --- a/conf/docker/dataverse-glassfish/Dockerfile +++ b/conf/docker/dataverse-glassfish/Dockerfile @@ -1,5 +1,99 @@ -# Glassfish and Dataverse in one Docker image, as suggested by danmcp. -# -# TODO: Start with the Dataverse 4.2.3 version at https://github.com/pdurbin/ndslabs-dataverse/blob/v4.2.3.1/dockerfiles/dataverse/Dockerfile +FROM centos:7.2.1511 +MAINTAINER Dataverse (support@dataverse.org) + +COPY glassfish-4.1.zip /tmp +COPY weld-osgi-bundle-2.2.10.Final-glassfish4.jar /tmp +COPY default.config /tmp +COPY dvinstall.zip /tmp + +# Install dependencies +#RUN yum install -y unzip +RUN yum install -y \ + cronie \ + git \ + java-1.8.0-openjdk-devel \ + nc \ + perl \ + postgresql \ + sha1sum \ + unzip \ + wget + +ENV GLASSFISH_DOWNLOAD_SHA1 d1a103d06682eb08722fbc9a93089211befaa080 +ENV GLASSFISH_DIRECTORY "/usr/local/glassfish4" +ENV HOST_DNS_ADDRESS "localhost" +ENV POSTGRES_DB "dvndb" +ENV POSTGRES_USER "dvnapp" +ENV RSERVE_USER "rserve" +ENV RSERVE_PASSWORD "rserve" + +#RUN ls /tmp # -# TODO: Once we get Dataverse 4.2.3 working, we can try Dataverse 4.7 from https://github.com/pdurbin/ndslabs-dataverse/blob/upgrade-4.7/dockerfiles/dataverse/Dockerfile +RUN find /tmp +# +#RUN exitEarly + +# Install Glassfish 4.1 +RUN cd /tmp \ + && unzip glassfish-4.1.zip \ + && mv glassfish4 /usr/local \ + && cd /usr/local/glassfish4/glassfish/modules \ + && rm weld-osgi-bundle.jar \ + #&& rm weld-osgi-bundle.jar + #&& wget http://central.maven.org/maven2/org/jboss/weld/weld-osgi-bundle/2.2.10.SP1/weld-osgi-bundle-2.2.10.SP1-glassfish4.jar \ + #&& cp /tmp/weld-osgi-bundle-2.2.10.SP1-glassfish4.jar . \ + && cp /tmp/weld-osgi-bundle-2.2.10.Final-glassfish4.jar . \ + #&& cp /tmp/weld-osgi-bundle-2.2.10.SP1-glassfish4.jar . + #&& echo "$GLASSFISH_DOWNLOAD_SHA1 weld-osgi-bundle-2.2.10.SP1-glassfish4.jar" | sha1sum -c - \ + #&& rm -rf /usr/local/glassfish4/glassfish/domains/domain1 \ + && rm /tmp/glassfish-4.1.zip + +#RUN exitEarlyBeforeJq +RUN yum -y install epel-release +RUN yum install -y jq + +# Install jq +#RUN cd /tmp \ +# && wget https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \ +# && mv jq-linux64 /usr/local/bin \ +# && chmod +x /usr/local/bin/jq-linux64 \ +# && ln -s /usr/local/bin/jq-linux64 /usr/local/bin/jq + +# Customized persistence xml to avoid database recreation +#RUN mkdir -p /tmp/WEB-INF/classes/META-INF/ +#COPY WEB-INF/classes/META-INF/persistence.xml /tmp/WEB-INF/classes/META-INF/ + +# Install Dataverse WAR +RUN cd ~ \ +# && wget https://github.com/IQSS/dataverse/releases/download/v4.7/dvinstall.zip \ + && cp /tmp/dvinstall.zip . \ + && unzip dvinstall.zip + #&& unzip dvinstall.zip \ + #&& jar -uf dvinstall/dataverse.war -C /tmp/ /WEB-INF/classes/META-INF/persistence.xml + +# Install iRods iCommands +#RUN cd /tmp \ +# && yum -y install epel-release \ +# && yum -y install ftp://ftp.renci.org/pub/irods/releases/4.1.6/centos7/irods-icommands-4.1.6-centos7-x86_64.rpm + +#COPY config-glassfish /root/dvinstall +#COPY restart-glassfish /root/dvinstall +#COPY config-dataverse /root/dvinstall + +#RUN cd /root/dvinstall && ./config-dataverse + +COPY ./entrypoint.sh / +#COPY ./ddl /root/dvinstall +#COPY ./init-postgres /root/dvinstall +#COPY ./init-glassfish /root/dvinstall +#COPY ./init-dataverse /root/dvinstall +#COPY ./setup-all.sh /root/dvinstall +#COPY ./setup-irods.sh /root/dvinstall +COPY ./Dockerfile / + +VOLUME /usr/local/glassfish4/glassfish/domains/domain1/files + +EXPOSE 8080 + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["dataverse"] diff --git a/conf/docker/dataverse-glassfish/entrypoint.sh b/conf/docker/dataverse-glassfish/entrypoint.sh new file mode 100755 index 00000000000..b50790f53ab --- /dev/null +++ b/conf/docker/dataverse-glassfish/entrypoint.sh @@ -0,0 +1,122 @@ +#!/bin/bash -x + +# Entrypoint script for Dataverse web application. This script waits +# for dependent services (Rserve, Postgres, Solr) to start before +# initializing Glassfish. + + + +set -e + +if [ "$1" = 'dataverse' ]; then + + export GLASSFISH_DIRECTORY=/usr/local/glassfish4 + export HOST_DNS_ADDRESS=localhost + + TIMEOUT=30 + + if [ -n "$RSERVE_SERVICE_HOST" ]; then + RSERVE_HOST=$RSERVE_SERVICE_HOST + elif [ -n "$RSERVE_PORT_6311_TCP_ADDR" ]; then + RSERVE_HOST=$RSERVE_PORT_6311_TCP_ADDR + elif [ -z "$RSERVE_HOST" ]; then + RSERVE_HOST="localhost" + fi + export RSERVE_HOST + + if [ -n "$RSERVE_SERVICE_PORT" ]; then + RSERVE_PORT=$RSERVE_SERVICE_PORT + elif [ -n "$RSERVE_PORT_6311_TCP_PORT" ]; then + RSERVE_PORT=$RSERVE_PORT_6311_TCP_PORT + elif [ -z "$RSERVE_PORT" ]; then + RSERVE_PORT="6311" + fi + export RSERVE_PORT + + echo "Using Rserve at $RSERVE_HOST:$RSERVE_PORT" + + if ncat $RSERVE_HOST $RSERVE_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then + echo Rserve running; + else + echo Optional service Rserve not running. + fi + + + # postgres + if [ -n "$POSTGRES_SERVICE_HOST" ]; then + POSTGRES_HOST=$POSTGRES_SERVICE_HOST + elif [ -n "$POSTGRES_PORT_5432_TCP_ADDR" ]; then + POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR + elif [ -z "$POSTGRES_HOST" ]; then + POSTGRES_HOST="localhost" + fi + export POSTGRES_HOST + + if [ -n "$POSTGRES_SERVICE_PORT" ]; then + POSTGRES_PORT=$POSTGRES_SERVICE_PORT + elif [ -n "$POSTGRES_PORT_5432_TCP_PORT" ]; then + POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT + else + POSTGRES_PORT=5432 + fi + export POSTGRES_PORT + + echo "Using Postgres at $POSTGRES_HOST:$POSTGRES_PORT" + + if ncat $POSTGRES_HOST $POSTGRES_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then + echo Postgres running; + else + echo Required service Postgres not running. Have you started the required services? + exit 1 + fi + + # solr + if [ -n "$SOLR_SERVICE_HOST" ]; then + SOLR_HOST=$SOLR_SERVICE_HOST + elif [ -n "$SOLR_PORT_8983_TCP_ADDR" ]; then + SOLR_HOST=$SOLR_PORT_8983_TCP_ADDR + elif [ -z "$SOLR_HOST" ]; then + SOLR_HOST="localhost" + fi + export SOLR_HOST + + if [ -n "$SOLR_SERVICE_PORT" ]; then + SOLR_PORT=$SOLR_SERVICE_PORT + elif [ -n "$SOLR_PORT_8983_TCP_PORT" ]; then + SOLR_PORT=$SOLR_PORT_8983_TCP_PORT + else + SOLR_PORT=8983 + fi + export SOLR_PORT + + echo "Using Solr at $SOLR_HOST:$SOLR_PORT" + + if ncat $SOLR_HOST $SOLR_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then + echo Solr running; + else + echo Required service Solr not running. Have you started the required services? + exit 1 + fi + + echo changing to dvinstall directory + cd ~/dvinstall + echo Copying the non-interactive file into place + cp /tmp/default.config . + echo Looking at first few lines of default.config + head default.config + # non-interactive install + echo Running non-interactive install + #./install -y -f > install.out 2> install.err + ./install -y -f + +# if [ -n "$DVICAT_PORT_1247_TCP_PORT" ]; then +# ./setup-irods.sh +# fi + + echo -e "\n\nDataverse started" + + sleep infinity +else + exec "$@" +fi + diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index 36a29a87cbc..bb3930dc59d 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -20,7 +20,7 @@ }, "spec": { "selector": { - "name": "ndslabs-dataverse" + "name": "iqss-dataverse-glassfish" }, "ports": [ { @@ -36,10 +36,10 @@ "kind": "ImageStream", "apiVersion": "v1", "metadata": { - "name": "ndslabs-dataverse" + "name": "dataverse-plus-glassfish" }, "spec": { - "dockerImageRepository": "ndslabs/dataverse" + "dockerImageRepository": "iqss/dataverse-glassfish" } }, { @@ -66,7 +66,7 @@ "kind": "DeploymentConfig", "apiVersion": "v1", "metadata": { - "name": "dataverse-glassfish", + "name": "deploy-dataverse-glassfish", "annotations": { "template.alpha.openshift.io/wait-for-ready": "true" } @@ -75,14 +75,14 @@ "template": { "metadata": { "labels": { - "name": "ndslabs-dataverse" + "name": "iqss-dataverse-glassfish" } }, "spec": { "containers": [ { - "name": "ndslabs-dataverse", - "image": "ndslabs-dataverse", + "name": "dataverse-plus-glassfish", + "image": "dataverse-plus-glassfish", "ports": [ { "containerPort": 8080, @@ -169,7 +169,7 @@ "rollingParams": { "updatePeriodSeconds": 1, "intervalSeconds": 1, - "timeoutSeconds": 120 + "timeoutSeconds": 300 }, "resources": {} }, @@ -179,11 +179,11 @@ "imageChangeParams": { "automatic": true, "containerNames": [ - "ndslabs-dataverse" + "dataverse-plus-glassfish" ], "from": { "kind": "ImageStreamTag", - "name": "ndslabs-dataverse:latest" + "name": "dataverse-plus-glassfish:4040-iqss-glassfish" } } }, @@ -219,7 +219,7 @@ ], "replicas": 1, "selector": { - "name": "ndslabs-dataverse" + "name": "iqss-dataverse-glassfish" } } } diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 03b239e6337..6827c30aeaa 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -331,7 +331,8 @@ If you've reconfigured from EZID to DataCite and are seeing ``Response code: 400 OpenShift --------- -From the Dataverse perspective, we are in the business of providing a "template" for OpenShift that describes how the various components we build our application on (Glassfish, PostgreSQL, Solr, the Dataverse war file itself, etc.) work together. We also plan to publish Docker images to DockerHub but for now we are trying to use the Dataverse images published to https://hub.docker.com/r/ndslabs/ +From the Dataverse perspective, we are in the business of providing a "template" for OpenShift that describes how the various components we build our application on (Glassfish, PostgreSQL, Solr, the Dataverse war file itself, etc.) work together. We publish Docker images to DockerHub at https://hub.docker.com/u/iqss/ that are used in the OpenShift template. Dataverse's use of Docker is documented below in a separate section. + The OpenShift template for Dataverse can be found at ``conf/openshift/openshift.json`` and if you need to hack on the template or related files under ``conf/docker`` it is recommended that you iterate on them using Minishift. @@ -386,7 +387,7 @@ Check Status of Dataverse Deployment to Minishift Review Logs of Dataverse Deployment to Minishift ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``oc logs -c ndslabs-dataverse $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep -v dataverse-glassfish-1-deploy)`` +``oc logs -c dataverse-plus-glassfish $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep -v dataverse-glassfish-1-deploy)`` Get a Shell (ssh/rsh) on Glassfish Server Deployed to Minishift ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -428,6 +429,25 @@ The following resources might be helpful. - https://blog.openshift.com/part-2-creating-a-template-a-technical-walkthrough/ - https://docs.openshift.com/enterprise/3.0/architecture/core_concepts/templates.html +Docker +------ + +Minishift makes use of Docker images on Docker Hub. To build new Docker images and push them to Docker Hub, you'll need to install Docker. + +Installing Docker +~~~~~~~~~~~~~~~~~ + +On Linux, you can probably get Docker from your package manager. + +On Mac, download the ``.dmg`` from https://www.docker.com and install it. As of this writing is it known as Docker Community Edition for Mac. + +We're working with Docker in the context of Minishift so if you haven't installed Minishift yet, follow the instructions above and make sure you get the Dataverse Docker images running in Minishift before you start messing with them. + +Get Set Up to Push Docker Images to Minishift Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +FIXME https://docs.openshift.org/latest/minishift/openshift/openshift-docker-registry.html indicates that it should be possible to make use of the builtin registry in Minishift while iterating on Docker images but you may get "unauthorized: authentication required" when trying to push to it as reported at https://github.com/minishift/minishift/issues/817 so until we figure this out, you must push to Docker Hub instead. Run ``docker login`` and use the ``conf/docker/build.sh`` script to push Docker images you create to https://hub.docker.com/u/iqss/ + ---- Previous: :doc:`intro` | Next: :doc:`version-control` From 6e4ef452a172b2b4d56bccee20f03b679438f175 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 22 Sep 2017 15:30:56 -0400 Subject: [PATCH 13/25] use the right Docker tag for Dataverse/Glassfish #4040 --- conf/openshift/openshift.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index bb3930dc59d..6732a08c13c 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -183,7 +183,7 @@ ], "from": { "kind": "ImageStreamTag", - "name": "dataverse-plus-glassfish:4040-iqss-glassfish" + "name": "dataverse-plus-glassfish:4040-docker-openshift" } } }, From 8040a5dcabb37e90edbec6a95f570405cafe1d8b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Sep 2017 14:25:42 -0400 Subject: [PATCH 14/25] document dataverse.timerServer JVM option --- doc/sphinx-guides/source/installation/config.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a48afe9fd87..9bcd9adae17 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -576,6 +576,11 @@ dataverse.handlenet.admprivphrase +++++++++++++++++++++++++++++++++ This JVM setting is also part of **handles** configuration. The Handle.Net installer lets you choose whether to encrypt the admcredfile private key or not. If you do encrypt it, this is the pass phrase that it's encrypted with. +dataverse.timerServer ++++++++++++++++++++++ + +This JVM option is only relevant if you plan to run multiple Glassfish servers for redundancy. Only one Glassfish server can act as the dedicated timer server and for details on promoting or demoting a Glassfish server to handle this responsibility, see :doc:`/admin/timers`. + Database Settings ----------------- @@ -1063,4 +1068,4 @@ This setting is experimental and related to Repository Storage Abstraction Layer Limit on how many guestbook entries to display on the guestbook-responses page. By default, only the 5000 most recent entries will be shown. Use the standard settings API in order to change the limit. For example, to set it to 10,000, make the following API call: -``curl -X PUT -d 10000 http://localhost:8080/api/admin/settings/:GuestbookResponsesPageDisplayLimit`` \ No newline at end of file +``curl -X PUT -d 10000 http://localhost:8080/api/admin/settings/:GuestbookResponsesPageDisplayLimit`` From 815e48ab76aa8716f3e1091eaa982a5da81a179e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Sep 2017 14:29:26 -0400 Subject: [PATCH 15/25] document how to create Docker images #4040 --- .../source/developers/dev-environment.rst | 68 ++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 6827c30aeaa..1dc75c68490 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -358,17 +358,25 @@ Make the oc Command Executable Log in to Minishift ~~~~~~~~~~~~~~~~~~~ -``oc login`` +Note that if you just installed Minishift, you are probably logged in already, but it doesn't hurt to log in again. + +``oc login --username developer --password=whatever`` Use "developer" as the username and a couple characters as the password. Allow Containers to Run as Root in Minishift ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This isn't ideal, but for now we're allowing containers to run as root. FIXME: Eventually, we should create containers that don't require root. +For now we're allowing containers to run as root. Until the images are fixed to run as non-root, run the following command: ``oc adm policy add-scc-to-user anyuid -z default --as system:admin`` +FIXME: Eventually, we should create containers that don't require root. When we do. Make sure Dataverse still runs on Minishift after you've stopped allowing containers to run as root by issuing the following command: + +``oc adm policy remove-scc-from-user anyuid -z default --as system:admin`` + +For more information on improving Docker images to run as non-root, see "Support Arbitrary User IDs" at https://docs.openshift.org/latest/creating_images/guidelines.html#openshift-origin-specific-guidelines + Create a Minishift Project ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -420,6 +428,12 @@ Cleaning up ``oc delete project project1`` +Making Changes +~~~~~~~~~~~~~~ + +If you're interested in using Minishift for development and want to change the Dataverse code, you will need to get set up to create Docker images based on your changes and push them to a Docker registry such as Docker Hub. See the section below on Docker for details. + + Minishift Resources ~~~~~~~~~~~~~~~~~~~ @@ -441,8 +455,58 @@ On Linux, you can probably get Docker from your package manager. On Mac, download the ``.dmg`` from https://www.docker.com and install it. As of this writing is it known as Docker Community Edition for Mac. +On Windows, FIXME ("Docker Community Edition for Windows" maybe???). + We're working with Docker in the context of Minishift so if you haven't installed Minishift yet, follow the instructions above and make sure you get the Dataverse Docker images running in Minishift before you start messing with them. +Editing Dataverse Docker Images +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As of this writing, the Dataverse Docker images we publish under https://hub.docker.com/u/iqss/ are highly experimental. They're tagged with branch names like ``kick-the-tires`` rather than release numbers. + +Change to the docker directory: + +``cd conf/docker`` + +Edit one of the files: + +``vim dataverse-glassfish/Dockerfile`` + +At this point you want to build the image and run it. We are assuming you want to run it in your Minishift environment. We will be building your image and pushing it to Docker Hub. Then you will be pulling the image down from Docker Hub to run in your Minishift installation. If this sounds inefficient, you're right, but we haven't been able to figure out how to make use of Minishift's built in registry (see below) so we're pushing to Docker Hub instead. + +Log in to Docker Hub with an account that has access to push to the ``iqss`` organization: + +``docker login`` + +(If you don't have access to push to the ``iqss`` organization, you can push elsewhere and adjust your ``openshift.json`` file accordingly.) + +Build and push the images to Docker Hub: + +``./build.sh`` + +Note that you will see output such as ``digest: sha256:213b6380e6ee92607db5d02c9e88d7591d81f4b6d713224d47003d5807b93d4b`` that should later be reflected in Minishift to indicate that you are using the latest image you just pushed to Docker Hub. + +You can get a list of all repos under the ``iqss`` organization with this: + +``curl https://hub.docker.com/v2/repositories/iqss/`` + +To see a specific repo: + +``curl https://hub.docker.com/v2/repositories/iqss/dataverse-glassfish/`` + +Known issues with Dataverse Docker images +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Again, Dataverse Docker images are highly experimental at this point. As of this writing, their purpose is primarily for kicking the tires on Dataverse. Here are some known issues: + +- The Dataverse installer is run in the entrypoint script every time you run the image. Ideally, Dataverse would be installed in the Dockerfile instead. Dataverse is being installed in the entrypoint script because it needs PosgreSQL to be up already so that database tables can be created when the war file is deployed. +- The Docker images have to be run as root. See the discussion above. +- The storage should be abstracted. Storage of data files and PostgreSQL data. Probably Solr data. +- Better tuning of memory by examining ``/sys/fs/cgroup/memory/memory.limit_in_bytes`` and incorporating this into the Dataverse installation script. +- Only a single Glassfish server can be used. See "Dedicated timer server in a Dataverse server cluster" in the :doc:`/admin/timers` section of the Installation Guide. +- Only a single PostgreSQL server can be used. +- Only a single Solr server can be used. + Get Set Up to Push Docker Images to Minishift Registry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 43d384459fc03c851f433911e045eb555ef70419 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Sep 2017 14:59:08 -0400 Subject: [PATCH 16/25] add NDS Labs Workbench for testing Dataverse #4040 #4152 --- doc/sphinx-guides/source/installation/prep.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index ba8bf08cd4a..3b57c849f64 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -14,6 +14,13 @@ We'll try to get you up and running as quickly as possible, but we thought you m Choose Your Own Installation Adventure -------------------------------------- +NDS Labs Workbench (for Testing Only) ++++++++++++++++++++++++++++++++++++++ + +The National Data Service (NDS) is community-driven effort guided by the National Data Service Consortium. NDS Labs has packaged Dataverse as `one of many data management tools `_ that can be quickly deployed for evaluation purposes in their tool based on Kubernetes called NDS Labs Workbench. To get started, visit http://www.nationaldataservice.org/projects/labs.html . + +Please note that the version of Dataverse in NDS Labs Workbench may lag behind the latest release. Craig Willis from NDS Labs did an excellent job of adding Dataverse 4 to NDS Labs Workbench and the Dataverse team hopes to some day take over the creation of Docker images so the latest version of Dataverse can be evaluated in the workbench. + OpenShift (for Testing Only) ++++++++++++++++++++++++++++ From ce949c93a556b12fd46829ae37ff4808adc3a95f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Sep 2017 15:39:18 -0400 Subject: [PATCH 17/25] Docker tag with no issue number (kick-the-tires) #4040 The idea is that we won't mess with the kick-the-tires for a bit so that people can play with Minishift without that tag constantly changing under their feet while we work on other efforts in other tags such as getting the containers to run as non-root and getting them to run in 1 GB of memory. --- conf/docker/build.sh | 10 ++++++---- conf/openshift/openshift.json | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 9eb20f9a857..3fd24d3f109 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -1,14 +1,16 @@ #!/bin/sh # Creates images and pushes them to Docker Hub. GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +# FIXME: Make this script dynamic so you can switch the tag to the branch you're on or a tagged release. +TAG=kick-the-tires # Use "conf" directory as context so we can copy schema.xml into Solr image. -docker build -t iqss/dataverse-solr:$GIT_BRANCH -f solr/Dockerfile ../../conf -docker push iqss/dataverse-solr:$GIT_BRANCH +docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf +docker push iqss/dataverse-solr:$TAG # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war cp ../../scripts/installer/dvinstall.zip dataverse-glassfish cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish cp ../../downloads/glassfish-4.1.zip dataverse-glassfish cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish -docker build -t iqss/dataverse-glassfish:$GIT_BRANCH dataverse-glassfish -docker push iqss/dataverse-glassfish:$GIT_BRANCH +docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish +docker push iqss/dataverse-glassfish:$TAG diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index 6732a08c13c..0f658ace99f 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -183,7 +183,7 @@ ], "from": { "kind": "ImageStreamTag", - "name": "dataverse-plus-glassfish:4040-docker-openshift" + "name": "dataverse-plus-glassfish:kick-the-tires" } } }, @@ -209,7 +209,7 @@ ], "from": { "kind": "ImageStreamTag", - "name": "iqss-dataverse-solr:4040-docker-openshift" + "name": "iqss-dataverse-solr:kick-the-tires" } } }, From b84526c4f66626f40ee309e0e850ab8f9e7bfeef Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Sep 2017 11:07:10 -0400 Subject: [PATCH 18/25] get Dataverse/Glassfish container running as non-root #4040 --- conf/docker/build.sh | 4 +++ conf/docker/dataverse-glassfish/Dockerfile | 29 +++++++++---------- conf/docker/dataverse-glassfish/entrypoint.sh | 19 ++++++++++-- .../source/developers/dev-environment.rst | 26 +++++++++++++++-- 4 files changed, 58 insertions(+), 20 deletions(-) diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 3fd24d3f109..3a9bf27b1e0 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -3,6 +3,9 @@ GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) # FIXME: Make this script dynamic so you can switch the tag to the branch you're on or a tagged release. TAG=kick-the-tires +# kick-the-tires should be relatively stable. Push to tags with branch names to iterate on the images. +#TAG=$GIT_BRANCH +echo Images will be pushed to Docker Hub with the tag $TAG # Use "conf" directory as context so we can copy schema.xml into Solr image. docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf docker push iqss/dataverse-solr:$TAG @@ -13,4 +16,5 @@ cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfis cp ../../downloads/glassfish-4.1.zip dataverse-glassfish cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish +# FIXME: Check the output of `docker build` and only push on success. docker push iqss/dataverse-glassfish:$TAG diff --git a/conf/docker/dataverse-glassfish/Dockerfile b/conf/docker/dataverse-glassfish/Dockerfile index a71aabd6ee1..939ce98fb72 100644 --- a/conf/docker/dataverse-glassfish/Dockerfile +++ b/conf/docker/dataverse-glassfish/Dockerfile @@ -34,19 +34,26 @@ RUN find /tmp #RUN exitEarly # Install Glassfish 4.1 + RUN cd /tmp \ && unzip glassfish-4.1.zip \ && mv glassfish4 /usr/local \ && cd /usr/local/glassfish4/glassfish/modules \ && rm weld-osgi-bundle.jar \ - #&& rm weld-osgi-bundle.jar - #&& wget http://central.maven.org/maven2/org/jboss/weld/weld-osgi-bundle/2.2.10.SP1/weld-osgi-bundle-2.2.10.SP1-glassfish4.jar \ - #&& cp /tmp/weld-osgi-bundle-2.2.10.SP1-glassfish4.jar . \ && cp /tmp/weld-osgi-bundle-2.2.10.Final-glassfish4.jar . \ - #&& cp /tmp/weld-osgi-bundle-2.2.10.SP1-glassfish4.jar . - #&& echo "$GLASSFISH_DOWNLOAD_SHA1 weld-osgi-bundle-2.2.10.SP1-glassfish4.jar" | sha1sum -c - \ - #&& rm -rf /usr/local/glassfish4/glassfish/domains/domain1 \ - && rm /tmp/glassfish-4.1.zip + #FIXME: Patch Grizzly too! + && echo "Done installing and patching Glassfish" + +RUN chmod g=u /etc/passwd + +RUN mkdir -p /home/glassfish +RUN chgrp -R 0 /home/glassfish && \ + chmod -R g=u /home/glassfish + +RUN mkdir -p /usr/local/glassfish4 +RUN chgrp -R 0 /usr/local/glassfish4 && \ + chmod -R g=u /usr/local/glassfish4 + #RUN exitEarlyBeforeJq RUN yum -y install epel-release @@ -63,14 +70,6 @@ RUN yum install -y jq #RUN mkdir -p /tmp/WEB-INF/classes/META-INF/ #COPY WEB-INF/classes/META-INF/persistence.xml /tmp/WEB-INF/classes/META-INF/ -# Install Dataverse WAR -RUN cd ~ \ -# && wget https://github.com/IQSS/dataverse/releases/download/v4.7/dvinstall.zip \ - && cp /tmp/dvinstall.zip . \ - && unzip dvinstall.zip - #&& unzip dvinstall.zip \ - #&& jar -uf dvinstall/dataverse.war -C /tmp/ /WEB-INF/classes/META-INF/persistence.xml - # Install iRods iCommands #RUN cd /tmp \ # && yum -y install epel-release \ diff --git a/conf/docker/dataverse-glassfish/entrypoint.sh b/conf/docker/dataverse-glassfish/entrypoint.sh index b50790f53ab..bc1b7eb3f93 100755 --- a/conf/docker/dataverse-glassfish/entrypoint.sh +++ b/conf/docker/dataverse-glassfish/entrypoint.sh @@ -4,7 +4,17 @@ # for dependent services (Rserve, Postgres, Solr) to start before # initializing Glassfish. - +echo "whoami before..." +whoami +if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + # Make `whoami` return the glassfish user. # See https://docs.openshift.org/3.6/creating_images/guidelines.html#openshift-origin-specific-guidelines + # Fancy bash magic from https://github.com/RHsyseng/container-rhel-examples/blob/1208dcd7d4f431fc6598184dba6341b9465f4197/starter-arbitrary-uid/bin/uid_entrypoint#L4 + echo "${USER_NAME:-glassfish}:x:$(id -u):0:${USER_NAME:-glassfish} user:/home/glassfish:/bin/bash" >> /etc/passwd + fi +fi +echo "whoami after" +whoami set -e @@ -98,8 +108,11 @@ if [ "$1" = 'dataverse' ]; then exit 1 fi - echo changing to dvinstall directory - cd ~/dvinstall + GLASSFISH_INSTALL_DIR="/usr/local/glassfish4" + cd $GLASSFISH_INSTALL_DIR + cp /tmp/dvinstall.zip $GLASSFISH_INSTALL_DIR + unzip dvinstall.zip + cd dvinstall echo Copying the non-interactive file into place cp /tmp/default.config . echo Looking at first few lines of default.config diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 1dc75c68490..00c09f68597 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -390,6 +390,14 @@ Create a Dataverse App within the Minishift Project Check Status of Dataverse Deployment to Minishift ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``oc status`` + +Once images have been downloaded from Docker Hub, the output below will change from ``Pulling`` to ``Pulled``. + +``oc get events | grep Pull`` + +This is a deep dive: + ``oc get all`` Review Logs of Dataverse Deployment to Minishift @@ -406,12 +414,24 @@ From the ``rsh`` prompt you could run something like the following to build conf ``curl -L localhost:8080`` +Make the Dataverse App Available Via HTTP +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Make the Dataverse App Available to Your Browser -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The following curl command is expected to fail until you "expose" the HTTP service. + +``curl http://dataverse-glassfish-service-project1.192.168.99.102.nip.io/api/info/version`` + +Expose the Dataverse web service: ``oc expose svc/dataverse-glassfish-service`` +Make Sure the Dataverse API is Working +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This should show a version number: + +``curl http://dataverse-glassfish-service-project1.192.168.99.102.nip.io/api/info/version`` + Log into Minishift and Visit Dataverse in your Browser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -426,6 +446,8 @@ You should be able to log in with username "dataverseAdmin" and password "admin" Cleaning up ~~~~~~~~~~~ +Note that it can take a few minutes for the deletion of a project to be complete and there doesn't seem to be a great way to know when it's safe to run ``oc new-project project1`` again, slowing down the development feedback loop. FIXME: Find a way to iterate faster. + ``oc delete project project1`` Making Changes From ca6b6becbef0847d78fd3cb0aae20a54d61992a3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Sep 2017 13:28:19 -0400 Subject: [PATCH 19/25] Dataverse is too fat to run in 1 GB of memory #4040 Back out of the "kick the tires" story of using OpenShift Online instead of Vagrant because Dataverse won't fit in the free tier which limits your application to 1 GB of total memory. PostgreSQL and Solr each seem to run fine in 256 MB of memory but Dataverse/Glassfish can't run in 512MB of memory. That said, logic has been added to the installer to check if we're running in Docker and how much memory we have. We don't change the Glassfish heap size based on this value, however, because the war files fails to deploy. --- conf/openshift/openshift.json | 10 +++++++ .../source/installation/index.rst | 1 - .../source/installation/openshift.rst | 24 --------------- .../source/installation/prep.rst | 5 ---- scripts/installer/install | 29 +++++++++++++++++++ 5 files changed, 39 insertions(+), 30 deletions(-) delete mode 100644 doc/sphinx-guides/source/installation/openshift.rst diff --git a/conf/openshift/openshift.json b/conf/openshift/openshift.json index 0f658ace99f..ec0442d401c 100644 --- a/conf/openshift/openshift.json +++ b/conf/openshift/openshift.json @@ -140,6 +140,11 @@ "value": "pgDatabaseValue" } ], + "resources": { + "limits": { + "memory": "256Mi" + } + }, "imagePullPolicy": "IfNotPresent", "securityContext": { "capabilities": {}, @@ -155,6 +160,11 @@ "protocol": "TCP" } ], + "resources": { + "limits": { + "memory": "256Mi" + } + }, "imagePullPolicy": "IfNotPresent", "securityContext": { "capabilities": {}, diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index 185fb94b4e3..b8423e77ae5 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -20,4 +20,3 @@ Installation Guide geoconnect shibboleth oauth2 - openshift diff --git a/doc/sphinx-guides/source/installation/openshift.rst b/doc/sphinx-guides/source/installation/openshift.rst deleted file mode 100644 index 71758cd23f9..00000000000 --- a/doc/sphinx-guides/source/installation/openshift.rst +++ /dev/null @@ -1,24 +0,0 @@ -================================= -Installing Dataverse on OpenShift -================================= - -.. contents:: |toctitle| - :local: - -Familiarize Yourself with OpenShift ------------------------------------ - -https://docs.openshift.com/online/getting_started/index.html - -Create OpenShift Account ------------------------- - -Create Project --------------- - -Import YAML/JSON ----------------- - -Upload ``conf/openshift/openshift.json``. - -FIXME: Getting "cannot create projects at the cluster scope" diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index 3b57c849f64..035106e62be 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -21,11 +21,6 @@ The National Data Service (NDS) is community-driven effort guided by the Nationa Please note that the version of Dataverse in NDS Labs Workbench may lag behind the latest release. Craig Willis from NDS Labs did an excellent job of adding Dataverse 4 to NDS Labs Workbench and the Dataverse team hopes to some day take over the creation of Docker images so the latest version of Dataverse can be evaluated in the workbench. -OpenShift (for Testing Only) -++++++++++++++++++++++++++++ - -If you would like to kick the tires on your own installation of Dataverse, you can install it for free on OpenShift Online, a platform as a service (PaaS) offering from Red Hat. For details, please see the :doc:`openshift` section. - Vagrant (for Testing Only) ++++++++++++++++++++++++++ diff --git a/scripts/installer/install b/scripts/installer/install index a620cb00eaa..f86883ec17f 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -967,6 +967,35 @@ if ( -e "/proc/meminfo" && open MEMINFO, "/proc/meminfo" ) { close MEMINFO; +# TODO: Figure out how to determine the amount of memory when running in Docker +# because we're wondering if Dataverse can run in the free OpenShift Online +# offering that only gives you 1 GB of memory. Obviously, if this is someone's +# first impression of Dataverse, we want to to run well! What if you try to +# ingest a large file or perform other memory-intensive operations? For more +# context, see https://github.com/IQSS/dataverse/issues/4040#issuecomment-331282286 + if ( -e "/sys/fs/cgroup/memory/memory.limit_in_bytes" && open CGROUPMEM, "/sys/fs/cgroup/memory/memory.limit_in_bytes" ) { + print "We must be running in Docker! Fancy!\n"; + while ( my $limitline = ) { + # The goal of this cgroup check is for + # "Setting the heap limit for Glassfish to 750MB" + # to change to some other value, based on memory available. + print "/sys/fs/cgroup/memory/memory.limit_in_bytes: $limitline\n"; + my $limit_in_kb = $limitline / 1024; + print "Docker limit_in_kb = $limit_in_kb but ignoring\n"; + # In openshift.json, notice how PostgreSQL and Solr have + # resources.limits.memory set to "256Mi". + # If you try to give the Dataverse/Glassfish container twice + # as much memory (512 MB) and allow $sys_mem_total to + # be set below, you should see the following: + # "Setting the heap limit for Glassfish to 192MB." + # FIXME: dataverse.war will not deploy with only 512 GB of memory. + # Again, the goal is 1 GB total (512MB + 256MB + 256MB) for + # Glassfish, PostgreSQL, and Solr to fit in the free OpenShift tier. + #print "setting sys_mem_total to: $limit_in_kb\n"; + #$sys_mem_total = $limit_in_kb; + } + close CGROUPMEM; + } } elsif ( -x "/usr/sbin/sysctl" ) { # MacOS X, probably... From e920e9f78cd02b019fac783d62d720d7a6fa8c86 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Sep 2017 13:37:50 -0400 Subject: [PATCH 20/25] cleanup #4040 --- conf/docker/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 3a9bf27b1e0..12f29b88304 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -10,6 +10,7 @@ echo Images will be pushed to Docker Hub with the tag $TAG docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf docker push iqss/dataverse-solr:$TAG # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. +# FIXME: Automate the building of dataverse.war and dvinstall.zip. Think about https://github.com/IQSS/dataverse/issues/3974 and https://github.com/IQSS/dataverse/pull/3975 cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war cp ../../scripts/installer/dvinstall.zip dataverse-glassfish cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish From 74767a69ec2b44c69638734ac324fb8d36d06f2b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Sep 2017 13:48:12 -0400 Subject: [PATCH 21/25] remove cruft from botch conflict resolution #4040 --- doc/sphinx-guides/source/installation/config.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index efecf643f12..3c39d41f873 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1209,4 +1209,3 @@ You can replace the default dataset metadata fields that are displayed above fil ``curl http://localhost:8080/api/admin/settings/:CustomDatasetSummaryFields -X PUT -d 'producer,subtitle,alternativeTitle'`` You have to put the datasetFieldType name attribute in the :CustomDatasetSummaryFields setting for this to work. ->>>>>>> develop From 4dd759b35b72507e316b1db10ae3bd759fd5b2f0 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 29 Sep 2017 14:23:08 -0400 Subject: [PATCH 22/25] let build script push arbitrary tags to Docker Hub #4040 --- conf/docker/build.sh | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 12f29b88304..a4828ba607f 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -1,11 +1,32 @@ #!/bin/sh # Creates images and pushes them to Docker Hub. -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -# FIXME: Make this script dynamic so you can switch the tag to the branch you're on or a tagged release. -TAG=kick-the-tires -# kick-the-tires should be relatively stable. Push to tags with branch names to iterate on the images. -#TAG=$GIT_BRANCH -echo Images will be pushed to Docker Hub with the tag $TAG +# The "kick-the-tires" tag should be relatively stable. No breaking changes. +# Push to custom tags or tags based on branch names to iterate on the images. +if [ -z "$1" ]; then + echo "No argument supplied. Please specify \"branch\" or \"custom my-custom-tag\" for experiments or \"stable\" if your change won't break anything." + exit 1 +fi + +if [ "$1" == 'branch' ]; then + echo "We'll push a tag to the branch you're on." + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + TAG=$GIT_BRANCH +elif [ "$1" == 'stable' ]; then + echo "We'll push a tag to the most stable tag (which isn't saying much!)." + TAG=kick-the-tires +elif [ "$1" == 'custom' ]; then + if [ -z "$1" ]; then + echo "You must provide a custom tag as the second argument." + exit 1 + else + echo "We'll push a custom tag." + TAG=$2 + fi +else + echo "Unexpected argument: $1. Exiting. Run with no arguments for help." + exit 1 +fi +echo Images will be pushed to Docker Hub with the tag \"$TAG\". # Use "conf" directory as context so we can copy schema.xml into Solr image. docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf docker push iqss/dataverse-solr:$TAG From 8a48b4f8aaeb0dae72e8da7f3a158c8eea1edcc7 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 11 Oct 2017 17:13:52 -0400 Subject: [PATCH 23/25] Documentation edits minishift #4040 --- .../source/developers/dev-environment.rst | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 7893d64edca..041722fbd6d 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -29,7 +29,7 @@ As a `Java Enterprise Edition Date: Thu, 12 Oct 2017 09:50:32 -0400 Subject: [PATCH 24/25] doc fix: all containers run as non-root now #4040 --- .../source/developers/dev-environment.rst | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 041722fbd6d..853c188601d 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -356,7 +356,7 @@ Start Minishift ``minishift start --vm-driver=virtualbox`` Make the OpenShift Client Binary (oc) Executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``eval $(minishift oc-env)`` @@ -369,19 +369,6 @@ Note that if you just installed Minishift, you are probably logged in already, b Use "developer" as the username and a couple characters as the password. -Allow Containers to Run as Root in Minishift -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For now we're allowing containers to run as root. Until the images are fixed to run as non-root, run the following command: - -``oc adm policy add-scc-to-user anyuid -z default --as system:admin`` - -FIXME: Eventually, we should create containers that don't require root. When we do, run the following command to ensure Dataverse still runs on Minishift after you've stopped allowing containers to run as root: - -``oc adm policy remove-scc-from-user anyuid -z default --as system:admin`` - -For more information on improving Docker images to run as non-root, see "Support Arbitrary User IDs" at https://docs.openshift.org/latest/creating_images/guidelines.html#openshift-origin-specific-guidelines - Create a Minishift Project ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -428,7 +415,7 @@ First, check the IP address of your minishift cluster. If this differs from the ``minishift ip`` -The following curl command is expected to fail until you "expose" the HTTP service. +The following curl command is expected to fail until you "expose" the HTTP service. Please note that the IP address may be different. ``curl http://dataverse-glassfish-service-project1.192.168.99.100.nip.io/api/info/version`` @@ -439,7 +426,7 @@ Expose the Dataverse web service: Make Sure the Dataverse API is Working ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This should show a version number: +This should show a version number but please note that the IP address may be different: ``curl http://dataverse-glassfish-service-project1.192.168.99.100.nip.io/api/info/version`` @@ -450,7 +437,7 @@ Log into Minishift and Visit Dataverse in your Browser - username: developer - password: developer -Visit https://192.168.99.100:8443/console/project/project1/browse/routes and click http://dataverse-glassfish-service-project1.192.168.99.100.nip.io/ or whatever is shows under "Routes External Traffic". This assumes you named your project ``project1``. +Visit https://192.168.99.100:8443/console/project/project1/browse/routes and click http://dataverse-glassfish-service-project1.192.168.99.100.nip.io/ or whatever is shows under "Routes External Traffic" (the IP address may be different). This assumes you named your project ``project1``. You should be able to log in with username "dataverseAdmin" and password "admin". @@ -466,6 +453,20 @@ Making Changes If you're interested in using Minishift for development and want to change the Dataverse code, you will need to get set up to create Docker images based on your changes and push them to a Docker registry such as Docker Hub. See the section below on Docker for details. +Runnning Containers to Run as Root in Minishift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is **not** recommended to run containers as root in Minishift because for security reasons OpenShift doesn't support running containers as root. However, it's good to know how to allow containers to run as root in case you need to work on a Docker image to make it run as non-root. + +For more information on improving Docker images to run as non-root, see "Support Arbitrary User IDs" at https://docs.openshift.org/latest/creating_images/guidelines.html#openshift-origin-specific-guidelines + +Let's say you have a container that you suspect works fine when it runs as root. You want to see it working as-is before you start hacking on the Dockerfile and entrypoint file. You can configure Minishift to allow containers to run as root with this command: + +``oc adm policy add-scc-to-user anyuid -z default --as system:admin`` + +Once you are done testing you can revert Minishift back to not allowing containers to run as root with this command: + +``oc adm policy remove-scc-from-user anyuid -z default --as system:admin`` Minishift Resources ~~~~~~~~~~~~~~~~~~~ From 1e7fb53889539b70a040270f2d62bcb689e1b5fa Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Tue, 17 Oct 2017 12:23:27 -0400 Subject: [PATCH 25/25] File path clarity#4040 --- doc/sphinx-guides/source/developers/dev-environment.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 853c188601d..73cf0925549 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -377,7 +377,7 @@ Create a Minishift Project Create a Dataverse App within the Minishift Project ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running this command will download images from Docker Hub and use them to create a Dataverse application. +Run this command from inside the vanilla Dataverse application to download images from Docker Hub and use them to create a Dataverse Minishift application. Alternatively, the ``openshift.json`` file can be downloaded directly from our github repo. ``oc new-app conf/openshift/openshift.json``