From bc3c765906ff8701ede8211fe2c9004c53156409 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Tue, 10 Dec 2019 12:41:05 +0000 Subject: [PATCH 01/14] try to build image that will auto populate the container with some data --- service/scripts/db-create-psql.sh | 19 ++++--------------- service/scripts/db-create-psql.sh.bak | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) create mode 100755 service/scripts/db-create-psql.sh.bak diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 96a146f3..06295abb 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,17 +1,6 @@ #!/bin/bash -psql -d template1 -c 'create extension hstore;' - -psql < Date: Tue, 10 Dec 2019 12:53:40 +0000 Subject: [PATCH 02/14] typo sql instead of db file --- service/scripts/db-create-psql.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 06295abb..28e917e5 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,6 +1,5 @@ #!/bin/bash -git clone https://github.com/ddelpiano/scidash-artifacts -cd scidash-artifacts/database +cd /app/scidash-artifacts/database gunzip `ls | head -n 1` -pg_restore --clean -d scidash `ls *sql | head -n 1` +pg_restore --clean -d scidash `ls *db | head -n 1` From ec01b5e830d801875bb7da549d32fd7661009731 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Tue, 10 Dec 2019 12:58:02 +0000 Subject: [PATCH 03/14] typo part 2 --- service/scripts/db-create-psql.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 28e917e5..43ae6550 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,5 +1,6 @@ #!/bin/bash -cd /app/scidash-artifacts/database +git clone https://github.com/ddelpiano/scidash-artifacts +cd scidash-artifacts/database gunzip `ls | head -n 1` pg_restore --clean -d scidash `ls *db | head -n 1` From ac7817a749c98927b16818a29c42e27bc1464def Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Tue, 10 Dec 2019 12:59:50 +0000 Subject: [PATCH 04/14] permission issue --- service/scripts/db-create-psql.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 43ae6550..0c9a6a0c 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,6 +1,8 @@ #!/bin/bash +cd /tmp git clone https://github.com/ddelpiano/scidash-artifacts cd scidash-artifacts/database gunzip `ls | head -n 1` pg_restore --clean -d scidash `ls *db | head -n 1` +cd - From 2e7bf07f025f3afaf3037438b48c5f72516eb3f1 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Tue, 10 Dec 2019 21:16:19 -0500 Subject: [PATCH 05/14] feat: split scidash from virgo see issue 400 --- requirements.in | 1 + requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 755ab266..04185443 100644 --- a/requirements.in +++ b/requirements.in @@ -13,3 +13,4 @@ celery[redis] django-celery-beat django-celery-results django-db-logger +git+git://github.com/ddelpiano/neuronunit@4.0.0#egg=neuronunit diff --git a/requirements.txt b/requirements.txt index 746f70bf..813c6d41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile +# pip-compile # amqp==2.4.2 # via kombu asgiref==2.3.2 # via channels @@ -27,6 +27,7 @@ django==1.11.7 djangorestframework-jwt==1.11.0 djangorestframework==3.7.1 drf-writable-nested==0.5.1 +git+git://github.com/ddelpiano/neuronunit@4.0.0#egg=neuronunit hyperlink==18.0.0 # via twisted idna==2.8 # via hyperlink incremental==17.5.0 # via twisted From 3c8de7eff2714d114f1dd23c372e9d42512ac642 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Wed, 11 Dec 2019 21:06:50 -0500 Subject: [PATCH 06/14] fix: typo in privileged --- service/deployment/docker-compose.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/service/deployment/docker-compose.yml b/service/deployment/docker-compose.yml index bcb082aa..99ed0bcf 100644 --- a/service/deployment/docker-compose.yml +++ b/service/deployment/docker-compose.yml @@ -11,14 +11,22 @@ services: - "5432" volumes: - ./database:/var/lib/postgresql/ - scidash: - image: metacell/scidash:latest + scidash-virgo: + image: metacell/scidash_virgo:v1.0.0 + expose: + - "8080" mem_reservation: 5120m mem_limit: 10240m - priviliged: true + privileged: true shm_size: 512M + depends_on: + - scidash-redis + - scidash-postgres + scidash: + image: metacell/scidash:latest expose: - "8000" depends_on: - scidash-redis - scidash-postgres + - scidash-virgo From a0add295e80df27ff5b45270472b8bbcf978c1d9 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Wed, 11 Dec 2019 21:08:45 -0500 Subject: [PATCH 07/14] feat: split vigro and scidash docker images --- service/docker/Dockerfile-scidash | 32 ++++++++++----------- service/docker/Dockerfile-virgo | 5 ++-- service/dotenv/env-docker | 10 +++++-- service/dotenv/scidash_env | 12 ++++++-- service/geppetto/GeppettoConfiguration.json | 2 +- service/geppetto/tomcat-server.xml | 2 +- 6 files changed, 37 insertions(+), 26 deletions(-) diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index bcdc6ff3..93aa6cbf 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -1,6 +1,6 @@ -FROM metacell/scidash_virgo:v1.0.0 +FROM python:3.6 -USER root +USER root # BUILD VARIABLES ARG ROOT=/ @@ -9,32 +9,31 @@ ARG NRN_SYMLINK=/Applications/NEURON-7.6/nrn/ ARG DOTENV_FILE=env-docker ARG STATIC_DIR=$APP_DIR/scidash/static ARG GEPPETTO_DIR=$STATIC_DIR/org.geppetto.frontend/src/main/webapp -ARG SCIDASH_BRANCH=4.0.1 +ARG SCIDASH_BRANCH=feature/400 ENV SERVER_HOME $APP_DIR/virgo-tomcat-server -WORKDIR $ROOT +RUN useradd -ms /bin/bash developer +ENV HOME /home/developer -#COPYING PROJECT -WORKDIR $APP_DIR -RUN git clone -b $SCIDASH_BRANCH https://github.com/MetaCell/scidash +WORKDIR $ROOT # INSTALLING REQUIREMENTS -RUN apt-get install -y curl wget lsof +RUN apt-get install -y curl wget RUN curl -sL https://deb.nodesource.com/setup_9.x | bash RUN apt-get update && apt-get -y install nodejs +RUN curl https://www.npmjs.com/install.sh | sh + +#COPYING PROJECT +WORKDIR $APP_DIR +RUN git clone -b $SCIDASH_BRANCH https://github.com/MetaCell/scidash -WORKDIR /nrn -RUN chown -R developer ./ WORKDIR $APP_DIR/scidash -RUN pip install virtualenv +RUN pip install virtualenv RUN virtualenv venv-py -p python3.6 -WORKDIR /usr/local/nrn/src/nrnpython -RUN python setup.py install WORKDIR $APP_DIR/scidash RUN make install-backend-with-env -RUN make install-sciunit-neuronunit RUN make install-frontend WORKDIR $GEPPETTO_DIR @@ -42,8 +41,8 @@ RUN ls -la RUN npm run build-dev-noTest WORKDIR $APP_DIR/scidash -RUN rm -rf .git &&\ - rm -rf static/org.geppetto.frontend/.git &&\ +RUN rm -rf .git &&\ + rm -rf static/org.geppetto.frontend/.git &&\ rm -rf static/org.geppetto.frontend/extension/geppetto-scidash/.git &&\ rm -rf sciunit/.git &&\ rm -rf neuronunit/.git @@ -51,5 +50,4 @@ RUN chown -R developer ./ RUN cp ./service/dotenv/scidash_env .env USER developer - CMD ./service/scripts/run.sh diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index d65d2bc7..a8b0f34d 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -14,6 +14,7 @@ ARG SCIDASH_BRANCH=development WORKDIR / RUN useradd -ms /bin/bash developer ENV HOME /home/developer +RUN printf "\ndeb http://deb.debian.org/debian/ sid main\n" >> /etc/apt/sources.list RUN apt-get update && \ apt-get install -y openjdk-8-jdk && \ apt-get install -y ant && \ @@ -80,14 +81,14 @@ RUN ls -l /usr/local/nrn/ # -== INSTALL VIRGO ==- RUN mkdir $VIRGO_DIR -RUN apt-get install curl +RUN apt-get -y install curl RUN curl -L 'http://www.eclipse.org/downloads/download.php?file=/virgo/release/VP/3.7.2.RELEASE/virgo-tomcat-server-3.7.2.RELEASE.zip&r=1' > virgo.zip RUN unzip virgo.zip RUN cp -r ./virgo-tomcat-server-3.7.2.RELEASE/* $VIRGO_DIR RUN rm $VIRGO_DIR/configuration/java-server.profile RUN cp /git/scidash/service/geppetto/java-server.profile $VIRGO_DIR/configuration/ RUN rm $VIRGO_DIR/configuration/tomcat-server.xml -RUN cp /git/scidash/service/geppetto/tomcat-server.xml $VIRGO_DIR/configuration/ +RUN cat /git/scidash/service/geppetto/tomcat-server.xml | sed 's/127.0.0.1/0.0.0.0/' > $VIRGO_DIR/configuration/tomcat-server.xml RUN chmod u+x $VIRGO_DIR/bin/*.sh ENV SERVER_HOME $VIRGO_DIR RUN chmod 777 -R $VIRGO_DIR diff --git a/service/dotenv/env-docker b/service/dotenv/env-docker index 86631363..3fa461d6 100644 --- a/service/dotenv/env-docker +++ b/service/dotenv/env-docker @@ -12,6 +12,12 @@ STATIC_DIR=static REDIS_URL=redis://scidash-redis:6379 -GEPPETTO_SERVLET_URL='ws://localhost:8080/org.geppetto.frontend/GeppettoServlet' -GEPPETTO_BASE_URL='http://localhost:8080/org.geppetto.frontend/geppetto' +# for localhost +# GEPPETTO_HOST=localhost +# for docker +GEPPETTO_HOST=scidash-virgo +GEPPETTO_PORT=8080 +GEPPETTO_SERVLET_URL=ws://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/GeppettoServlet +GEPPETTO_BASE_URL=http://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/geppetto + BASE_PROJECT_FILES_HOST='http://localhost:8000/static/projects/' diff --git a/service/dotenv/scidash_env b/service/dotenv/scidash_env index 66526853..e3de7f16 100644 --- a/service/dotenv/scidash_env +++ b/service/dotenv/scidash_env @@ -12,6 +12,12 @@ REDIS_URL=redis://scidash-redis:6379 STATIC_URL='/static/' STATIC_DIR='static' -# GEPPETTO_SERVLET_URL='ws://scidash-virgo:8080/org.geppetto.frontend/GeppettoServlet' -# GEPPETTO_BASE_URL='http://scidash-virgo:8080/org.geppetto.frontend/geppetto' -# BASE_PROJECT_FILES_HOST='http://scidash-virgo:8000/static/projects/' +# for localhost +# GEPPETTO_HOST=localhost +# for docker +GEPPETTO_HOST=scidash-virgo +GEPPETTO_PORT=8080 +GEPPETTO_SERVLET_URL=ws://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/GeppettoServlet +GEPPETTO_BASE_URL=http://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/geppetto + +BASE_PROJECT_FILES_HOST='http://localhost:8000/static/projects/' diff --git a/service/geppetto/GeppettoConfiguration.json b/service/geppetto/GeppettoConfiguration.json index 5b8a1b8b..e336de2e 100644 --- a/service/geppetto/GeppettoConfiguration.json +++ b/service/geppetto/GeppettoConfiguration.json @@ -1,7 +1,7 @@ { "_README": "http://docs.geppetto.org/en/latest/build.html", "contextPath": "org.geppetto.frontend", - "useSsl": true, + "useSsl": false, "embedded": false, "embedderURL": ["/"], "rootRedirect": "", diff --git a/service/geppetto/tomcat-server.xml b/service/geppetto/tomcat-server.xml index 53aabe37..8e88d374 100644 --- a/service/geppetto/tomcat-server.xml +++ b/service/geppetto/tomcat-server.xml @@ -27,7 +27,7 @@ - From 5bd6448632cd1026cf052db4e2bc53098b7c031f Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 09:56:09 -0500 Subject: [PATCH 08/14] fix: don't run make run-virgo-staging in the scidash server --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fe239f2b..ddc75484 100644 --- a/Makefile +++ b/Makefile @@ -58,8 +58,7 @@ run-dev: migrate run-staging: migrate make run-django-staging & \ make run-celery & \ - make run-celery-beat & \ - make run-virgo-staging + make run-celery-beat django-migrate: migrations migrate From 481e1104ca50ebf9c27367872532f0675aa801f6 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 10:35:22 -0500 Subject: [PATCH 09/14] fix: docker compose to expose to the host port 8000 --- service/deployment/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/service/deployment/docker-compose.yml b/service/deployment/docker-compose.yml index 99ed0bcf..1d0810c8 100644 --- a/service/deployment/docker-compose.yml +++ b/service/deployment/docker-compose.yml @@ -4,17 +4,17 @@ services: scidash-redis: image: redis expose: - - "6379" + - 6379 scidash-postgres: image: metacell/scidash_db:latest expose: - - "5432" + - 5432 volumes: - ./database:/var/lib/postgresql/ scidash-virgo: image: metacell/scidash_virgo:v1.0.0 expose: - - "8080" + - 8080 mem_reservation: 5120m mem_limit: 10240m privileged: true @@ -24,8 +24,8 @@ services: - scidash-postgres scidash: image: metacell/scidash:latest - expose: - - "8000" + ports: + - 8000:8000 depends_on: - scidash-redis - scidash-postgres From 3d6806bdb81f62635cccceed8a688a4a518f1315 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 10:36:38 -0500 Subject: [PATCH 10/14] fix: add missing migration to git repo --- .../migrations/0046_auto_20191211_0156.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 scidash/sciunittests/migrations/0046_auto_20191211_0156.py diff --git a/scidash/sciunittests/migrations/0046_auto_20191211_0156.py b/scidash/sciunittests/migrations/0046_auto_20191211_0156.py new file mode 100644 index 00000000..7e28b229 --- /dev/null +++ b/scidash/sciunittests/migrations/0046_auto_20191211_0156.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-12-11 01:56 +from __future__ import unicode_literals + +import datetime +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import scidash.sciunittests.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0045_auto_20190712_1041'), + ] + + operations = [ + migrations.AddField( + model_name='testclass', + name='default_params', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, encoder=scidash.sciunittests.models.JSONEncoder, null=True), + ), + migrations.AlterField( + model_name='scoreinstance', + name='timestamp', + field=models.DateTimeField(default=datetime.datetime.today), + ), + migrations.AlterField( + model_name='testinstance', + name='timestamp', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='testsuite', + name='timestamp', + field=models.DateTimeField(default=datetime.datetime.today), + ), + ] From e6d263cdc0f0d7275eab71bf3456a314303f7815 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 14:35:31 -0500 Subject: [PATCH 11/14] fix: issue with incorrect url path to load the p.json file from --- service/dotenv/env-docker | 2 +- service/dotenv/scidash_env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/service/dotenv/env-docker b/service/dotenv/env-docker index 3fa461d6..9965aa72 100644 --- a/service/dotenv/env-docker +++ b/service/dotenv/env-docker @@ -20,4 +20,4 @@ GEPPETTO_PORT=8080 GEPPETTO_SERVLET_URL=ws://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/GeppettoServlet GEPPETTO_BASE_URL=http://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/geppetto -BASE_PROJECT_FILES_HOST='http://localhost:8000/static/projects/' +BASE_PROJECT_FILES_HOST='http://scidash:8000/static/projects/' diff --git a/service/dotenv/scidash_env b/service/dotenv/scidash_env index e3de7f16..9e12c06d 100644 --- a/service/dotenv/scidash_env +++ b/service/dotenv/scidash_env @@ -20,4 +20,4 @@ GEPPETTO_PORT=8080 GEPPETTO_SERVLET_URL=ws://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/GeppettoServlet GEPPETTO_BASE_URL=http://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/geppetto -BASE_PROJECT_FILES_HOST='http://localhost:8000/static/projects/' +BASE_PROJECT_FILES_HOST='http://scidash:8000/static/projects/' From f9a7a925c4b1ac46d9bf6ef24b92890b8124f71d Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 15:09:03 -0500 Subject: [PATCH 12/14] fix: renames 0046_auto_20191211_0156.py -> 0046_auto_20190827_0221.py --- .../{0046_auto_20191211_0156.py => 0046_auto_20190827_0221.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scidash/sciunittests/migrations/{0046_auto_20191211_0156.py => 0046_auto_20190827_0221.py} (100%) diff --git a/scidash/sciunittests/migrations/0046_auto_20191211_0156.py b/scidash/sciunittests/migrations/0046_auto_20190827_0221.py similarity index 100% rename from scidash/sciunittests/migrations/0046_auto_20191211_0156.py rename to scidash/sciunittests/migrations/0046_auto_20190827_0221.py From a8fe4db334725458b99b4298dab45eafd20d07fb Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 15:58:51 -0500 Subject: [PATCH 13/14] fix: added --pidfile= to celery beat to ignore pidfie creation --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ddc75484..406b7bc2 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ run-celery: celery -A scidash.main worker -l info --concurrency=5 --maxtasksperchild=20 run-celery-beat: - celery -A scidash.main beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler + celery -A scidash.main beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler --pidfile= run-virgo-staging: /bin/bash /opt/virgo/bin/startup.sh From 3e764c6cc76e910315820cd8d7530725d8a5a763 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Tue, 17 Dec 2019 21:09:37 +0100 Subject: [PATCH 14/14] fix: initial timestamp from/to, search max settings.SCIDASH_INITIAL_SEARCH_QUARTERS quarters back, if not found show settings.SCIDASH_INITIAL_SEARCH_QUARTERS quarters back period --- scidash/sciunittests/views.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index a70d419c..99e91b50 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -17,16 +17,10 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - three_month_ago = current_date_iso - three_month_period - six_month_ago = three_month_ago - three_month_period - nine_month_ago = six_month_ago - three_month_period - tvelwe_month_ago = nine_month_ago - three_month_period - acceptable_period = None - for period in [ - three_month_ago, six_month_ago, nine_month_ago, tvelwe_month_ago - ]: + for quarter in range(1, s.SCIDASH_INITIAL_SEARCH_QUARTERS + 1): + period = current_date_iso - quarter*three_month_period count = ScoreInstance.objects.filter( timestamp__gte=period, timestamp__lt=current_date_iso ).count() @@ -36,7 +30,7 @@ def get(self, request, *args, **kwargs): break if acceptable_period is None: - acceptable_period = tvelwe_month_ago + acceptable_period = period return Response( {