From 3c7bc681046839e87682ac442287a7593df6d625 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Tue, 10 Dec 2019 15:45:01 +0000 Subject: [PATCH 001/235] temporary fix for the db --- service/scripts/db-create-psql.sh | 23 ++++++++++++++++++++--- service/scripts/db-create-psql.sh.bak | 17 ----------------- 2 files changed, 20 insertions(+), 20 deletions(-) delete mode 100755 service/scripts/db-create-psql.sh.bak diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 0c9a6a0c..d266e2f6 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,8 +1,25 @@ #!/bin/bash +su -c "psql -d template1 -c 'create extension hstore;' + +psql < Date: Tue, 10 Dec 2019 15:56:52 +0000 Subject: [PATCH 002/235] removed su internal --- service/scripts/db-create-psql.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index d266e2f6..f45290a2 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -16,7 +16,7 @@ GRANT ALL PRIVILEGES ON DATABASE scidash TO scidash_admin; ALTER USER scidash_admin CREATEDB EOF -su postgres +sleep 5 cd /tmp git clone https://github.com/ddelpiano/scidash-artifacts cd scidash-artifacts/database From 247d799e1fbb6a9cce2876edf83fbc267890bb04 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Tue, 10 Dec 2019 17:29:12 +0000 Subject: [PATCH 003/235] checking user permissions --- service/scripts/db-create-psql.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index f45290a2..f1e78a0f 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,5 +1,6 @@ #!/bin/bash +whoami su -c "psql -d template1 -c 'create extension hstore;' psql < Date: Wed, 11 Dec 2019 12:43:10 +0000 Subject: [PATCH 004/235] testing 1 --- service/docker/Dockerfile-postgres | 7 +++++-- service/docker/build_database.sh | 4 ++-- service/scripts/db-create-psql.sh | 12 +++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index aca1f1c5..2adf9140 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -13,7 +13,10 @@ RUN apt-get update && \ RUN mkdir $APP_DIR WORKDIR $APP_DIR -RUN git clone -b deployment https://github.com/MetaCell/scidash +RUN echo "nother test5" +RUN chmod 777 /tmp +RUN git clone -b feature/393 https://github.com/MetaCell/scidash RUN cp scidash/service/database/backup_database.sh /backup_database.sh RUN chmod 0644 /backup_database.sh -RUN cp scidash/service/scripts/db-create-psql.sh /docker-entrypoint-initdb.d/db-create-psql.sh \ No newline at end of file +RUN cp scidash/service/scripts/db-create-psql.sh /docker-entrypoint-initdb.d/db-create-psql.sh +RUN whoami diff --git a/service/docker/build_database.sh b/service/docker/build_database.sh index cd6191c4..dcfe5d17 100755 --- a/service/docker/build_database.sh +++ b/service/docker/build_database.sh @@ -6,9 +6,9 @@ while true; do case $yn in [Yy]* ) read -p "Please type the tag you want to use for this build (default will use the latest and overwrite this). [latest/user_input] > " tag; if [[ -z "$tag" ]]; then - docker build --no-cache -f Dockerfile-postgres -t metacell/scidash_db:latest . + docker build -f Dockerfile-postgres -t metacell/scidash_db:latest . else - docker build --no-chace -f Dockerfile-postgres -t metacell/scidash_db:$tag . + docker build -f Dockerfile-postgres -t metacell/scidash_db:$tag . fi break;; [Nn]* ) exit;; diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index f1e78a0f..2ecbccb7 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,7 +1,8 @@ #!/bin/bash -whoami -su -c "psql -d template1 -c 'create extension hstore;' +echo "step 1" + +psql -d template1 -c 'create extension hstore;' psql < Date: Wed, 11 Dec 2019 12:43:47 +0000 Subject: [PATCH 005/235] removed unnecessary step in docker --- service/docker/Dockerfile-postgres | 1 - 1 file changed, 1 deletion(-) diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index 2adf9140..47ee2a5e 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -13,7 +13,6 @@ RUN apt-get update && \ RUN mkdir $APP_DIR WORKDIR $APP_DIR -RUN echo "nother test5" RUN chmod 777 /tmp RUN git clone -b feature/393 https://github.com/MetaCell/scidash RUN cp scidash/service/database/backup_database.sh /backup_database.sh From 400d5f91317a7bb5f4a985b67e64645dfc1aa19a Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Wed, 11 Dec 2019 13:20:35 +0000 Subject: [PATCH 006/235] suppressed bash exit 1 --- service/scripts/db-create-psql.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 2ecbccb7..92c4628b 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -1,6 +1,4 @@ -#!/bin/bash - -echo "step 1" +#!/bin/sh -x psql -d template1 -c 'create extension hstore;' @@ -18,11 +16,9 @@ GRANT ALL PRIVILEGES ON DATABASE scidash TO scidash_admin; ALTER USER scidash_admin CREATEDB EOF -echo " step 2" sleep 5 cd /tmp git clone https://github.com/ddelpiano/scidash-artifacts -echo "step 3" cd scidash-artifacts/database gunzip `ls | head -n 1` -pg_restore --clean -d scidash `ls *db | head -n 1` +pg_restore --clean -d scidash `ls *db | head -n 1` || true From 0becfd4058fc35099129ce9669dcd2e2f126825a Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Mon, 16 Dec 2019 05:46:38 +0000 Subject: [PATCH 007/235] travis scidash CI --- Makefile | 4 +- docker-compose.yml | 16 ---- service/deployment/docker-compose.yml | 5 +- service/docker/Dockerfile-postgres | 7 +- service/docker/Dockerfile-scidash | 28 +++--- service/docker/Dockerfile-virgo | 85 ++++++++++------- service/docker/build_database.sh | 11 ++- service/docker/copy.sh | 58 ++++++++++++ service/scripts/install-backend.sh | 35 +++++-- service/scripts/install-frontend.sh | 45 ++++++++- service/scripts/trigger-travis.sh | 126 ++++++++++++++++++++++++++ 11 files changed, 340 insertions(+), 80 deletions(-) delete mode 100644 docker-compose.yml create mode 100755 service/docker/copy.sh create mode 100755 service/scripts/trigger-travis.sh diff --git a/Makefile b/Makefile index 406b7bc2..c5d7f6de 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ install-backend-with-env: @echo "===========================" @echo "= Install backend =" @echo "===========================" - @./service/scripts/install-backend.sh -v + @./service/scripts/install-backend.sh -v $(ARGS) create-db: @echo "===========================" @@ -140,3 +140,5 @@ push-scidash-db: @echo "= Push scidash db image =" @echo "===========================" @./service/scripts/push-image-scidash-db.sh +%: + @: diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 0a0203f5..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '2' - -services: - scidash-redis: - image: redis - scidash-postgres: - image: metacell/scidash-db - ports: - - 5432:5432 - scidash: - image: metacell/scidash:latest - ports: - - 80:8000 - depends_on: - - scidash-redis - - scidash-postgres diff --git a/service/deployment/docker-compose.yml b/service/deployment/docker-compose.yml index 1d0810c8..1ccf5c6b 100644 --- a/service/deployment/docker-compose.yml +++ b/service/deployment/docker-compose.yml @@ -7,12 +7,14 @@ services: - 6379 scidash-postgres: image: metacell/scidash_db:latest + container_name: scidash_db expose: - 5432 volumes: - ./database:/var/lib/postgresql/ scidash-virgo: - image: metacell/scidash_virgo:v1.0.0 + image: metacell/scidash_virgo:latest + container_name: scidash_virgo expose: - 8080 mem_reservation: 5120m @@ -24,6 +26,7 @@ services: - scidash-postgres scidash: image: metacell/scidash:latest + container_name: scidash ports: - 8000:8000 depends_on: diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index 47ee2a5e..d6f08219 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -2,6 +2,9 @@ FROM postgres:9.4 ARG ROOT=/ ARG APP_DIR=/app +ARG SCIDASH_BRANCH=development +ARG originBranch=development +ARG defaultBranch=development WORKDIR $ROOT @@ -13,9 +16,9 @@ RUN apt-get update && \ RUN mkdir $APP_DIR WORKDIR $APP_DIR +COPY ./copy.sh $APP_DIR/ RUN chmod 777 /tmp -RUN git clone -b feature/393 https://github.com/MetaCell/scidash +RUN ./copy.sh https://github.com/MetaCell/scidash "${SCIDASH_BRANCH}" "${originBranch}" "${defaultBranch}" RUN cp scidash/service/database/backup_database.sh /backup_database.sh RUN chmod 0644 /backup_database.sh RUN cp scidash/service/scripts/db-create-psql.sh /docker-entrypoint-initdb.d/db-create-psql.sh -RUN whoami diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index 93aa6cbf..d92d3a8f 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -9,7 +9,10 @@ ARG NRN_SYMLINK=/Applications/NEURON-7.6/nrn/ ARG DOTENV_FILE=env-docker ARG STATIC_DIR=$APP_DIR/scidash/static ARG GEPPETTO_DIR=$STATIC_DIR/org.geppetto.frontend/src/main/webapp -ARG SCIDASH_BRANCH=feature/400 +ARG targetBranch=development +ARG originBranch=development +ARG defaultBranch=development + ENV SERVER_HOME $APP_DIR/virgo-tomcat-server RUN useradd -ms /bin/bash developer @@ -22,32 +25,31 @@ RUN apt-get install -y curl wget RUN curl -sL https://deb.nodesource.com/setup_9.x | bash RUN apt-get update && apt-get -y install nodejs RUN curl https://www.npmjs.com/install.sh | sh +RUN pip install virtualenv +RUN mkdir $APP_DIR +RUN chown developer $APP_DIR +RUN chown -R developer /home/developer/.config + +USER developer -#COPYING PROJECT +# COPYING PROJECT +COPY ./copy.sh $APP_DIR WORKDIR $APP_DIR -RUN git clone -b $SCIDASH_BRANCH https://github.com/MetaCell/scidash +RUN $APP_DIR/copy.sh https://github.com/Metacell/scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" WORKDIR $APP_DIR/scidash -RUN pip install virtualenv RUN virtualenv venv-py -p python3.6 WORKDIR $APP_DIR/scidash -RUN make install-backend-with-env -RUN make install-frontend +RUN make ARGS="-b $targetBranch" install-backend-with-env +RUN make ARGS="-b $targetBranch" install-frontend WORKDIR $GEPPETTO_DIR RUN ls -la RUN npm run build-dev-noTest WORKDIR $APP_DIR/scidash -RUN rm -rf .git &&\ - rm -rf static/org.geppetto.frontend/.git &&\ - rm -rf static/org.geppetto.frontend/extension/geppetto-scidash/.git &&\ - rm -rf sciunit/.git &&\ - rm -rf neuronunit/.git -RUN chown -R developer ./ RUN cp ./service/dotenv/scidash_env .env -USER developer CMD ./service/scripts/run.sh diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index a8b0f34d..19c4f861 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -4,11 +4,16 @@ ARG APP_DIR=/app ARG SOURCES_DIR=$APP_DIR/sources ARG VIRGO_DIR=/opt/virgo ARG SCIDASH_REPO_FOLDER=/git -ARG GEPPETTO_REPO=http://github.com/openworm/org.geppetto +ARG GEPPETTO_REPO=http://github.com/openworm/org.geppetto.git ARG ALPHA_TAG=v0.4.2-alpha ARG SIMFLAG_TAG=v0.4.2-simflag ARG SCIDASH_BRANCH=development +ARG alphaBranch=v0.4.2-alpha +ARG simflagBranch=v0.4.2-simflag +ARG targetBranch=development +ARG originBranch=development +ARG defaultBranch=development # -== Install OpenJDK and certificates ==- WORKDIR / @@ -34,6 +39,7 @@ RUN mkdir $SCIDASH_REPO_FOLDER WORKDIR $APP_DIR RUN mkdir $SOURCES_DIR WORKDIR $SCIDASH_REPO_FOLDER +COPY ./copy.sh $APP_DIR # -== Download SCIDASH repo ==- @@ -92,48 +98,57 @@ RUN cat /git/scidash/service/geppetto/tomcat-server.xml | sed 's/127.0.0.1/0.0.0 RUN chmod u+x $VIRGO_DIR/bin/*.sh ENV SERVER_HOME $VIRGO_DIR RUN chmod 777 -R $VIRGO_DIR -RUN mkdir rm $VIRGO_DIR/repository/usr +RUN mkdir $VIRGO_DIR/repository/usr RUN mvn --version # -== INSTALL GEPPETTO ==- WORKDIR $SOURCES_DIR -RUN git clone $GEPPETTO_REPO -b $ALPHA_TAG -WORKDIR $SOURCES_DIR/org.geppetto/utilities/source_setup -RUN rm $SOURCES_DIR/org.geppetto/utilities/source_setup/config.json +RUN $APP_DIR/copy.sh http://github.com/openworm/org.geppetto.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" RUN cp /git/scidash/service/geppetto/config.json $SOURCES_DIR/org.geppetto/utilities/source_setup RUN cp /git/scidash/service/geppetto/setup.py $SOURCES_DIR/org.geppetto/utilities/source_setup RUN cp /git/scidash/service/geppetto/geppetto.plan $SOURCES_DIR/org.geppetto/ -RUN python2 setup.py - -WORKDIR $SOURCES_DIR/org.geppetto.core -RUN git checkout $SIMFLAG_TAG -RUN rm ./src/main/java/META-INF/spring/app-config.xml -RUN cp /git/scidash/service/geppetto/core/app-config.xml ./src/main/java/META-INF/spring - -WORKDIR $SOURCES_DIR/org.geppetto.frontend -RUN git checkout $SIMFLAG_TAG -RUN rm ./src/main/webapp/WEB-INF/spring/app-config.xml -RUN cp /git/scidash/service/geppetto/frontend/app-config.xml ./src/main/webapp/WEB-INF/spring/ - -WORKDIR $SOURCES_DIR/org.geppetto.model -RUN git checkout $ALPHA_TAG - -WORKDIR $SOURCES_DIR/org.geppetto.model.neuroml -RUN git checkout $ALPHA_TAG - -WORKDIR $SOURCES_DIR/org.geppetto.simulation -RUN git checkout $SIMFLAG_TAG - -WORKDIR $SOURCES_DIR/org.geppetto.simulator.external -RUN git checkout $ALPHA_TAG - -WORKDIR $SOURCES_DIR/org.geppetto.simulator.scidash -RUN git checkout development -# RUN rm ./src/main/java/META-INF/spring/app-config.xml -# RUN cp /git/scidash/service/geppetto/simulator.scidash/app-config.xml ./src/main/java/META-INF/spring -WORKDIR $SOURCES_DIR/org.geppetto -RUN mvn -Dhttps.protocols=TLSv1.2 -DcontextPath=org.geppetto.frontend -DuseSsl=false -DskipTests install +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" &&\ + cd org.geppetto.model &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.model\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ + +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.core.git "${simflagBranch}" "${simflagBranch}" "${simflagBranch}" &&\ + cd org.geppetto.core && cp /git/scidash/service/geppetto/core/app-config.xml ./src/main/java/META-INF/spring &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.core\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ + +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.neuroml.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" &&\ + cd org.geppetto.model.neuroml &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.model.neuroml\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ + +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulation.git "${simflagBranch}" "${simflagBranch}" "${simflagBranch}" &&\ + cd org.geppetto.simulation &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.simulation\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ + +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulator.external.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" &&\ + cd org.geppetto.simulator.external &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.simulator.external\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ + +RUN $APP_DIR/copy.sh https://github.com/Metacell/org.geppetto.simulator.scidash.git "${defaultBranch}" "${defaultBranch}" "${defaultBranch}" &&\ + cd org.geppetto.simulator.scidash &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.simulator.scidash\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ + +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.frontend.git "${simflagBranch}" "${simflagBranch}" "${simflagBranch}" &&\ + cd org.geppetto.frontend && cp /git/scidash/service/geppetto/frontend/app-config.xml ./src/main/webapp/WEB-INF/spring/ &&\ + /bin/echo -e "\e[96mMaven install org.geppetto.frontend\e[0m" &&\ + mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ + rm -rf src && cd ../ WORKDIR $SOURCES_DIR/org.geppetto/utilities/source_setup RUN python2 update_server.py diff --git a/service/docker/build_database.sh b/service/docker/build_database.sh index dcfe5d17..bdd973f3 100755 --- a/service/docker/build_database.sh +++ b/service/docker/build_database.sh @@ -1,4 +1,11 @@ #!/bin/bash + +branchname=$1 +if [ -z "$branchname" ] +then + branchname="development" +fi + echo "We are going to build the scidash Database, be carefull since" echo " this can overwrite the existing container if another one is already running" while true; do @@ -6,9 +13,9 @@ while true; do case $yn in [Yy]* ) read -p "Please type the tag you want to use for this build (default will use the latest and overwrite this). [latest/user_input] > " tag; if [[ -z "$tag" ]]; then - docker build -f Dockerfile-postgres -t metacell/scidash_db:latest . + docker build --build-arg SCIDASH_BRANCH=${branchname} -f Dockerfile-postgres -t metacell/scidash_db:latest . else - docker build -f Dockerfile-postgres -t metacell/scidash_db:$tag . + docker build --build-arg SCIDASH_BRANCH=${branchname} -f Dockerfile-postgres -t metacell/scidash_db:$tag . fi break;; [Nn]* ) exit;; diff --git a/service/docker/copy.sh b/service/docker/copy.sh new file mode 100755 index 00000000..03655026 --- /dev/null +++ b/service/docker/copy.sh @@ -0,0 +1,58 @@ +#!/bin/bash +clone_url=$1 +target_branch=$2 +origin_branch=$3 +default_branch=$4 + +################################################################################################# +# +# Use it for copying repositories: +# - org.geppetto.frontend +# - org.geppetto.model +# - org.geppetto.core +# - etc +# +# It requires: +# - target_branch: main branch to test ($TRAVIS_BRANCH) +# - origin_branch: branch where the new code was written ($TRAVIS_PULL_REQUEST_BRANCH) +# - default_branch: branch to use as main branch in case $main_branch does not exist +# +################################################################################################# + +name=`echo $clone_url | sed 's~^.*/~~g' | sed 's~.git~~g'` +mkdir $name + +if [ -z $origin_branch ]; then + git ls-remote --heads --tags $clone_url | grep -E 'refs/(heads|tags)/'$target_branch > /dev/null + if [ $? -eq 0 ]; then + download_url=`echo $clone_url | sed 's~\.git~/archive/'$target_branch'.zip~g'` + /bin/echo -e "\e[1;35mDownloading <$target_branch> branch for $name.\e[0m" + else + download_url=`echo $clone_url | sed 's~\.git~/archive/'$default_branch'.zip~g'` + /bin/echo -e "\e[1;35mDownloading <$default_branch> branch for $name.\e[0m" + fi + curl -sL $download_url | bsdtar -xzf - -C $name --strip-components 1 +else + # clone repository + /bin/echo -e "\e[1;35mCloning <$default_branch> branch for <$name> repository.\e[0m" + git clone -b $default_branch $clone_url + cd $name + # check if target_branch exists in the repo and ckeckout default_branch if it does not + git show-ref | grep -E 'refs/remotes/origin/'$target_branch > /dev/null + if [ $? -eq 0 ]; then + /bin/echo -e "\e[1;35mCheckout <$target_branch> branch.\e[0m" + git checkout $target_branch + else + /bin/echo -e "\e[1;35m<$target_branch> branch does not exist in <$name> repository, we stay on <$default_branch> branch.\e[0m" + fi + # check if origin_branch exists in the repo and merge origin_branch in target branch if it does + git show-ref | grep -E 'refs/remotes/origin/'$origin_branch > /dev/null + if [ $? -eq 0 ]; then + /bin/echo -e "\e[1;35mMerging <$origin_branch> branch into (<$target_branch>|<$default_branch>) for <$name> repository.\e[0m" + git merge --no-edit --no-stat --no-summary --quiet --no-progress origin/$origin_branch + else + /bin/echo -e "\e[1;35m<$origin_branch> branch does not exist in <$name> repository, no merge will be performed on this repository.\e[0m" + fi + rm -rf .git + cd .. +fi diff --git a/service/scripts/install-backend.sh b/service/scripts/install-backend.sh index c85662f4..bcd0dc18 100755 --- a/service/scripts/install-backend.sh +++ b/service/scripts/install-backend.sh @@ -2,14 +2,31 @@ virtualenv=false; pygeppetto_folder="pygeppetto-django" -pygeppetto_django_repo="https://github.com/MetaCell/pygeppetto-django.git -b 4.0.1 $pygeppetto_folder"; +pygeppetto_branch="4.0.1" +pygeppetto_django_repo="https://github.com/MetaCell/pygeppetto-django.git" -while getopts "v" opt +POSITIONAL=() +while [[ $# -gt 0 ]] do - case $opt in - v) virtualenv=true;; - esac +key="$1" + +case $key in + -v|--virtualenv) + virtualenv=true + shift # past argument + ;; + -b|--branch) + pygeppetto_branch="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; +esac done +set -- "${POSITIONAL[@]}" # restore positional parameters if [ "$virtualenv" = true ] ; then virtualenv -p python3 ./venv; @@ -18,7 +35,13 @@ fi pip install -r requirements.txt; -git clone $pygeppetto_django_repo; +git ls-remote --heads --tags $pygeppetto_django_repo | grep -E 'refs/(heads|tags)/'$pygeppetto_branch > /dev/null + +if [ $? -eq 0 ]; then + git clone -b $pygeppetto_branch $pygeppetto_django_repo $pygeppetto_folder +else + git clone -b 4.0.1 $pygeppetto_django_repo $pygeppetto_folder +fi cd $pygeppetto_folder; diff --git a/service/scripts/install-frontend.sh b/service/scripts/install-frontend.sh index f76218b4..d7bf2aa4 100755 --- a/service/scripts/install-frontend.sh +++ b/service/scripts/install-frontend.sh @@ -4,8 +4,30 @@ node -v root_path=$PWD; -geppetto_repo="https://github.com/openworm/org.geppetto.frontend.git -b v0.4.2-beta"; -extension_repo="https://github.com/MetaCell/geppetto-scidash.git -b 4.0.2"; +geppetto_repo="https://github.com/openworm/org.geppetto.frontend.git" +geppetto_branch="v0.4.2-beta" +extension_repo="https://github.com/MetaCell/geppetto-scidash.git" +extension_branch="4.0.2" + +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -b|--branch) + geppetto_branch="$2" + extension_branch="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; +esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters geppetto_path="./static/org.geppetto.frontend"; geppetto_app_path="$geppetto_path/src/main/webapp"; @@ -13,8 +35,23 @@ extension_path="$geppetto_app_path/extensions/geppetto-scidash"; sample_config_path="./service/geppetto/GeppettoConfiguration.json"; actual_config_path="$geppetto_app_path/GeppettoConfiguration.json"; -git clone $geppetto_repo $geppetto_path; -git clone $extension_repo $extension_path; +# The lines below can be commented out when we will be in line with the latest geppetto, differently +# if we try to test development this won't work since we are diverging from geppetto frontend at the moment +#git ls-remote --heads --tags $geppetto_repo | grep -E 'refs/(heads|tags)/'$geppetto_branch > /dev/null + +#if [ $? -eq 0 ]; then +# git clone -b $geppetto_branch $geppetto_repo $geppetto_path; +#else + git clone -b v0.4.2-beta $geppetto_repo $geppetto_path; +#fi + +git ls-remote --heads --tags $extension_repo | grep -E 'refs/(heads|tags)/'$extension_branch > /dev/null + +if [ $? -eq 0 ]; then + git clone -b $extension_branch $extension_repo $extension_path; +else + git clone -b 4.0.2 $extension_repo $extension_path; +fi cd $geppetto_app_path; diff --git a/service/scripts/trigger-travis.sh b/service/scripts/trigger-travis.sh new file mode 100755 index 00000000..a1a08ead --- /dev/null +++ b/service/scripts/trigger-travis.sh @@ -0,0 +1,126 @@ +#!/bin/sh -f + +# Trigger a new Travis-CI job. +# Ordinarily, a new Travis job is triggered when a commit is pushed to a +# GitHub repository. The trigger-travis.sh script provides a programmatic +# way to trigger a new Travis job. + +# Usage: +# trigger-travis.sh [--pro] [--branch BRANCH] GITHUBID GITHUBPROJECT TRAVIS_ACCESS_TOKEN [MESSAGE] +# For example: +# trigger-travis.sh typetools checker-framework `cat ~/private/.travis-access-token` "Trigger for testing" +# +# where --pro means to use travis-ci.com instead of travis-ci.org, and +# where TRAVIS_ACCESS_TOKEN is, or ~/private/.travis-access-token contains, +# the Travis access token. +# +# Your Travis access token is the text after "Your access token is " in +# the output of this compound command: +# travis login && travis token +# (If the travis program isn't installed, then use either of these two commands: +# gem install travis +# sudo apt-get install ruby-dev && sudo gem install travis +# Don't do "sudo apt-get install travis" which installs a trajectory analyzer.) +# Note that the Travis access token output by `travis token` differs from the +# Travis token available at https://travis-ci.org/profile . +# If you store it in in a file, make sure the file is not readable by others, +# for example by running: chmod og-rwx ~/private/.travis-access-token + +# To use this script to trigger a dependent build in Travis, do two things: +# +# 1. Set an environment variable TRAVIS_ACCESS_TOKEN by navigating to +# https://travis-ci.org/MYGITHUBID/MYGITHUBPROJECT/settings +# The TRAVIS_ACCESS_TOKEN environment variable will be set when Travis runs +# the job, but won't be visible to anyone browsing https://travis-ci.org/. +# +# 2. Add the following to your .travis.yml file, where you replace +# OTHERGITHUB* by a specific downstream project, but you leave +# $TRAVIS_ACCESS_TOKEN as literal text: +# +# jobs: +# include: +# - stage: trigger downstream +# jdk: oraclejdk8 +# script: | +# echo "TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST" +# if [[ ($TRAVIS_BRANCH == master) && +# ($TRAVIS_PULL_REQUEST == false) ]] ; then +# curl -LO --retry 3 https://raw.github.com/mernst/plume-lib/master/bin/trigger-travis.sh +# sh trigger-travis.sh OTHERGITHUBID OTHERGITHUBPROJECT $TRAVIS_ACCESS_TOKEN +# fi + +# TODO: Show how to use the --branch command-line argument. +# TODO: Enable the script to clone a particular branch rather than master. +# This would require a way to know the relationships among branches in +# different GitHub projects. It's easier to run all your tests within a +# single Travis job, if they fit within Travis's 50-minute time limit. + +# An alternative to this script would be to install the Travis command-line +# client and then run: +# travis restart -r OTHERGITHUBID/OTHERGITHUBPROJECT +# That is undesirable because it restarts an old job, destroying its history, +# rather than starting a new job which is our goal. + +# Parts of this script were originally taken from +# http://docs.travis-ci.com/user/triggering-builds/ + + +if [ "$#" -lt 3 ] || [ "$#" -ge 7 ]; then + echo "Wrong number of arguments $# to trigger-travis.sh; run like:" + echo " trigger-travis.sh [--pro] [--branch BRANCH] GITHUBID GITHUBPROJECT TRAVIS_ACCESS_TOKEN [MESSAGE]" >&2 + exit 1 +fi + +if [ "$1" = "--pro" ] ; then + TRAVIS_URL=travis-ci.com + shift +else + TRAVIS_URL=travis-ci.org +fi + +if [ "$1" = "--branch" ] ; then + shift + BRANCH="$1" + shift +else + BRANCH=master +fi + +USER=$1 +REPO=$2 +TOKEN=$3 +if [ $# -eq 4 ] ; then + MESSAGE=",\"message\": \"$4\"" +elif [ -n "$TRAVIS_REPO_SLUG" ] ; then + MESSAGE=",\"message\": \"Triggered by upstream build of $TRAVIS_REPO_SLUG commit "`git rev-parse --short HEAD`"\"" +else + MESSAGE="" +fi +## For debugging: +# echo "USER=$USER" +# echo "REPO=$REPO" +# echo "TOKEN=$TOKEN" +# echo "MESSAGE=$MESSAGE" + +body="{ +\"request\": { + \"branch\":\"$BRANCH\" + $MESSAGE +}}" + +# It does not work to put / in place of %2F in the URL below. I'm not sure why. +curl -s -X POST \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -H "Travis-API-Version: 3" \ + -H "Authorization: token ${TOKEN}" \ + -d "$body" \ + https://api.${TRAVIS_URL}/repo/${USER}%2F${REPO}/requests \ + | tee /tmp/travis-request-output.$$.txt + +if grep -q '"@type": "error"' /tmp/travis-request-output.$$.txt; then + exit 1 +fi +if grep -q 'access denied' /tmp/travis-request-output.$$.txt; then + exit 1 +fi From 3348c4991c211eceef2099b5ce3483daad7b9076 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Tue, 17 Dec 2019 21:14:35 +0100 Subject: [PATCH 008/235] fix: initial timestamp from/to, search max settings.SCIDASH_INITIAL_SEARCH_QUARTERS quarters back, if not found show settings.SCIDASH_INITIAL_SEARCH_QUARTERS quarters back period --- scidash/main/settings.py | 4 ++++ scidash/sciunittests/views.py | 12 +++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/scidash/main/settings.py b/scidash/main/settings.py index 9d0b0b9c..0fd219bc 100644 --- a/scidash/main/settings.py +++ b/scidash/main/settings.py @@ -249,3 +249,7 @@ CELERY_RESULT_BACKEND = 'django-db' NO_IMPORT_TAG = 'unschedulable' + +# SCIDASH +# Initial search number of quarters to search back in time. +SCIDASH_INITIAL_SEARCH_QUARTERS = 12 diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index a70d419c..99e91b50 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -17,16 +17,10 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - three_month_ago = current_date_iso - three_month_period - six_month_ago = three_month_ago - three_month_period - nine_month_ago = six_month_ago - three_month_period - tvelwe_month_ago = nine_month_ago - three_month_period - acceptable_period = None - for period in [ - three_month_ago, six_month_ago, nine_month_ago, tvelwe_month_ago - ]: + for quarter in range(1, s.SCIDASH_INITIAL_SEARCH_QUARTERS + 1): + period = current_date_iso - quarter*three_month_period count = ScoreInstance.objects.filter( timestamp__gte=period, timestamp__lt=current_date_iso ).count() @@ -36,7 +30,7 @@ def get(self, request, *args, **kwargs): break if acceptable_period is None: - acceptable_period = tvelwe_month_ago + acceptable_period = period return Response( { From 626f2cd701f4731e18081c6215bc2d71917770ce Mon Sep 17 00:00:00 2001 From: Jesus M Date: Wed, 18 Dec 2019 19:04:38 +0900 Subject: [PATCH 009/235] Use SCIDASH_BRANCH variable --- service/docker/Dockerfile-postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index d6f08219..238acd57 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -2,7 +2,7 @@ FROM postgres:9.4 ARG ROOT=/ ARG APP_DIR=/app -ARG SCIDASH_BRANCH=development +ARG SCIDASH_BRANCH=$SCIDASH_BRANCH ARG originBranch=development ARG defaultBranch=development From 1e535295648ce9745fb4620071ae79dbd67eb9f8 Mon Sep 17 00:00:00 2001 From: Jesus M Date: Wed, 18 Dec 2019 19:36:44 +0900 Subject: [PATCH 010/235] Update Dockerfile-postgres --- service/docker/Dockerfile-postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index 238acd57..d6f08219 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -2,7 +2,7 @@ FROM postgres:9.4 ARG ROOT=/ ARG APP_DIR=/app -ARG SCIDASH_BRANCH=$SCIDASH_BRANCH +ARG SCIDASH_BRANCH=development ARG originBranch=development ARG defaultBranch=development From 38606a92b0d172b71e62db76f4b235eb27af65bf Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Wed, 18 Dec 2019 11:51:18 +0100 Subject: [PATCH 011/235] fix: return class_name + ( import_path ) as class_name --- scidash/sciunittests/serializers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scidash/sciunittests/serializers.py b/scidash/sciunittests/serializers.py index b5471bbb..ded04cab 100644 --- a/scidash/sciunittests/serializers.py +++ b/scidash/sciunittests/serializers.py @@ -17,7 +17,6 @@ class TestSuiteSerializer(GetOrCreateMixin, WritableNestedModelSerializer): - owner = ScidashUserSerializer( default=serializers.CurrentUserDefault(), read_only=True ) @@ -30,9 +29,17 @@ class Meta: class TestClassSerializer( GetByKeyOrCreateMixin, WritableNestedModelSerializer ): + class_name = serializers.SerializerMethodField() units_name = serializers.CharField(required=False) key = 'import_path' + def get_class_name(self, obj): + # return class_name + ( first part of import_path ) + return obj.class_name + \ + (' (' + \ + '.'.join(obj.import_path.split('.')[0:-1]) + + ')').replace(' ()', '') + class Meta: model = TestClass fields = '__all__' @@ -132,7 +139,6 @@ class Meta: class ScoreClassSerializer( GetByKeyOrCreateMixin, WritableNestedModelSerializer ): - key = 'class_name' class Meta: From 15037775d10077e2230b28afa0ee0b9dbae5a242 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 19 Dec 2019 12:10:57 +0100 Subject: [PATCH 012/235] fix: code cleanup --- scidash/sciunittests/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scidash/sciunittests/serializers.py b/scidash/sciunittests/serializers.py index ded04cab..97f369dc 100644 --- a/scidash/sciunittests/serializers.py +++ b/scidash/sciunittests/serializers.py @@ -36,7 +36,7 @@ class TestClassSerializer( def get_class_name(self, obj): # return class_name + ( first part of import_path ) return obj.class_name + \ - (' (' + \ + (' (' + '.'.join(obj.import_path.split('.')[0:-1]) + ')').replace(' ()', '') From 4cdc0abca64b48928a321d59b3b7eb2aa39ffd51 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 19 Dec 2019 12:49:52 +0100 Subject: [PATCH 013/235] feat: add index on scoreinstances(timestamp) --- scidash/sciunittests/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scidash/sciunittests/models.py b/scidash/sciunittests/models.py index 2f17f852..f3094fb1 100644 --- a/scidash/sciunittests/models.py +++ b/scidash/sciunittests/models.py @@ -243,6 +243,9 @@ def prediction(self): class Meta: ordering = ['-timestamp'] + indexes = [ + models.Index(fields=['-timestamp',]), + ] def __str__(self): return "Score for {0} in {1} test instance".format( From 1138ba34af5910cf341bad403400599ce87e8f62 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 10:36:38 -0500 Subject: [PATCH 014/235] fix: add missing migration to git repo --- .../migrations/0046_auto_20191211_0156.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 scidash/sciunittests/migrations/0046_auto_20191211_0156.py diff --git a/scidash/sciunittests/migrations/0046_auto_20191211_0156.py b/scidash/sciunittests/migrations/0046_auto_20191211_0156.py new file mode 100644 index 00000000..7e28b229 --- /dev/null +++ b/scidash/sciunittests/migrations/0046_auto_20191211_0156.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-12-11 01:56 +from __future__ import unicode_literals + +import datetime +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import scidash.sciunittests.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0045_auto_20190712_1041'), + ] + + operations = [ + migrations.AddField( + model_name='testclass', + name='default_params', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, encoder=scidash.sciunittests.models.JSONEncoder, null=True), + ), + migrations.AlterField( + model_name='scoreinstance', + name='timestamp', + field=models.DateTimeField(default=datetime.datetime.today), + ), + migrations.AlterField( + model_name='testinstance', + name='timestamp', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='testsuite', + name='timestamp', + field=models.DateTimeField(default=datetime.datetime.today), + ), + ] From abfdda4ed81f0b7d564a7ba3303b4bd8c844488c Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 15:09:03 -0500 Subject: [PATCH 015/235] fix: renames 0046_auto_20191211_0156.py -> 0046_auto_20190827_0221.py --- .../{0046_auto_20191211_0156.py => 0046_auto_20190827_0221.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scidash/sciunittests/migrations/{0046_auto_20191211_0156.py => 0046_auto_20190827_0221.py} (100%) diff --git a/scidash/sciunittests/migrations/0046_auto_20191211_0156.py b/scidash/sciunittests/migrations/0046_auto_20190827_0221.py similarity index 100% rename from scidash/sciunittests/migrations/0046_auto_20191211_0156.py rename to scidash/sciunittests/migrations/0046_auto_20190827_0221.py From d28559bea9f90a3c46b3fc974c0ca033d40d98aa Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 19 Dec 2019 13:23:18 +0100 Subject: [PATCH 016/235] feat: added index on scoreinstance.timestamp desc --- .../migrations/0047_auto_20191219_1217.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scidash/sciunittests/migrations/0047_auto_20191219_1217.py diff --git a/scidash/sciunittests/migrations/0047_auto_20191219_1217.py b/scidash/sciunittests/migrations/0047_auto_20191219_1217.py new file mode 100644 index 00000000..3d8f5f1b --- /dev/null +++ b/scidash/sciunittests/migrations/0047_auto_20191219_1217.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-12-19 12:17 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0046_auto_20190827_0221'), + ] + + operations = [ + migrations.AddIndex( + model_name='scoreinstance', + index=models.Index(fields=['-timestamp'], name='sciunittest_timesta_aa7614_idx'), + ), + ] From 479f55e56efe06609b41a545f50c6d7234100f94 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Fri, 20 Dec 2019 10:09:20 +0100 Subject: [PATCH 017/235] feat: return acceptable_period based on top N scores --- scidash/sciunittests/views.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index 99e91b50..150e8f88 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -9,7 +9,21 @@ class DateRangeView(APIView): def get(self, request, *args, **kwargs): - three_month_period = datetime.timedelta(3 * 365 / 12) + """Returns the initial search period (acceptable_period). + for the first N (settings.ACCEPTABLE_SCORE_INSTANCES_AMOUNT) scores. + + Parameters + ---------- + - + + Returns + ------- + JSON object + { + "current_date": "", + "acceptable_period": "" + } + """ current_date = datetime.date.today() + datetime.timedelta(days=1) current_date_iso = datetime.datetime( year=current_date.year, @@ -17,20 +31,11 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - acceptable_period = None - - for quarter in range(1, s.SCIDASH_INITIAL_SEARCH_QUARTERS + 1): - period = current_date_iso - quarter*three_month_period - count = ScoreInstance.objects.filter( - timestamp__gte=period, timestamp__lt=current_date_iso - ).count() - - if count > s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT: - acceptable_period = period - break - - if acceptable_period is None: - acceptable_period = period + scores = ScoreInstance.objects.filter(timestamp__lt=current_date_iso)[:s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT] + if scores: + acceptable_period = scores[len(scores)-1].timestamp + else: + acceptable_period = current_date_iso - datetime.date.year return Response( { From c42955703a54ad100f407f7796dbf371f7eeff75 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Fri, 20 Dec 2019 10:31:11 +0100 Subject: [PATCH 018/235] fix: rework acceptable_period calculation --- scidash/sciunittests/views.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index 150e8f88..e3109d9d 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -31,11 +31,20 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - scores = ScoreInstance.objects.filter(timestamp__lt=current_date_iso)[:s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT] + scores = ScoreInstance.objects.filter( + timestamp__lt=current_date_iso).order_by('-timestamp') \ + [:s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT] if scores: - acceptable_period = scores[len(scores)-1].timestamp + # found scores, acceptable period is scores last.timestamp + # because sorting is DESC timestamp + acceptable_period = scores.reverse()[0].timestamp else: - acceptable_period = current_date_iso - datetime.date.year + # acceptable period defaults to current date - 1 year + acceptable_period = datetime.datetime( + year=current_date.year-1, + month=current_date.month, + day=current_date.day + ) return Response( { From 947206ab6ccd601ce3b8d117f51b792d0b0f579b Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Fri, 20 Dec 2019 22:59:00 +0100 Subject: [PATCH 019/235] travis file updated --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..68302bc3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: node_js +sudo: required +node_js: + - "9" +install: + - npm install jest@24.8.0 puppeteer@1.17.0 jest-puppeteer@4.3.0 @babel/preset-env@7.4.5 url-join@4.0.0 @babel/core@7.4.5 +script: + - echo -e "Trigger python unit testing in here when they will be ready" || true +after_success: + - bash $TRAVIS_BUILD_DIR/service/scripts/trigger_travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN From 4667d1d457a6582e829c9f580e263e80eba78af0 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:01:29 +0100 Subject: [PATCH 020/235] checking travis file --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 68302bc3..61ec5862 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,4 @@ script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - bash $TRAVIS_BUILD_DIR/service/scripts/trigger_travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + From 5e90cd03c37b27fb65bb1302cede0ef71bee5077 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:06:43 +0100 Subject: [PATCH 021/235] travis build dir issue --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 61ec5862..31abfa02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,7 @@ install: script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: + - ls -la $TRAVIS_BUILD_DIR + - ls -la ./ - bash $TRAVIS_BUILD_DIR/service/scripts/trigger_travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN From c9c93515c031bb3efe2b245bfd8ce8ae86a90228 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:09:27 +0100 Subject: [PATCH 022/235] travis file typo fixed --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 31abfa02..4925f19b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,5 @@ install: script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - - ls -la $TRAVIS_BUILD_DIR - - ls -la ./ - - bash $TRAVIS_BUILD_DIR/service/scripts/trigger_travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN From 9965bd62ab8a17fb19a4aed15dec04da8b4a58d4 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:19:37 +0100 Subject: [PATCH 023/235] travis file typo fixed --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4925f19b..a14e19d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,6 @@ install: script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + - echo $TRAVIS_BRANCH + - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b ${TRAVIS_BRANCH} Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN From 5d7cf9bf5f66ee96d232d605ec7527c7b6f71972 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:32:37 +0100 Subject: [PATCH 024/235] check branch for downstream job --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a14e19d3..0a90a795 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,5 +8,8 @@ script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - echo $TRAVIS_BRANCH - - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b ${TRAVIS_BRANCH} Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + else + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN From 976c74b22ed4704a3bafc04279d6fca47dc490e4 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:39:09 +0100 Subject: [PATCH 025/235] check branch for downstream job 2 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0a90a795..b47e8b7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,4 +12,5 @@ after_success: bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN else bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + fi From 31d39e8c7d377dd729391cd17527748071318346 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:42:42 +0100 Subject: [PATCH 026/235] check branch for downstream job 3 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b47e8b7a..aa3ff58d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ script: after_success: - echo $TRAVIS_BRANCH - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; else - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN - fi + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + fi; From dab156bbf27c23f26b66c04a9094579e54c94e4b Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:53:44 +0100 Subject: [PATCH 027/235] check branch for downstream job 4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aa3ff58d..264d0758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: required node_js: - "9" install: - - npm install jest@24.8.0 puppeteer@1.17.0 jest-puppeteer@4.3.0 @babel/preset-env@7.4.5 url-join@4.0.0 @babel/core@7.4.5 + - npm install jest@24.8.0 script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: From 2f5b34be7d545b8f20a3eefd47d2b79d34cb5bcc Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 00:57:18 +0100 Subject: [PATCH 028/235] check branch for downstream job 5 --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 264d0758..8ba73777 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,6 @@ script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - echo $TRAVIS_BRANCH - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; - else - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; - fi; + - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; fi; + - if [ "$TRAVIS_PULL_REQUEST" != "true" ]; then bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; fi; From a3ed394faf3073269068ac3ee3d6c4227153a029 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 01:06:49 +0100 Subject: [PATCH 029/235] check branch for downstream job 6 --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ba73777..6b42a306 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ install: script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - - echo $TRAVIS_BRANCH - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; fi; - - if [ "$TRAVIS_PULL_REQUEST" != "true" ]; then bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; fi; + - test $TRAVIS_PULL_REQUEST = "false" && bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + - test $TRAVIS_PULL_REQUEST = "true" && bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN From 0b62dfa3040ab27f919357560575fe666eb59268 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:20:30 +0100 Subject: [PATCH 030/235] check branch for downstream job 7 --- .travis.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b42a306..8f651d37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,14 @@ sudo: required node_js: - "9" install: - - npm install jest@24.8.0 + - npm install jest@24.8.0 puppeteer@1.17.0 jest-puppeteer@4.3.0 @babel/preset-env@7.4.5 url-join@4.0.0 @babel/core@7.4.5 script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - - test $TRAVIS_PULL_REQUEST = "false" && bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN - - test $TRAVIS_PULL_REQUEST = "true" && bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + - echo $TRAVIS_BRANCH + - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then + $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + else + $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + fi From a36f28b86e6e8e4b4ae665b617355b13b9ed70aa Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:27:27 +0100 Subject: [PATCH 031/235] check branch for downstream job 8 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f651d37..5c10c269 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - echo $TRAVIS_BRANCH - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; else - $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; fi From fbc0faeba7acf52ba19f013b19ff3c641b796723 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:30:55 +0100 Subject: [PATCH 032/235] check branch for downstream job 9 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5c10c269..bddf3e4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ script: - echo -e "Trigger python unit testing in here when they will be ready" || true after_success: - echo $TRAVIS_BRANCH + - echo $TRAVIS_PULL_REQUEST - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; else From c78480c0bd692fad15a7680bd3f2ced68be69f48 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:35:56 +0100 Subject: [PATCH 033/235] check branch for downstream job 10 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index bddf3e4a..62d034ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,12 @@ script: after_success: - echo $TRAVIS_BRANCH - echo $TRAVIS_PULL_REQUEST + - echo $TRAVIS_PULL_REQUEST_BRANCH - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + echo "I am in the first condition, so I am evaluating"; else bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + echo "I am in the second condition"; fi From ca7c3952e9486825ad90e2e220832f4c9dbfc895 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:46:40 +0100 Subject: [PATCH 034/235] check branch for downstream job 11 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62d034ce..0724e02e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,10 @@ after_success: - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_PULL_REQUEST_BRANCH - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; echo "I am in the first condition, so I am evaluating"; else - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; echo "I am in the second condition"; fi From 5ebbe911e648071db865b897493e17576f1ebb5b Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:51:16 +0100 Subject: [PATCH 035/235] troubleshooting access denied --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0724e02e..14a2259e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ after_success: bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; echo "I am in the second condition"; fi - + - cat /tmp/travis-request-output* From fd71623e304848497b41f4fae6c52d7ef639da1a Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 02:56:46 +0100 Subject: [PATCH 036/235] troubleshooting access denied 2 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14a2259e..2ee1f454 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,10 @@ after_success: - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_PULL_REQUEST_BRANCH - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + bash "${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_PULL_REQUEST_BRANCH} Metacell geppetto-scidash ${TRAVIS_ACCESS_TOKEN}" ; echo "I am in the first condition, so I am evaluating"; else - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + bash "${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_BRANCH} Metacell geppetto-scidash {$TRAVIS_ACCESS_TOKEN}" ; echo "I am in the second condition"; fi - cat /tmp/travis-request-output* From d1f6874e57325433ddd3c2c04ce3b864b29411e7 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 03:21:30 +0100 Subject: [PATCH 037/235] troubleshooting access denied 3 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ee1f454..1eb7bd05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,10 @@ after_success: - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_PULL_REQUEST_BRANCH - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then - bash "${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_PULL_REQUEST_BRANCH} Metacell geppetto-scidash ${TRAVIS_ACCESS_TOKEN}" ; + `${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_PULL_REQUEST_BRANCH} Metacell geppetto-scidash ${TRAVIS_ACCESS_TOKEN}` ; echo "I am in the first condition, so I am evaluating"; else - bash "${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_BRANCH} Metacell geppetto-scidash {$TRAVIS_ACCESS_TOKEN}" ; + `${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_BRANCH} Metacell geppetto-scidash {$TRAVIS_ACCESS_TOKEN}` ; echo "I am in the second condition"; fi - cat /tmp/travis-request-output* From 8bab33769d4987fb367b9dca9ad862820eb2d068 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 03:26:16 +0100 Subject: [PATCH 038/235] settings fixed --- .travis.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1eb7bd05..4a569ffc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,5 @@ after_success: - echo $TRAVIS_BRANCH - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_PULL_REQUEST_BRANCH - - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then - `${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_PULL_REQUEST_BRANCH} Metacell geppetto-scidash ${TRAVIS_ACCESS_TOKEN}` ; - echo "I am in the first condition, so I am evaluating"; - else - `${TRAVIS_BUILD_DIR}/service/scripts/trigger-travis.sh -b ${TRAVIS_BRANCH} Metacell geppetto-scidash {$TRAVIS_ACCESS_TOKEN}` ; - echo "I am in the second condition"; - fi + - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN - cat /tmp/travis-request-output* From b875138da215cb8d6d74802f4a01c25497d71901 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 03:40:42 +0100 Subject: [PATCH 039/235] settings fixed 2 --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a569ffc..7a3e94b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,5 +10,11 @@ after_success: - echo $TRAVIS_BRANCH - echo $TRAVIS_PULL_REQUEST - echo $TRAVIS_PULL_REQUEST_BRANCH - - bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh -b $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN + - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh --branch $TRAVIS_PULL_REQUEST_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + echo "I am in the first condition, so I am evaluating"; + else + bash $TRAVIS_BUILD_DIR/service/scripts/trigger-travis.sh --branch $TRAVIS_BRANCH Metacell geppetto-scidash $TRAVIS_ACCESS_TOKEN ; + echo "I am in the second condition"; + fi - cat /tmp/travis-request-output* From 6592a4d119a8c0015b6d4cd4a47cd0d051d73dc5 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 16:40:10 +0100 Subject: [PATCH 040/235] re-aligned branches to use the CI geppetto-scidash --- service/docker/Dockerfile-scidash | 6 +++--- service/docker/Dockerfile-virgo | 28 +++++++++++++--------------- service/scripts/install-backend.sh | 4 ++-- service/scripts/install-frontend.sh | 8 ++++---- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index d92d3a8f..3e934c8f 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -9,9 +9,9 @@ ARG NRN_SYMLINK=/Applications/NEURON-7.6/nrn/ ARG DOTENV_FILE=env-docker ARG STATIC_DIR=$APP_DIR/scidash/static ARG GEPPETTO_DIR=$STATIC_DIR/org.geppetto.frontend/src/main/webapp -ARG targetBranch=development -ARG originBranch=development -ARG defaultBranch=development +ARG targetBranch=geppetto-scidash +ARG originBranch=geppetto-scidash +ARG defaultBranch=geppetto-scidash ENV SERVER_HOME $APP_DIR/virgo-tomcat-server diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index 19c4f861..220e5521 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -7,13 +7,11 @@ ARG SCIDASH_REPO_FOLDER=/git ARG GEPPETTO_REPO=http://github.com/openworm/org.geppetto.git ARG ALPHA_TAG=v0.4.2-alpha ARG SIMFLAG_TAG=v0.4.2-simflag -ARG SCIDASH_BRANCH=development +ARG SCIDASH_BRANCH=geppetto-scidash -ARG alphaBranch=v0.4.2-alpha -ARG simflagBranch=v0.4.2-simflag -ARG targetBranch=development -ARG originBranch=development -ARG defaultBranch=development +ARG targetBranch=geppetto-scidash +ARG originBranch=geppetto-scidash +ARG defaultBranch=geppetto-scidash # -== Install OpenJDK and certificates ==- WORKDIR / @@ -43,7 +41,7 @@ COPY ./copy.sh $APP_DIR # -== Download SCIDASH repo ==- -RUN git clone -b $SCIDASH_BRANCH https://github.com/MetaCell/scidash +RUN $APP_DIR/copy.sh http://github.com/Metacell/scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" # -== INSTALL MAVEN ==- WORKDIR /tmp @@ -103,48 +101,48 @@ RUN mvn --version # -== INSTALL GEPPETTO ==- WORKDIR $SOURCES_DIR -RUN $APP_DIR/copy.sh http://github.com/openworm/org.geppetto.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" +RUN $APP_DIR/copy.sh http://github.com/openworm/org.geppetto.git "${targetBranch}" "${originBranch}" "${defaultBranch}" RUN cp /git/scidash/service/geppetto/config.json $SOURCES_DIR/org.geppetto/utilities/source_setup RUN cp /git/scidash/service/geppetto/setup.py $SOURCES_DIR/org.geppetto/utilities/source_setup RUN cp /git/scidash/service/geppetto/geppetto.plan $SOURCES_DIR/org.geppetto/ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.model &&\ /bin/echo -e "\e[96mMaven install org.geppetto.model\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.core.git "${simflagBranch}" "${simflagBranch}" "${simflagBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.core.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.core && cp /git/scidash/service/geppetto/core/app-config.xml ./src/main/java/META-INF/spring &&\ /bin/echo -e "\e[96mMaven install org.geppetto.core\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.neuroml.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.neuroml.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.model.neuroml &&\ /bin/echo -e "\e[96mMaven install org.geppetto.model.neuroml\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulation.git "${simflagBranch}" "${simflagBranch}" "${simflagBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulation.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.simulation &&\ /bin/echo -e "\e[96mMaven install org.geppetto.simulation\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulator.external.git "${alphaBranch}" "${alphaBranch}" "${alphaBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulator.external.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.simulator.external &&\ /bin/echo -e "\e[96mMaven install org.geppetto.simulator.external\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/Metacell/org.geppetto.simulator.scidash.git "${defaultBranch}" "${defaultBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/Metacell/org.geppetto.simulator.scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.simulator.scidash &&\ /bin/echo -e "\e[96mMaven install org.geppetto.simulator.scidash\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.frontend.git "${simflagBranch}" "${simflagBranch}" "${simflagBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.frontend.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.frontend && cp /git/scidash/service/geppetto/frontend/app-config.xml ./src/main/webapp/WEB-INF/spring/ &&\ /bin/echo -e "\e[96mMaven install org.geppetto.frontend\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ diff --git a/service/scripts/install-backend.sh b/service/scripts/install-backend.sh index bcd0dc18..88cc9a32 100755 --- a/service/scripts/install-backend.sh +++ b/service/scripts/install-backend.sh @@ -2,7 +2,7 @@ virtualenv=false; pygeppetto_folder="pygeppetto-django" -pygeppetto_branch="4.0.1" +pygeppetto_branch="geppetto-scidash" pygeppetto_django_repo="https://github.com/MetaCell/pygeppetto-django.git" POSITIONAL=() @@ -40,7 +40,7 @@ git ls-remote --heads --tags $pygeppetto_django_repo | grep -E 'refs/(heads|tags if [ $? -eq 0 ]; then git clone -b $pygeppetto_branch $pygeppetto_django_repo $pygeppetto_folder else - git clone -b 4.0.1 $pygeppetto_django_repo $pygeppetto_folder + git clone -b geppetto-scidash $pygeppetto_django_repo $pygeppetto_folder fi cd $pygeppetto_folder; diff --git a/service/scripts/install-frontend.sh b/service/scripts/install-frontend.sh index d7bf2aa4..2f9a119f 100755 --- a/service/scripts/install-frontend.sh +++ b/service/scripts/install-frontend.sh @@ -5,9 +5,9 @@ node -v root_path=$PWD; geppetto_repo="https://github.com/openworm/org.geppetto.frontend.git" -geppetto_branch="v0.4.2-beta" +geppetto_branch="geppett-scidash" extension_repo="https://github.com/MetaCell/geppetto-scidash.git" -extension_branch="4.0.2" +extension_branch="geppetto-scidash" POSITIONAL=() while [[ $# -gt 0 ]] @@ -42,7 +42,7 @@ actual_config_path="$geppetto_app_path/GeppettoConfiguration.json"; #if [ $? -eq 0 ]; then # git clone -b $geppetto_branch $geppetto_repo $geppetto_path; #else - git clone -b v0.4.2-beta $geppetto_repo $geppetto_path; + git clone -b geppetto-scidash $geppetto_repo $geppetto_path; #fi git ls-remote --heads --tags $extension_repo | grep -E 'refs/(heads|tags)/'$extension_branch > /dev/null @@ -50,7 +50,7 @@ git ls-remote --heads --tags $extension_repo | grep -E 'refs/(heads|tags)/'$exte if [ $? -eq 0 ]; then git clone -b $extension_branch $extension_repo $extension_path; else - git clone -b 4.0.2 $extension_repo $extension_path; + git clone -b geppetto-scidash $extension_repo $extension_path; fi cd $geppetto_app_path; From 9695bfc3e3f0d2c759fecee206674754c2cfbe44 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 16:52:34 +0100 Subject: [PATCH 041/235] fixed copy.sh retrieval --- service/docker/Dockerfile-postgres | 3 ++- service/docker/Dockerfile-scidash | 3 ++- service/docker/Dockerfile-virgo | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index d6f08219..290ff7f0 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -16,7 +16,8 @@ RUN apt-get update && \ RUN mkdir $APP_DIR WORKDIR $APP_DIR -COPY ./copy.sh $APP_DIR/ +RUN git clone https://github.com/ddelpiano/travis_utils +RUN cp travis_utils/copy.sh $APP_DIR RUN chmod 777 /tmp RUN ./copy.sh https://github.com/MetaCell/scidash "${SCIDASH_BRANCH}" "${originBranch}" "${defaultBranch}" RUN cp scidash/service/database/backup_database.sh /backup_database.sh diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index 3e934c8f..cfe42d47 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -33,7 +33,8 @@ RUN chown -R developer /home/developer/.config USER developer # COPYING PROJECT -COPY ./copy.sh $APP_DIR +RUN git clone https://github.com/ddelpiano/travis_utils +RUN cp travis_utils/copy.sh $APP_DIR WORKDIR $APP_DIR RUN $APP_DIR/copy.sh https://github.com/Metacell/scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index 220e5521..95adc873 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -37,7 +37,8 @@ RUN mkdir $SCIDASH_REPO_FOLDER WORKDIR $APP_DIR RUN mkdir $SOURCES_DIR WORKDIR $SCIDASH_REPO_FOLDER -COPY ./copy.sh $APP_DIR +RUN git clone https://github.com/ddelpiano/travis_utils +RUN cp travis_utils/copy.sh $APP_DIR # -== Download SCIDASH repo ==- From ea2c52186c68726ba90131bba1da21a56d791e90 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 18:17:01 +0100 Subject: [PATCH 042/235] fixed issue with geppetto-scidash CI branch --- service/docker/Dockerfile-postgres | 6 +++--- service/docker/Dockerfile-scidash | 2 +- service/docker/Dockerfile-virgo | 8 +++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/service/docker/Dockerfile-postgres b/service/docker/Dockerfile-postgres index 290ff7f0..85904751 100644 --- a/service/docker/Dockerfile-postgres +++ b/service/docker/Dockerfile-postgres @@ -2,8 +2,8 @@ FROM postgres:9.4 ARG ROOT=/ ARG APP_DIR=/app -ARG SCIDASH_BRANCH=development -ARG originBranch=development +ARG targetBranch=geppetto-scidash +ARG originBranch=geppetto-scidash ARG defaultBranch=development WORKDIR $ROOT @@ -19,7 +19,7 @@ WORKDIR $APP_DIR RUN git clone https://github.com/ddelpiano/travis_utils RUN cp travis_utils/copy.sh $APP_DIR RUN chmod 777 /tmp -RUN ./copy.sh https://github.com/MetaCell/scidash "${SCIDASH_BRANCH}" "${originBranch}" "${defaultBranch}" +RUN ./copy.sh https://github.com/MetaCell/scidash "${targetBranch}" "${originBranch}" "${defaultBranch}" RUN cp scidash/service/database/backup_database.sh /backup_database.sh RUN chmod 0644 /backup_database.sh RUN cp scidash/service/scripts/db-create-psql.sh /docker-entrypoint-initdb.d/db-create-psql.sh diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index cfe42d47..55a4b59c 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -11,7 +11,7 @@ ARG STATIC_DIR=$APP_DIR/scidash/static ARG GEPPETTO_DIR=$STATIC_DIR/org.geppetto.frontend/src/main/webapp ARG targetBranch=geppetto-scidash ARG originBranch=geppetto-scidash -ARG defaultBranch=geppetto-scidash +ARG defaultBranch=development ENV SERVER_HOME $APP_DIR/virgo-tomcat-server diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index 95adc873..afd4f90f 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -5,13 +5,11 @@ ARG SOURCES_DIR=$APP_DIR/sources ARG VIRGO_DIR=/opt/virgo ARG SCIDASH_REPO_FOLDER=/git ARG GEPPETTO_REPO=http://github.com/openworm/org.geppetto.git -ARG ALPHA_TAG=v0.4.2-alpha -ARG SIMFLAG_TAG=v0.4.2-simflag ARG SCIDASH_BRANCH=geppetto-scidash ARG targetBranch=geppetto-scidash ARG originBranch=geppetto-scidash -ARG defaultBranch=geppetto-scidash +ARG defaultBranch=development # -== Install OpenJDK and certificates ==- WORKDIR / @@ -91,9 +89,9 @@ RUN curl -L 'http://www.eclipse.org/downloads/download.php?file=/virgo/release/V RUN unzip virgo.zip RUN cp -r ./virgo-tomcat-server-3.7.2.RELEASE/* $VIRGO_DIR RUN rm $VIRGO_DIR/configuration/java-server.profile -RUN cp /git/scidash/service/geppetto/java-server.profile $VIRGO_DIR/configuration/ +RUN cp $APP_DIR/git/scidash/service/geppetto/java-server.profile $VIRGO_DIR/configuration/ RUN rm $VIRGO_DIR/configuration/tomcat-server.xml -RUN cat /git/scidash/service/geppetto/tomcat-server.xml | sed 's/127.0.0.1/0.0.0.0/' > $VIRGO_DIR/configuration/tomcat-server.xml +RUN cat $APP_DIR/git/scidash/service/geppetto/tomcat-server.xml | sed 's/127.0.0.1/0.0.0.0/' > $VIRGO_DIR/configuration/tomcat-server.xml RUN chmod u+x $VIRGO_DIR/bin/*.sh ENV SERVER_HOME $VIRGO_DIR RUN chmod 777 -R $VIRGO_DIR From 95eaf4f9be68b89f8c984e26667dabbabf1b5697 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 18:24:54 +0100 Subject: [PATCH 043/235] removed defaultBranch for virgo CI, typo done earlier --- service/docker/Dockerfile-virgo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index afd4f90f..60c1b271 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -9,7 +9,7 @@ ARG SCIDASH_BRANCH=geppetto-scidash ARG targetBranch=geppetto-scidash ARG originBranch=geppetto-scidash -ARG defaultBranch=development +ARG defaultBranch=geppetto-scidash # -== Install OpenJDK and certificates ==- WORKDIR / From e5ccdcc0f651026e6d4088c7b7a0cedd37e9d835 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sat, 21 Dec 2019 18:25:36 +0100 Subject: [PATCH 044/235] removed defaultBranch for virgo CI, typo done earlier 2 --- service/docker/Dockerfile-scidash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index 55a4b59c..cfe42d47 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -11,7 +11,7 @@ ARG STATIC_DIR=$APP_DIR/scidash/static ARG GEPPETTO_DIR=$STATIC_DIR/org.geppetto.frontend/src/main/webapp ARG targetBranch=geppetto-scidash ARG originBranch=geppetto-scidash -ARG defaultBranch=development +ARG defaultBranch=geppetto-scidash ENV SERVER_HOME $APP_DIR/virgo-tomcat-server From b8be96c4c2a8794425aaae1b400fadcdf3cc9cf1 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sun, 22 Dec 2019 00:06:18 +0100 Subject: [PATCH 045/235] fixed path scidash repo cloned --- service/docker/Dockerfile-virgo | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index 60c1b271..0ef62bff 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -89,9 +89,9 @@ RUN curl -L 'http://www.eclipse.org/downloads/download.php?file=/virgo/release/V RUN unzip virgo.zip RUN cp -r ./virgo-tomcat-server-3.7.2.RELEASE/* $VIRGO_DIR RUN rm $VIRGO_DIR/configuration/java-server.profile -RUN cp $APP_DIR/git/scidash/service/geppetto/java-server.profile $VIRGO_DIR/configuration/ +RUN cp /git/scidash/service/geppetto/java-server.profile $VIRGO_DIR/configuration/ RUN rm $VIRGO_DIR/configuration/tomcat-server.xml -RUN cat $APP_DIR/git/scidash/service/geppetto/tomcat-server.xml | sed 's/127.0.0.1/0.0.0.0/' > $VIRGO_DIR/configuration/tomcat-server.xml +RUN cat /git/scidash/service/geppetto/tomcat-server.xml | sed 's/127.0.0.1/0.0.0.0/' > $VIRGO_DIR/configuration/tomcat-server.xml RUN chmod u+x $VIRGO_DIR/bin/*.sh ENV SERVER_HOME $VIRGO_DIR RUN chmod 777 -R $VIRGO_DIR From 34fd5ca4aa1e213b9560bf717d23a92bf121b879 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sun, 22 Dec 2019 01:02:27 +0100 Subject: [PATCH 046/235] permission fix --- service/docker/Dockerfile-scidash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index cfe42d47..8dcc76a7 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -29,12 +29,12 @@ RUN pip install virtualenv RUN mkdir $APP_DIR RUN chown developer $APP_DIR RUN chown -R developer /home/developer/.config +RUN git clone https://github.com/ddelpiano/travis_utils +RUN cp travis_utils/copy.sh $APP_DIR USER developer # COPYING PROJECT -RUN git clone https://github.com/ddelpiano/travis_utils -RUN cp travis_utils/copy.sh $APP_DIR WORKDIR $APP_DIR RUN $APP_DIR/copy.sh https://github.com/Metacell/scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" From 4bdf399ddadc864b15293ef5143f394c812f3549 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Tue, 7 Jan 2020 12:57:04 +0100 Subject: [PATCH 047/235] debug to true, testing new django version --- scidash/main/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scidash/main/settings.py b/scidash/main/settings.py index 9d0b0b9c..cd8bc6ea 100644 --- a/scidash/main/settings.py +++ b/scidash/main/settings.py @@ -34,7 +34,7 @@ SECRET_KEY = '4*0@ca#ocm*(1=12m(bfb2p8e$sk-%i4xlj=%$wkj3*&gs!%sr' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = [ "*" From a98c432239eb7fedb1c5cad260b113a66029f817 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Sat, 11 Jan 2020 14:20:59 -0500 Subject: [PATCH 048/235] fake commit for tesing failing travis integration testing --- scidash/sciunittests/tests/test_sciunit_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scidash/sciunittests/tests/test_sciunit_tests.py b/scidash/sciunittests/tests/test_sciunit_tests.py index 5a404c13..b2815685 100644 --- a/scidash/sciunittests/tests/test_sciunit_tests.py +++ b/scidash/sciunittests/tests/test_sciunit_tests.py @@ -482,3 +482,4 @@ def test_is_score_class_match_the_same_object(self): parsed_response = response.json() self.assertEqual(len(parsed_response), 1) + self.assertEqual(1, 2) # test for travis integration test failing From cc3e3bb393c4a6ac27ab4271fce308778ea2d3ca Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Sat, 11 Jan 2020 14:49:00 -0500 Subject: [PATCH 049/235] chore: revert breaking test --- scidash/sciunittests/tests/test_sciunit_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scidash/sciunittests/tests/test_sciunit_tests.py b/scidash/sciunittests/tests/test_sciunit_tests.py index b2815685..5a404c13 100644 --- a/scidash/sciunittests/tests/test_sciunit_tests.py +++ b/scidash/sciunittests/tests/test_sciunit_tests.py @@ -482,4 +482,3 @@ def test_is_score_class_match_the_same_object(self): parsed_response = response.json() self.assertEqual(len(parsed_response), 1) - self.assertEqual(1, 2) # test for travis integration test failing From 8d5704567a8224c5475ab92a72d4c943c09e9046 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Sun, 12 Jan 2020 15:52:38 -0500 Subject: [PATCH 050/235] fix: docker file scidash set git email/name feat: add --if-exists to pg restore database to hide not existing errors in build log feat: created virgo base image build feat: enable unit/integration testing in Scidash fix: create new writable SerializerMethodField based for api deserialization of TestClass, fixes bug introduced by get_class_name method in serializer fix: TestClassSerializer if TestClass import path is None use '' test: add coverage to dev requirements and .coveragerc file feat: added shortcut for python manage command and added variables for easier Makefile maintenance fix: add missing packages and test coverage packages fix: add ignore venv to flake8 config file chore: add run-tests command to Makefile test: fix tests routines and data feat: add coverage command to Makefile to show test coverage scidash backend test: added test to check connection to Virgo server feat: added run-tests to pre-commit hook chore: update requirements.txt and pin psycopg2 to 2.7.7 chore: add ignore coverage to .gitignore test: test if geppetto servlet manager is correctly installed --- .coveragerc | 7 ++ .flake8 | 2 +- .gitignore | 3 +- .travis.yml | 2 +- Makefile | 29 ++++- requirements-dev.txt | 3 + requirements.in | 3 +- requirements.txt | 90 ++++---------- scidash/general/serializers.py | 47 +++++++- .../general/{tests.py => tests/__init__.py} | 0 .../general/tests/test_geppetto_servlet.py | 15 +++ .../tests/test_data/score_object.json | 32 ++++- .../tests/test_sciunit_models.py | 19 +-- scidash/sciunittests/serializers.py | 19 +-- .../tests/test_data/score_object.json | 56 +++++++-- .../tests/test_data/score_objects_list.json | 111 ++++++++++++++---- .../sciunittests/tests/test_sciunit_tests.py | 10 +- service/docker/Dockerfile-scidash | 3 + service/docker/Dockerfile-virgo | 16 +-- service/docker/build_all.sh | 2 + service/hooks/pre-commit | 2 + service/scripts/db-create-psql.sh | 2 +- 22 files changed, 322 insertions(+), 151 deletions(-) create mode 100644 .coveragerc rename scidash/general/{tests.py => tests/__init__.py} (100%) create mode 100644 scidash/general/tests/test_geppetto_servlet.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..14112662 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +plugins = + django_coverage_plugin +omit = + venv/* + static/* + */migrations/* diff --git a/.flake8 b/.flake8 index 8d74efc7..1495397e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 79 -exclude = .git,__pycache__,build,dist,.tox,*/migrations/*,env,__init__.py,settings.py,*/node_modules/*,routing.py +exclude = .git,__pycache__,build,dist,.tox,*/migrations/*,env,venv,__init__.py,settings.py,*/node_modules/*,routing.py diff --git a/.gitignore b/.gitignore index 8cca934a..035e3f04 100644 --- a/.gitignore +++ b/.gitignore @@ -39,9 +39,8 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt -# Unit test / coverage reports +# Unit tests / coverage reports htmlcov/ -.tox/ .coverage .coverage.* .cache diff --git a/.travis.yml b/.travis.yml index 7a3e94b4..0cf3ee88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ node_js: install: - npm install jest@24.8.0 puppeteer@1.17.0 jest-puppeteer@4.3.0 @babel/preset-env@7.4.5 url-join@4.0.0 @babel/core@7.4.5 script: - - echo -e "Trigger python unit testing in here when they will be ready" || true + - echo -e "python unit/integration testing running in geppetto-scidash travis" || true after_success: - echo $TRAVIS_BRANCH - echo $TRAVIS_PULL_REQUEST diff --git a/Makefile b/Makefile index c5d7f6de..44cda15c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,10 @@ + +VENV:=$(shell if [ -d "venv" ]; then echo "venv/bin/"; else echo ""; fi) +PYTHON:=$(VENV)python3 +PIP:=$(VENV)/bin/pip +MANAGE:="manage.py" +MANAGECMD=$(PYTHON) $(MANAGE) + install: create-db install-sciunit-neuronunit install-frontend install-backend @echo "===========================" @echo "= Finished =" @@ -63,19 +70,19 @@ run-staging: migrate django-migrate: migrations migrate migrations: - ./manage.py makemigrations + $(MANAGECMD) makemigrations migrate: - ./manage.py migrate + $(MANAGECMD) migrate superuser: - ./manage.py createsuperuser + $(MANAGECMD) createsuperuser run-django: - ./manage.py runserver + $(MANAGECMD) runserver run-django-staging: - python3.6 manage.py runserver --insecure 0.0.0.0:8000 + $(MANAGECMD) runserver --insecure 0.0.0.0:8000 run-frontend: cd static/org.geppetto.frontend/src/main/webapp/; npm run build-dev-noTest:watch; @@ -89,6 +96,18 @@ run-celery-beat: run-virgo-staging: /bin/bash /opt/virgo/bin/startup.sh +manage/%: + $(MANAGECMD) $* + +run-tests: run-django-tests + +run-django-tests: + $(MANAGECMD) test + +coverage: + coverage run $(MANAGE) test + coverage report -m + lint: flake8-lint isort-lint yapf-lint format: yapf-format isort-format diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e835830..deb0384c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,6 @@ flake8 yapf isort +coverage +django_coverage_plugin +pip-tools diff --git a/requirements.in b/requirements.in index 5c1d6477..be31ba8a 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ -psycopg2 django==1.11.23 +psycopg2==2.7.7 channels==2.1.2 djangorestframework==3.7.1 drf-writable-nested @@ -14,3 +14,4 @@ django-celery-beat django-celery-results django-db-logger git+git://github.com/ddelpiano/neuronunit@4.0.0#egg=neuronunit +wheel==0.33.6 diff --git a/requirements.txt b/requirements.txt index b36df93f..36e7dc7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,45 +2,24 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile +# pip-compile requirements.in # -airspeed==0.5.4dev-20150515 -allensdk==0.14.5 amqp==2.4.2 # via kombu -apipkg==1.5 # via execnet -argparse==1.4.0 # via scoop asgiref==2.3.2 # via channels -asteval==0.9.18 # via lmfit async-timeout==3.0.1 # via asgiref -attrs==19.1.0 # via automat, jsonschema, twisted +attrs==19.1.0 # via automat, twisted autobahn==19.3.3 # via daphne automat==0.7.0 # via twisted -backcall==0.1.0 # via ipython -backports.tempfile==1.0 -backports.weakref==1.0.post1 # via backports.tempfile -beautifulsoup4==4.8.2 # via bs4 billiard==3.5.0.5 # via celery -bleach==3.1.0 # via nbconvert -git+https://github.com/russelljjarvis/BluePyOpt@scidash#egg=bluepyopt-9999 -bokeh==1.4.0 # via dask -bs4==0.0.1 -cachetools==0.8.0 # via airspeed celery[redis]==4.2.2 -cerberus==1.3.2 -certifi==2019.11.28 # via requests channels==2.1.2 -chardet==3.0.4 # via requests -click==7.0 # via distributed -cloudpickle==1.2.2 # via dask, distributed constantly==15.1.0 # via twisted -cycler==0.10.0 # via matplotlib -cypy==0.2.0 daphne==2.2.5 # via channels -dask[complete]==2.9.1 # via distributed +dask[complete]==2.9.0 # via distributed deap==1.3.0 decorator==4.4.1 # via ipyparallel, ipython, networkx, traitlets, validators defusedxml==0.6.0 # via nbconvert -distributed==2.9.1 # via dask +distributed==2.9.0 # via dask django-celery-beat==1.4.0 django-celery-results==1.0.4 django-db-logger==0.1.7 @@ -49,30 +28,21 @@ django-extensions==2.1.6 django-filter==1.1.0 django-material==1.5.2 django-timezone-field==3.0 # via django-celery-beat -django==1.11.23 +django==1.11.7 djangorestframework-jwt==1.11.0 djangorestframework==3.7.1 drf-writable-nested==0.5.1 -efel==3.0.70 -elephant==0.4.1 -entrypoints==0.3 # via nbconvert -execnet==1.7.1 -fsspec==0.6.2 # via dask -future==0.18.2 # via allensdk -gitdb2==2.0.6 # via gitpython -gitpython==3.0.5 -greenlet==0.4.15 # via scoop -h5py==2.10.0 # via allensdk -heapdict==1.0.1 # via zict +git+git://github.com/ddelpiano/neuronunit@4.0.0#egg=neuronunit hyperlink==18.0.0 # via twisted idna==2.8 # via hyperlink, requests igor==0.3 imageio==2.6.1 # via scikit-image +importlib-metadata==1.3.0 # via jsonschema incremental==17.5.0 # via twisted ipykernel==5.1.3 # via ipyparallel ipyparallel==6.2.4 ipython-genutils==0.2.0 # via ipyparallel, nbformat, traitlets -ipython==7.11.1 # via ipykernel, ipyparallel +ipython==7.10.2 # via ipykernel, ipyparallel jedi==0.15.2 # via ipython jinja2==2.10.3 # via allensdk, bokeh, nbconvert jsonschema==3.2.0 # via nbformat @@ -89,6 +59,7 @@ lxml==4.4.2 # via libneuroml markupsafe==1.1.1 # via jinja2 matplotlib==3.1.2 # via allensdk, scikit-image mistune==0.8.4 # via nbconvert +more-itertools==8.0.2 # via zipp msgpack==0.6.2 # via distributed nbconvert==5.6.1 nbformat==4.4.0 # via nbconvert @@ -106,54 +77,33 @@ partd==1.1.0 # via dask patsy==0.5.1 # via statsmodels pexpect==4.7.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==7.0.0 # via bokeh, imageio, scikit-image +pillow==6.2.1 # via bokeh, imageio, scikit-image prompt-toolkit==3.0.2 # via ipython psutil==5.6.7 # via distributed psycopg2==2.7.7 -ptyprocess==0.6.0 # via pexpect -pygments==2.5.2 # via ipython, nbconvert pyhamcrest==1.9.0 # via twisted pyjwt==1.7.1 # via djangorestframework-jwt pylems==0.4.9.3 git+https://github.com/rgerkin/pyneuroml@master#egg=pyneuroml-9999 pynn==0.9.5 -pynrrd==0.4.2 # via allensdk +pynrrd==0.4.1 # via allensdk pyparsing==2.4.6 # via matplotlib, packaging pyrsistent==0.15.6 # via jsonschema python-crontab==2.3.6 # via django-celery-beat -python-dateutil==2.8.0 # via bokeh, ipyparallel, jupyter-client, matplotlib, pandas, python-crontab -pytz==2018.9 # via celery, django, django-timezone-field, pandas -pywavelets==1.1.1 # via scikit-image -pyyaml==5.2 # via bokeh, dask, distributed -pyzmq==18.1.1 # via ipyparallel, jupyter-client, scoop -quantities==0.12.1 # via elephant, neo, pynn +python-dateutil==2.8.0 # via python-crontab +pytz==2018.9 # via celery, django, django-timezone-field redis==2.10.6 # via celery -requests-toolbelt==0.9.1 # via allensdk -requests==2.22.0 # via allensdk, requests-toolbelt rest-framework-cache==0.1 -scikit-image==0.16.2 # via allensdk -scipy==1.4.1 # via allensdk, elephant, lmfit, scikit-image, statsmodels -git+https://github.com/scidash/sciunit@dev#egg=sciunit-9999 -scoop==0.7.1.1 -simpleitk==1.2.4 # via allensdk -simplejson==3.17.0 # via allensdk -six==1.12.0 # via airspeed, allensdk, autobahn, automat, bleach, bokeh, cycler, django-extensions, efel, elephant, h5py, jsonschema, libneuroml, packaging, patsy, pyhamcrest, pyrsistent, python-dateutil, traitlets, txaio, validators -smmap2==2.0.5 # via gitdb2 -sortedcontainers==2.1.0 # via distributed -soupsieve==1.9.5 # via beautifulsoup4 -statsmodels==0.10.2 # via allensdk -tblib==1.6.0 # via distributed -testpath==0.4.4 # via nbconvert -toolz==0.10.0 # via dask, distributed, partd -tornado==6.0.3 # via bokeh, distributed, ipykernel, ipyparallel, jupyter-client -traitlets==4.3.3 # via ipykernel, ipyparallel, ipython, jupyter-client, jupyter-core, nbconvert, nbformat +six==1.12.0 # via autobahn, automat, django-extensions, pyhamcrest, python-dateutil, txaio twisted==18.9.0 # via daphne txaio==18.8.1 # via autobahn -uncertainties==3.1.2 # via lmfit -urllib3==1.25.7 # via requests -validators==0.14.1 vine==1.3.0 # via amqp -wcwidth==0.1.8 # via prompt-toolkit +wcwidth==0.1.7 # via prompt-toolkit webencodings==0.5.1 # via bleach +wheel==0.33.6 zict==1.0.0 # via distributed +zipp==0.6.0 # via importlib-metadata zope.interface==4.6.0 # via twisted + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/scidash/general/serializers.py b/scidash/general/serializers.py index 6a68ddff..16f97446 100644 --- a/scidash/general/serializers.py +++ b/scidash/general/serializers.py @@ -1,4 +1,4 @@ -from rest_framework import serializers +from rest_framework import serializers, fields from scidash.general.models import Tag @@ -7,3 +7,48 @@ class TagSerializer(serializers.ModelSerializer): class Meta: model = Tag fields = ('name', ) + + +class SerializerWritableMethodField(fields.ModelField): + """ + A writable (ModelField base) SerializerMethodField that get its + representation from calling a method on the parent serializer class. The + method called will be of the form "get_{field_name}", and should take a + single argument, which is the object being serialized. + + For example: + + class ExampleSerializer(self): + class_name = SciDashSerializerWritableMethodField( + model_field=TestClass()._meta.get_field('class_name')) + + def get_class_name(self, obj): + return ... # Calculate some data to return. + """ + + def __init__(self, method_name=None, **kwargs): + self.method_name = method_name + super(SerializerWritableMethodField, self).__init__(**kwargs) + + def bind(self, field_name, parent): + # In order to enforce a consistent style, we error if a redundant + # 'method_name' argument has been used. For example: + # my_fld = serializer.SerializerMethodField(method_name='get_my_fld') + default_method_name = 'get_{field_name}'.format(field_name=field_name) + assert self.method_name != default_method_name, ( + "It is redundant to specify `%s` on SerializerMethodField '%s' in " + "serializer '%s', because it is the same as the default method " + "name. Remove the `method_name` argument." % + (self.method_name, field_name, parent.__class__.__name__) + ) + + # The method name should default to `get_{field_name}`. + if self.method_name is None: + self.method_name = default_method_name + + super(SerializerWritableMethodField, self).bind( + field_name, parent) + + def to_representation(self, value): + method = getattr(self.parent, self.method_name) + return method(value) diff --git a/scidash/general/tests.py b/scidash/general/tests/__init__.py similarity index 100% rename from scidash/general/tests.py rename to scidash/general/tests/__init__.py diff --git a/scidash/general/tests/test_geppetto_servlet.py b/scidash/general/tests/test_geppetto_servlet.py new file mode 100644 index 00000000..82b8df46 --- /dev/null +++ b/scidash/general/tests/test_geppetto_servlet.py @@ -0,0 +1,15 @@ +from django.test import TestCase + +from pygeppetto_gateway.base import GeppettoServletManager + + +class GeppettoServletTest(TestCase): + @classmethod + def setUpClass(cls): + super(GeppettoServletTest, cls).setUpClass() + cls.servlet_manager = GeppettoServletManager() + + def test_ws_address(self): + self.assertEqual( + self.servlet_manager.host, + "ws://scidash-virgo:8080/org.geppetto.frontend/GeppettoServlet") diff --git a/scidash/sciunitmodels/tests/test_data/score_object.json b/scidash/sciunitmodels/tests/test_data/score_object.json index 70ad446b..61ded11f 100644 --- a/scidash/sciunitmodels/tests/test_data/score_object.json +++ b/scidash/sciunitmodels/tests/test_data/score_object.json @@ -1,11 +1,29 @@ { - "score_class": {"class_name": "ZScore", "url": "http://test-url.for/not-spaming-data/in-database"}, + "score_class": { + "class_name": "ZScore", + "url": "http://test-url.for/not-spaming-data/in-database" + }, "hash_id": "111", "model_instance": { "model_class": { "class_name": "ReducedModel", "url": "http://test-url.for/not-spaming-data/in-database", + "import_path": "neuronunit.models.static.StaticModel", + "memo": null, + "extra_capabilities": [], "capabilities": [ + { + "class_name": "ReceivesSquareCurrent" + }, + { + "class_name": "ProducesActionPotentials" + }, + { + "class_name": "ProducesSpikes" + }, + { + "class_name": "ProducesMembranePotential" + }, { "class_name": "CanBeReduced" } @@ -25,16 +43,18 @@ "test_instance": { "description": null, "hash_id": "111", - "test_suites": [{ - "hash": "testhash", - "name": "ReducedSuite" - }], + "test_suites": [ + { + "hash": "testhash", + "name": "ReducedSuite" + } + ], "test_class": { "class_name": "MyTest", "url": "http://test-url.for/not-spaming-data/in-database" }, "observation": { - "mean":"8", + "mean": "8", "std": "3", "url": "" }, diff --git a/scidash/sciunitmodels/tests/test_sciunit_models.py b/scidash/sciunitmodels/tests/test_sciunit_models.py index d0b825ac..4bc1806c 100644 --- a/scidash/sciunitmodels/tests/test_sciunit_models.py +++ b/scidash/sciunitmodels/tests/test_sciunit_models.py @@ -64,7 +64,7 @@ def test_if_capabilities_endpoint_works_correctly(self): parsed_response = parsed_response.pop() self.scrub(parsed_response, 'id') capabilities_data = data.get('model_instance') \ - .get('model_class').get('capabilities').pop() + .get('model_class').get('capabilities').pop() for key in capabilities_data.keys(): self.assertTrue(key in parsed_response) @@ -86,7 +86,7 @@ def test_if_model_class_endpoint_works_correctly(self): parsed_response = parsed_response.pop() self.scrub(parsed_response, 'id') model_class_data = data.get('model_instance') \ - .get('model_class') + .get('model_class') for key in model_class_data.keys(): self.assertTrue(key in parsed_response) @@ -118,18 +118,17 @@ def test_if_model_instance_endpoint_works_correctly(self): class SciunitModelMatchingClassObjects(TestCase): @classmethod - def setUpClass(cls): - super(SciunitModelMatchingClassObjects, cls).setUpClass() - - cls.model_class = { + def setUp(self): + self.model_class = { "class_name": "ScoreModelClass", "capabilities": [{ "class_name": "TestCapability" }], + "import_path": "neuronunit.models.static.StaticModel", "url": "http://test-score.url" } - cls.user = ScidashUser.objects.create_user( + self.user = ScidashUser.objects.create_user( 'admin', 'a@a.cc', 'montecarlo' ) @@ -139,12 +138,6 @@ def test_is_model_class_match_the_same_object(self): model_class_serializer = ModelClassSerializer(data=self.model_class) - if model_class_serializer.is_valid(): - model_class_serializer.save() - - model_class_serializer = None - model_class_serializer = ModelClassSerializer(data=self.model_class) - if model_class_serializer.is_valid(): model_class_serializer.save() diff --git a/scidash/sciunittests/serializers.py b/scidash/sciunittests/serializers.py index 97f369dc..ef55915b 100644 --- a/scidash/sciunittests/serializers.py +++ b/scidash/sciunittests/serializers.py @@ -2,13 +2,14 @@ import numpy as np from drf_writable_nested import WritableNestedModelSerializer -from rest_framework import serializers +from rest_framework import serializers, fields import sciunit from scidash.account.serializers import ScidashUserSerializer from scidash.general.helpers import import_class from scidash.general.mixins import GetByKeyOrCreateMixin, GetOrCreateMixin -from scidash.general.serializers import TagSerializer +from scidash.general.serializers import TagSerializer, \ + SerializerWritableMethodField from scidash.sciunitmodels.serializers import ModelInstanceSerializer from scidash.sciunittests.helpers import build_destructured_unit from scidash.sciunittests.models import ( @@ -27,18 +28,18 @@ class Meta: class TestClassSerializer( - GetByKeyOrCreateMixin, WritableNestedModelSerializer -): - class_name = serializers.SerializerMethodField() + GetByKeyOrCreateMixin, WritableNestedModelSerializer): + class_name = SerializerWritableMethodField( + model_field=TestClass()._meta.get_field('class_name')) units_name = serializers.CharField(required=False) key = 'import_path' def get_class_name(self, obj): # return class_name + ( first part of import_path ) - return obj.class_name + \ - (' (' + - '.'.join(obj.import_path.split('.')[0:-1]) - + ')').replace(' ()', '') + return obj.class_name + ( + ' (' + + '.'.join((obj.import_path if obj.import_path else '' + ).split('.')[0:-1]) + ')').replace(' ()', '') class Meta: model = TestClass diff --git a/scidash/sciunittests/tests/test_data/score_object.json b/scidash/sciunittests/tests/test_data/score_object.json index 70ad446b..f40e9ba6 100644 --- a/scidash/sciunittests/tests/test_data/score_object.json +++ b/scidash/sciunittests/tests/test_data/score_object.json @@ -1,11 +1,34 @@ { - "score_class": {"class_name": "ZScore", "url": "http://test-url.for/not-spaming-data/in-database"}, + "score_class": { + "class_name": "ZScore", + "url": "http://test-url.for/not-spaming-data/in-database" + }, "hash_id": "111", "model_instance": { "model_class": { "class_name": "ReducedModel", "url": "http://test-url.for/not-spaming-data/in-database", + "import_path": "neuronunit.models.reduced.ReducedModel", + "memo": null, + "extra_capabilities": [ + 24 + ], "capabilities": [ + { + "class_name": "Runnable" + }, + { + "class_name": "ReceivesSquareCurrent" + }, + { + "class_name": "ProducesActionPotentials" + }, + { + "class_name": "ProducesSpikes" + }, + { + "class_name": "ProducesMembranePotential" + }, { "class_name": "CanBeReduced" } @@ -14,6 +37,8 @@ "attributes": {}, "hash_id": "111", "name": "Izhikevich", + "status": "l", + "tags": [], "run_params": {}, "url": "https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml", "backend": "JNeuroML" @@ -23,21 +48,34 @@ "sort_key": 0.738882680363527, "score_type": "ZType", "test_instance": { + "name": "Default Name", "description": null, "hash_id": "111", - "test_suites": [{ - "hash": "testhash", - "name": "ReducedSuite" - }], + "status": "l", + "tags": [], + "params": null, + "verbose": 1, + "test_suites": [ + { + "hash": "testhash", + "name": "ReducedSuite" + } + ], "test_class": { "class_name": "MyTest", - "url": "http://test-url.for/not-spaming-data/in-database" + "url": "http://test-url.for/not-spaming-data/in-database", + "import_path": null, + "observation_schema": null, + "test_parameters_schema": null, + "units": null, + "memo": null, + "params_units": null, + "default_params": null }, "observation": { - "mean":"8", + "mean": "8", "std": "3", "url": "" - }, - "verbose": 1 + } } } diff --git a/scidash/sciunittests/tests/test_data/score_objects_list.json b/scidash/sciunittests/tests/test_data/score_objects_list.json index 9ea540a3..f102176d 100644 --- a/scidash/sciunittests/tests/test_data/score_objects_list.json +++ b/scidash/sciunittests/tests/test_data/score_objects_list.json @@ -1,14 +1,37 @@ [ { - "score_class": {"class_name": "ZScore", "url": "http://test-url.for/not-spaming-data/in-database"}, + "score_class": { + "class_name": "ZScore", + "url": "http://test-url.for/not-spaming-data/in-database" + }, "hash_id": "111", "model_instance": { "model_class": { "class_name": "ReducedModel_1", "url": "http://test-url.for/not-spaming-data/in-database-one", + "import_path": "neuronunit.models.reduced.ReducedModel", + "memo": null, + "extra_capabilities": [ + 14 + ], "capabilities": [ { - "class_name": "CanBeReduced_1" + "class_name": "Runnable" + }, + { + "class_name": "ReceivesSquareCurrent" + }, + { + "class_name": "ProducesActionPotentials" + }, + { + "class_name": "ProducesSpikes" + }, + { + "class_name": "ProducesMembranePotential" + }, + { + "class_name": "CanBeReduced" } ] }, @@ -16,6 +39,8 @@ "attributes": {}, "name": "Izhikevich", "run_params": {}, + "status": "l", + "tags": [], "url": "https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml", "backend": "JNeuroML" }, @@ -25,16 +50,18 @@ "score_type": "ZType_1", "test_instance": { "description": null, - "test_suites": [{ - "hash": "testhash", - "name": "ReducedSuite_1" - }], + "test_suites": [ + { + "hash": "testhash", + "name": "ReducedSuite_1" + } + ], "test_class": { "class_name": "MyTest_1", "url": "http://test-url.for/not-spaming-data/in-database" }, "observation": { - "mean":"8", + "mean": "8", "std": "3", "url": "" }, @@ -43,15 +70,33 @@ } }, { - "score_class": {"class_name": "ZScore", "url": "http://test-url.for/not-spaming-data/in-database"}, + "score_class": { + "class_name": "ZScore", + "url": "http://test-url.for/not-spaming-data/in-database" + }, "hash_id": "222", "model_instance": { "model_class": { "class_name": "ReducedModel_2", "url": "http://test-url.for/not-spaming-data/in-database-two", + "import_path": "neuronunit.models.static.StaticModel", + "memo": null, + "extra_capabilities": [], "capabilities": [ { - "class_name": "CanBeReduced_2" + "class_name": "ReceivesSquareCurrent" + }, + { + "class_name": "ProducesActionPotentials" + }, + { + "class_name": "ProducesSpikes" + }, + { + "class_name": "ProducesMembranePotential" + }, + { + "class_name": "CanBeReduced" } ] }, @@ -59,6 +104,8 @@ "attributes": {}, "name": "Izhikevich", "run_params": {}, + "status": "l", + "tags": [], "url": "https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml", "backend": "JNeuroML" }, @@ -69,16 +116,18 @@ "test_instance": { "description": null, "hash_id": "111", - "test_suites": [{ - "hash": "testhash", - "name": "ReducedSuite_2" - }], + "test_suites": [ + { + "hash": "testhash", + "name": "ReducedSuite_2" + } + ], "test_class": { "class_name": "MyTest_2", "url": "http://test-url.for/not-spaming-data/in-database" }, "observation": { - "mean":"8", + "mean": "8", "std": "3", "url": "" }, @@ -86,13 +135,31 @@ } }, { - "score_class": {"class_name": "ZScore", "url": "http://test-url.for/not-spaming-data/in-database"}, + "score_class": { + "class_name": "ZScore", + "url": "http://test-url.for/not-spaming-data/in-database" + }, "hash_id": "333", "model_instance": { "model_class": { "class_name": "ReducedModel", "url": "http://test-url.for/not-spaming-data/in-database-three", + "import_path": "neuronunit.models.static.StaticModel", + "memo": null, + "extra_capabilities": [], "capabilities": [ + { + "class_name": "ReceivesSquareCurrent" + }, + { + "class_name": "ProducesActionPotentials" + }, + { + "class_name": "ProducesSpikes" + }, + { + "class_name": "ProducesMembranePotential" + }, { "class_name": "CanBeReduced" } @@ -102,6 +169,8 @@ "hash_id": "311", "name": "Izhikevich", "run_params": {}, + "status": "l", + "tags": [], "url": "https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml", "backend": "JNeuroML" }, @@ -112,16 +181,18 @@ "test_instance": { "description": null, "hash_id": "111", - "test_suites": [{ - "hash": "testhash", - "name": "ReducedSuite" - }], + "test_suites": [ + { + "hash": "testhash", + "name": "ReducedSuite" + } + ], "test_class": { "class_name": "MyTest", "url": "http://test-url.for/not-spaming-data/in-database" }, "observation": { - "mean":"8", + "mean": "8", "std": "3", "url": "" }, diff --git a/scidash/sciunittests/tests/test_sciunit_tests.py b/scidash/sciunittests/tests/test_sciunit_tests.py index 5a404c13..2c239219 100644 --- a/scidash/sciunittests/tests/test_sciunit_tests.py +++ b/scidash/sciunittests/tests/test_sciunit_tests.py @@ -12,12 +12,11 @@ ) SAMPLE_OBJECT = os.path.join( - os.path.dirname(os.path.dirname(__file__)), 'test_data/score_object.json' + os.path.dirname(__file__), 'test_data/score_object.json' ) SAMPLE_OBJECT_LIST = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - 'test_data/score_objects_list.json' + os.path.dirname(__file__), 'test_data/score_objects_list.json' ) @@ -78,8 +77,10 @@ def test_if_scores_endpoint_works_correctly(self): self.scrub(parsed_response, 'build_info') self.scrub(parsed_response, 'hostname') self.scrub(parsed_response, 'owner') + self.scrub(parsed_response, 'units_name') parsed_keys = parsed_response.keys() + self.maxDiff=None for key in data.keys(): self.assertTrue(key in parsed_keys) self.assertEqual(data.get(key), parsed_response.get(key)) @@ -100,6 +101,7 @@ def test_if_test_instance_endpoint_works_correctly(self): self.scrub(parsed_response, 'id') self.scrub(parsed_response, 'timestamp') self.scrub(parsed_response, 'owner') + self.scrub(parsed_response, 'units_name') parsed_keys = parsed_response.keys() test_instance_data = data.get('test_instance') @@ -234,7 +236,7 @@ def test_scores_endpoint_filters_get_by_class_name(self): parsed_response = response.json() first_element = parsed_response[2] model_class_name = first_element.get('model_instance') \ - .get('model_class').get('class_name') + .get('model_class').get('class_name') filtered_url = '{}?model={}'.format( reverse('score-list'), model_class_name diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index 8dcc76a7..412553a4 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -36,6 +36,9 @@ USER developer # COPYING PROJECT WORKDIR $APP_DIR +# set git email and name for use with git merge +RUN git config --global user.email "scidash@metacell.us" +RUN git config --global user.name "SciDash" RUN $APP_DIR/copy.sh https://github.com/Metacell/scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" diff --git a/service/docker/Dockerfile-virgo b/service/docker/Dockerfile-virgo index 0ef62bff..abf7aff4 100644 --- a/service/docker/Dockerfile-virgo +++ b/service/docker/Dockerfile-virgo @@ -100,48 +100,48 @@ RUN mvn --version # -== INSTALL GEPPETTO ==- WORKDIR $SOURCES_DIR -RUN $APP_DIR/copy.sh http://github.com/openworm/org.geppetto.git "${targetBranch}" "${originBranch}" "${defaultBranch}" +RUN $APP_DIR/copy.sh http://github.com/openworm/org.geppetto.git geppetto-scidash "${originBranch}" "${defaultBranch}" RUN cp /git/scidash/service/geppetto/config.json $SOURCES_DIR/org.geppetto/utilities/source_setup RUN cp /git/scidash/service/geppetto/setup.py $SOURCES_DIR/org.geppetto/utilities/source_setup RUN cp /git/scidash/service/geppetto/geppetto.plan $SOURCES_DIR/org.geppetto/ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.model &&\ /bin/echo -e "\e[96mMaven install org.geppetto.model\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.core.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.core.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.core && cp /git/scidash/service/geppetto/core/app-config.xml ./src/main/java/META-INF/spring &&\ /bin/echo -e "\e[96mMaven install org.geppetto.core\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.neuroml.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.model.neuroml.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.model.neuroml &&\ /bin/echo -e "\e[96mMaven install org.geppetto.model.neuroml\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulation.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulation.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.simulation &&\ /bin/echo -e "\e[96mMaven install org.geppetto.simulation\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulator.external.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.simulator.external.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.simulator.external &&\ /bin/echo -e "\e[96mMaven install org.geppetto.simulator.external\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/Metacell/org.geppetto.simulator.scidash.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/Metacell/org.geppetto.simulator.scidash.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.simulator.scidash &&\ /bin/echo -e "\e[96mMaven install org.geppetto.simulator.scidash\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ rm -rf src && cd ../ -RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.frontend.git "${targetBranch}" "${originBranch}" "${defaultBranch}" &&\ +RUN $APP_DIR/copy.sh https://github.com/openworm/org.geppetto.frontend.git geppetto-scidash "${originBranch}" "${defaultBranch}" &&\ cd org.geppetto.frontend && cp /git/scidash/service/geppetto/frontend/app-config.xml ./src/main/webapp/WEB-INF/spring/ &&\ /bin/echo -e "\e[96mMaven install org.geppetto.frontend\e[0m" &&\ mvn -Dhttps.protocols=TLSv1.2 -DskipTests --quiet install &&\ diff --git a/service/docker/build_all.sh b/service/docker/build_all.sh index 7f0e00f7..da7cf182 100755 --- a/service/docker/build_all.sh +++ b/service/docker/build_all.sh @@ -1,5 +1,7 @@ #!/bin/bash ./build_database.sh +./build_virgo_base.sh ./build_virgo.sh ./build_scidash.sh + diff --git a/service/hooks/pre-commit b/service/hooks/pre-commit index 963e6a18..f8603d23 100644 --- a/service/hooks/pre-commit +++ b/service/hooks/pre-commit @@ -3,3 +3,5 @@ set -e make lint + +make run-tests diff --git a/service/scripts/db-create-psql.sh b/service/scripts/db-create-psql.sh index 92c4628b..c26decd4 100755 --- a/service/scripts/db-create-psql.sh +++ b/service/scripts/db-create-psql.sh @@ -21,4 +21,4 @@ cd /tmp git clone https://github.com/ddelpiano/scidash-artifacts cd scidash-artifacts/database gunzip `ls | head -n 1` -pg_restore --clean -d scidash `ls *db | head -n 1` || true +pg_restore --clean --if-exists -d scidash `ls *db | head -n 1` || true From 2925bf6f0115e4f300e1eff8e5121b9d07407c8c Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Sun, 12 Jan 2020 16:33:06 -0500 Subject: [PATCH 051/235] feat: add master and development badge to readme.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bffc5230..de034475 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://img.shields.io/travis/MetaCell/scidash/master.svg?style=flat-square&label=master)](https://travis-ci.org/MetaCell/scidash) +[![Build Status](https://img.shields.io/travis/MetaCell/scidash/development.svg?style=flat-square&label=develop)](https://travis-ci.org/MetaCell/scidash)

SciDash logo

From 047369d2ddd031913b467b049839b1945036efc7 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Mon, 13 Jan 2020 09:50:31 -0500 Subject: [PATCH 052/235] chore: removed copy.sh from docker folder sinces it's not part of this repo --- .env | 22 ++++++++++++---- service/docker/copy.sh | 58 ------------------------------------------ 2 files changed, 17 insertions(+), 63 deletions(-) delete mode 100755 service/docker/copy.sh diff --git a/.env b/.env index 31a14221..f08e5122 100644 --- a/.env +++ b/.env @@ -1,11 +1,23 @@ DEVELOPMENT=1 PRODUCTION=0 -DB_HOST='localhost' +DB_HOST=scidash-postgres DB_PORT=5432 -DB_NAME='scidash' +DB_NAME=scidash +DB_USER=scidash_admin +DB_PASSWORD=scidash_local_password -REDIS_URL='redis://' +STATIC_URL=/static/ +STATIC_DIR=static -STATIC_URL='/static/' -STATIC_DIR='static' +REDIS_URL=redis://scidash-redis:6379 + +# for localhost +# GEPPETTO_HOST=localhost +# for docker +GEPPETTO_HOST=scidash-virgo +GEPPETTO_PORT=8080 +GEPPETTO_SERVLET_URL=ws://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/GeppettoServlet +GEPPETTO_BASE_URL=http://${GEPPETTO_HOST}:${GEPPETTO_PORT}/org.geppetto.frontend/geppetto + +BASE_PROJECT_FILES_HOST='http://172.21.0.1:8000/static/projects/' diff --git a/service/docker/copy.sh b/service/docker/copy.sh deleted file mode 100755 index 03655026..00000000 --- a/service/docker/copy.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -clone_url=$1 -target_branch=$2 -origin_branch=$3 -default_branch=$4 - -################################################################################################# -# -# Use it for copying repositories: -# - org.geppetto.frontend -# - org.geppetto.model -# - org.geppetto.core -# - etc -# -# It requires: -# - target_branch: main branch to test ($TRAVIS_BRANCH) -# - origin_branch: branch where the new code was written ($TRAVIS_PULL_REQUEST_BRANCH) -# - default_branch: branch to use as main branch in case $main_branch does not exist -# -################################################################################################# - -name=`echo $clone_url | sed 's~^.*/~~g' | sed 's~.git~~g'` -mkdir $name - -if [ -z $origin_branch ]; then - git ls-remote --heads --tags $clone_url | grep -E 'refs/(heads|tags)/'$target_branch > /dev/null - if [ $? -eq 0 ]; then - download_url=`echo $clone_url | sed 's~\.git~/archive/'$target_branch'.zip~g'` - /bin/echo -e "\e[1;35mDownloading <$target_branch> branch for $name.\e[0m" - else - download_url=`echo $clone_url | sed 's~\.git~/archive/'$default_branch'.zip~g'` - /bin/echo -e "\e[1;35mDownloading <$default_branch> branch for $name.\e[0m" - fi - curl -sL $download_url | bsdtar -xzf - -C $name --strip-components 1 -else - # clone repository - /bin/echo -e "\e[1;35mCloning <$default_branch> branch for <$name> repository.\e[0m" - git clone -b $default_branch $clone_url - cd $name - # check if target_branch exists in the repo and ckeckout default_branch if it does not - git show-ref | grep -E 'refs/remotes/origin/'$target_branch > /dev/null - if [ $? -eq 0 ]; then - /bin/echo -e "\e[1;35mCheckout <$target_branch> branch.\e[0m" - git checkout $target_branch - else - /bin/echo -e "\e[1;35m<$target_branch> branch does not exist in <$name> repository, we stay on <$default_branch> branch.\e[0m" - fi - # check if origin_branch exists in the repo and merge origin_branch in target branch if it does - git show-ref | grep -E 'refs/remotes/origin/'$origin_branch > /dev/null - if [ $? -eq 0 ]; then - /bin/echo -e "\e[1;35mMerging <$origin_branch> branch into (<$target_branch>|<$default_branch>) for <$name> repository.\e[0m" - git merge --no-edit --no-stat --no-summary --quiet --no-progress origin/$origin_branch - else - /bin/echo -e "\e[1;35m<$origin_branch> branch does not exist in <$name> repository, no merge will be performed on this repository.\e[0m" - fi - rm -rf .git - cd .. -fi From 92dbeb21623f6d6ac216f88f0600b028c0f0b088 Mon Sep 17 00:00:00 2001 From: jrmartin Date: Mon, 13 Jan 2020 18:41:51 -0800 Subject: [PATCH 053/235] Re-arranging templates order, to get rid off jquery reference errors --- scidash/account/templates/registration/login.html | 3 +-- .../account/templates/registration/password-reset-done.html | 4 ++-- scidash/account/templates/registration/password-reset.html | 2 +- scidash/account/templates/registration/signup.html | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scidash/account/templates/registration/login.html b/scidash/account/templates/registration/login.html index 01f13a6a..40bf4f87 100644 --- a/scidash/account/templates/registration/login.html +++ b/scidash/account/templates/registration/login.html @@ -1,8 +1,7 @@ +{% include 'registration/base-template.html' %} {% include 'material/includes/material_css.html' %} {% include 'material/includes/material_js.html' %} -{% include 'registration/base-template.html' %} - {% load material_form %} {% block content %} diff --git a/scidash/account/templates/registration/password-reset-done.html b/scidash/account/templates/registration/password-reset-done.html index 7e4bc432..2dd50d8b 100644 --- a/scidash/account/templates/registration/password-reset-done.html +++ b/scidash/account/templates/registration/password-reset-done.html @@ -1,9 +1,9 @@ +{% include 'registration/base-template.html' %} {% include 'material/includes/material_css.html' %} {% include 'material/includes/material_js.html' %} -{% include 'registration/base-template.html' %} {% block content %} -
+