From 4f86b0d6c206488dc813644374f4ceb949c05b33 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 11:58:44 -0500 Subject: [PATCH 01/41] adding circleci to test --- .circleci/config.yml | 195 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..94e81806 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,195 @@ +# CircleCI build config to test different versions of Singuarity + +version: 2.1 + +workflows: + version: 2 + test: + jobs: + - test-singularity-3.1.0-python-3: + filters: + branches: + only: master + - test-singularity-3.1.0-python-2: + filters: + branches: + only: master + - test-singularity-2.6.1-python-2: + filters: + branches: + only: master + - test-singularity-2.6.1-python-3: + filters: + branches: + only: master + + +setup: &setup + name: install setup + command: | + sudo apt-get update && \ + sudo apt-get install -y wget git jq + sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers + + +install_spython: &install_spython + name: install spython + command: | + $HOME/conda/bin/python setup.py install + + +install_singularity_2: &install_singularity_2 + name: install Singularity 2.6.1 + command: | + which singularity &> /dev/null + + if [ $? -eq 0 ]; then + echo "Singularity is installed." + else + sudo apt-get update && sudo apt-get -y install git \ + build-essential \ + libtool \ + squashfs-tools \ + autotools-dev \ + libarchive-dev \ + automake \ + autoconf \ + debootstrap \ + yum \ + uuid-dev \ + libssl-dev \ + python3-dev + wget https://github.com/sylabs/singularity/releases/download/2.6.1/singularity-2.6.1.tar.gz + tar -xzvf singularity-2.6.1.tar.gz + cd singularity-2.6.1 && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install + + +install_singularity_3: &install_singularity_3 + name: install Singularity 3.1 + command: | + which singularity &> /dev/null + + if [ $? -eq 0 ]; then + echo "Singularity is installed." + export GOPATH=/go + export PATH=$PATH:/usr/local/go/bin + else + + sudo apt-get install -y build-essential \ + squashfs-tools \ + libtool \ + uuid-dev \ + libssl-dev \ + libgpgme11-dev \ + libseccomp-dev \ + pkg-config + + # Install GoLang + wget https://dl.google.com/go/go${GO_VERSION}.src.tar.gz && \ + tar -C /usr/local -xzf go${GO_VERSION}.src.tar.gz + + export PATH=$PATH:/usr/local/go/bin && \ + sudo mkdir -p /go && \ + sudo chmod -R 7777 /go + export GOPATH=/go && \ + go get -u github.com/golang/dep/cmd/dep && \ + mkdir -p ${GOPATH}/src/github.com/sylabs && \ + cd ${GOPATH}/src/github.com/sylabs && \ + wget https://github.com/sylabs/singularity/releases/download/v${singularity_version}/singularity-${singularity_version}.tar.gz && \ + tar -xzvf singularity-${singularity_version}.tar.gz && \ + cd singularity && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + sudo make -C builddir install + fi + + +install_python_3: &install_python_3 + name: install Python 3.5 dependencies + command: | + ls $HOME + if [ ! -d "/home/circleci/conda" ]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda + export PATH=$HOME/conda/bin:$PATH + $HOME/conda/bin/python setup.py install + else + echo "Miniconda 3 is already installed, continuing to build." + fi + +install_python_2: &install_python_2 + name: install Python 3.5 dependencies + command: | + ls $HOME + if [ ! -d "/home/circleci/conda" ]; then + wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh + /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda + export PATH=$HOME/conda/bin:$PATH + $HOME/conda/bin/python setup.py install + else + echo "Miniconda 2 is already installed, continuing to build." + fi + +test_spython: &test_spython + name: Test Singularity Python + command: | + cd ~/repo + export PATH=$PATH:/opt/circleci/.pyenv/shims + $HOME/conda/bin/python -m unittest tests.test_client + $HOME/conda/bin/python -m unittest tests.test_utils + + +jobs: + test-singularity-3.1.0-python-3: + machine: true + working_directory: ~/repo + steps: + - checkout + - run: *setup + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_3 + - run: *install_singularity_3 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython + + test-singularity-3.1.0-python-2: + machine: true + working_directory: ~/repo + steps: + - checkout + - run: *setup + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_2 + - run: *install_singularity_3 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython + + test-singularity-2.6.1-python-3: + machine: true + working_directory: ~/repo + steps: + - checkout + - run: *setup + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_3 + - run: *install_singularity_2 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython From 4e32c7cee0a22781477e1614153bacf51a7ef0e3 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:00:47 -0500 Subject: [PATCH 02/41] adding circleci to test --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 94e81806..41973f88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,6 @@ workflows: branches: only: master - setup: &setup name: install setup command: | From 0cbbf3e817fb4652d763ff14ce1c06d60e51e48f Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:14:22 -0500 Subject: [PATCH 03/41] adding circleci to test --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41973f88..a81d008b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,6 @@ setup: &setup sudo apt-get install -y wget git jq sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers - install_spython: &install_spython name: install spython command: | From 90452943b2512fd65ba7e284042ee373cc39e82b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:29:00 -0500 Subject: [PATCH 04/41] adding circleci to test --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a81d008b..a008e2ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,4 @@ # CircleCI build config to test different versions of Singuarity - version: 2.1 workflows: From 5438e88baa7709e3b66ef27637573becb8ee1f0a Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:41:40 -0500 Subject: [PATCH 05/41] adding circleci to test --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a008e2ae..4d1559f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,19 +8,19 @@ workflows: - test-singularity-3.1.0-python-3: filters: branches: - only: master + ignore: master - test-singularity-3.1.0-python-2: filters: branches: - only: master + ignore: master - test-singularity-2.6.1-python-2: filters: branches: - only: master + ignore: master - test-singularity-2.6.1-python-3: filters: branches: - only: master + ignore: master setup: &setup name: install setup @@ -59,7 +59,7 @@ install_singularity_2: &install_singularity_2 wget https://github.com/sylabs/singularity/releases/download/2.6.1/singularity-2.6.1.tar.gz tar -xzvf singularity-2.6.1.tar.gz cd singularity-2.6.1 && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install - + fi install_singularity_3: &install_singularity_3 name: install Singularity 3.1 From 17029a223635b034f7637ec5bb336e9896018072 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:43:29 -0500 Subject: [PATCH 06/41] adding circleci to test --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d1559f7..6ded0f8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,6 @@ version: 2.1 workflows: - version: 2 test: jobs: - test-singularity-3.1.0-python-3: From a0fcca868b46b7446589ee7ebcd905c99d561b4c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:47:04 -0500 Subject: [PATCH 07/41] test --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ded0f8e..4d1559f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 workflows: + version: 2 test: jobs: - test-singularity-3.1.0-python-3: From 9fa5aa801c5f47cdd651de3b74ed741542551808 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:49:17 -0500 Subject: [PATCH 08/41] test --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d1559f7..6879cc8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,19 +5,19 @@ workflows: version: 2 test: jobs: - - test-singularity-3.1.0-python-3: + - test-singularity-3-python-3: filters: branches: ignore: master - - test-singularity-3.1.0-python-2: + - test-singularity-3-python-2: filters: branches: ignore: master - - test-singularity-2.6.1-python-2: + - test-singularity-2-python-2: filters: branches: ignore: master - - test-singularity-2.6.1-python-3: + - test-singularity-2-python-3: filters: branches: ignore: master @@ -137,7 +137,7 @@ test_spython: &test_spython jobs: - test-singularity-3.1.0-python-3: + test-singularity-3-python-3: machine: true working_directory: ~/repo steps: @@ -155,7 +155,7 @@ jobs: key: v1-dependencies - run: *test_spython - test-singularity-3.1.0-python-2: + test-singularity-3-python-2: machine: true working_directory: ~/repo steps: @@ -173,7 +173,7 @@ jobs: key: v1-dependencies - run: *test_spython - test-singularity-2.6.1-python-3: + test-singularity-2-python-3: machine: true working_directory: ~/repo steps: From 9a7c665bd79410b75642222c88a5b124b278a16b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:50:15 -0500 Subject: [PATCH 09/41] test --- .circleci/config.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6879cc8d..f7e567da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -190,3 +190,21 @@ jobs: - /home/circleci/conda key: v1-dependencies - run: *test_spython + + test-singularity-2-python-2: + machine: true + working_directory: ~/repo + steps: + - checkout + - run: *setup + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_2 + - run: *install_singularity_2 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython From 70c2bd90f7af281c0b6c87808dfc77f9bbe17019 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 12:56:30 -0500 Subject: [PATCH 10/41] test --- .circleci/config.yml | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7e567da..f3b2f994 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,28 +38,22 @@ install_spython: &install_spython install_singularity_2: &install_singularity_2 name: install Singularity 2.6.1 command: | - which singularity &> /dev/null - - if [ $? -eq 0 ]; then - echo "Singularity is installed." - else - sudo apt-get update && sudo apt-get -y install git \ - build-essential \ - libtool \ - squashfs-tools \ - autotools-dev \ - libarchive-dev \ - automake \ - autoconf \ - debootstrap \ - yum \ - uuid-dev \ - libssl-dev \ - python3-dev - wget https://github.com/sylabs/singularity/releases/download/2.6.1/singularity-2.6.1.tar.gz - tar -xzvf singularity-2.6.1.tar.gz - cd singularity-2.6.1 && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install - fi + sudo apt-get update && sudo apt-get -y install git \ + build-essential \ + libtool \ + squashfs-tools \ + autotools-dev \ + libarchive-dev \ + automake \ + autoconf \ + debootstrap \ + yum \ + uuid-dev \ + libssl-dev \ + python3-dev + wget https://github.com/sylabs/singularity/releases/download/2.6.1/singularity-2.6.1.tar.gz + tar -xzvf singularity-2.6.1.tar.gz + cd singularity-2.6.1 && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install install_singularity_3: &install_singularity_3 name: install Singularity 3.1 From 93bfd4564c2a253d4629687893d689d44ef4682b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 13:01:11 -0500 Subject: [PATCH 11/41] testing --- .circleci/config.yml | 61 +++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3b2f994..402ec4a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,41 +58,32 @@ install_singularity_2: &install_singularity_2 install_singularity_3: &install_singularity_3 name: install Singularity 3.1 command: | - which singularity &> /dev/null - - if [ $? -eq 0 ]; then - echo "Singularity is installed." - export GOPATH=/go - export PATH=$PATH:/usr/local/go/bin - else - - sudo apt-get install -y build-essential \ - squashfs-tools \ - libtool \ - uuid-dev \ - libssl-dev \ - libgpgme11-dev \ - libseccomp-dev \ - pkg-config - - # Install GoLang - wget https://dl.google.com/go/go${GO_VERSION}.src.tar.gz && \ - tar -C /usr/local -xzf go${GO_VERSION}.src.tar.gz - - export PATH=$PATH:/usr/local/go/bin && \ - sudo mkdir -p /go && \ - sudo chmod -R 7777 /go - export GOPATH=/go && \ - go get -u github.com/golang/dep/cmd/dep && \ - mkdir -p ${GOPATH}/src/github.com/sylabs && \ - cd ${GOPATH}/src/github.com/sylabs && \ - wget https://github.com/sylabs/singularity/releases/download/v${singularity_version}/singularity-${singularity_version}.tar.gz && \ - tar -xzvf singularity-${singularity_version}.tar.gz && \ - cd singularity && \ - ./mconfig -p /usr/local && \ - make -C builddir && \ - sudo make -C builddir install - fi + sudo apt-get install -y build-essential \ + squashfs-tools \ + libtool \ + uuid-dev \ + libssl-dev \ + libgpgme11-dev \ + libseccomp-dev \ + pkg-config + + # Install GoLang + wget https://dl.google.com/go/go${GO_VERSION}.src.tar.gz && \ + tar -C /usr/local -xzf go${GO_VERSION}.src.tar.gz + + export PATH=$PATH:/usr/local/go/bin && \ + sudo mkdir -p /go && \ + sudo chmod -R 7777 /go + export GOPATH=/go && \ + go get -u github.com/golang/dep/cmd/dep && \ + mkdir -p ${GOPATH}/src/github.com/sylabs && \ + cd ${GOPATH}/src/github.com/sylabs && \ + wget https://github.com/sylabs/singularity/releases/download/v${singularity_version}/singularity-${singularity_version}.tar.gz && \ + tar -xzvf singularity-${singularity_version}.tar.gz && \ + cd singularity && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + sudo make -C builddir install install_python_3: &install_python_3 From f117494e1d8ad6931f6aa96149134fe8bbdf3a54 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 13:04:20 -0500 Subject: [PATCH 12/41] testing --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 402ec4a8..01ce8064 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,8 +78,8 @@ install_singularity_3: &install_singularity_3 go get -u github.com/golang/dep/cmd/dep && \ mkdir -p ${GOPATH}/src/github.com/sylabs && \ cd ${GOPATH}/src/github.com/sylabs && \ - wget https://github.com/sylabs/singularity/releases/download/v${singularity_version}/singularity-${singularity_version}.tar.gz && \ - tar -xzvf singularity-${singularity_version}.tar.gz && \ + wget https://github.com/sylabs/singularity/releases/download/v3.1.0/singularity-3.1.0.tar.gz && \ + tar -xzvf singularity-3.1.0.tar.gz && \ cd singularity && \ ./mconfig -p /usr/local && \ make -C builddir && \ From 31f087ad6c038c90fa4278f362f243d833e2a47d Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 13:06:00 -0500 Subject: [PATCH 13/41] testing --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01ce8064..7c107525 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,6 +116,7 @@ test_spython: &test_spython name: Test Singularity Python command: | cd ~/repo + ls export PATH=$PATH:/opt/circleci/.pyenv/shims $HOME/conda/bin/python -m unittest tests.test_client $HOME/conda/bin/python -m unittest tests.test_utils From b080dc9ee627e21e61ee00c8fb508b9be8c4a439 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 13:19:50 -0500 Subject: [PATCH 14/41] wrong pwd --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c107525..e6b39439 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,7 +115,7 @@ install_python_2: &install_python_2 test_spython: &test_spython name: Test Singularity Python command: | - cd ~/repo + cd ~/repo/spython ls export PATH=$PATH:/opt/circleci/.pyenv/shims $HOME/conda/bin/python -m unittest tests.test_client From d93f480ce6fab6919ec7bf8cf3b5c278b79408a3 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 13:29:34 -0500 Subject: [PATCH 15/41] go version missing --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6b39439..7368e4f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,8 +68,8 @@ install_singularity_3: &install_singularity_3 pkg-config # Install GoLang - wget https://dl.google.com/go/go${GO_VERSION}.src.tar.gz && \ - tar -C /usr/local -xzf go${GO_VERSION}.src.tar.gz + wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ + tar -C /usr/local -xzf go1.11.5.src.tar.gz export PATH=$PATH:/usr/local/go/bin && \ sudo mkdir -p /go && \ From 23f0c927d2e04d0ad632f72f0ebde94239ae5e72 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 13:36:56 -0500 Subject: [PATCH 16/41] untar with sudo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7368e4f8..93effaab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ install_singularity_3: &install_singularity_3 # Install GoLang wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ - tar -C /usr/local -xzf go1.11.5.src.tar.gz + sudo tar -C /usr/local -xzf go1.11.5.src.tar.gz export PATH=$PATH:/usr/local/go/bin && \ sudo mkdir -p /go && \ From 7100130b4d9d1d1efbeaa46eb9411d5be3d9f450 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:11:18 -0500 Subject: [PATCH 17/41] untar with sudo --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 93effaab..05e66550 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,9 @@ install_singularity_3: &install_singularity_3 # Install GoLang wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ - sudo tar -C /usr/local -xzf go1.11.5.src.tar.gz + sudo mkdir -p /usr/local/go + cmhod -R 7777 /usr/local/go + tar -C /usr/local -xzf go1.11.5.src.tar.gz export PATH=$PATH:/usr/local/go/bin && \ sudo mkdir -p /go && \ From 4ccd0f04ddedc238b3ae6da794bfdbe33c30c313 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:12:42 -0500 Subject: [PATCH 18/41] untar with sudo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05e66550..00fcafd4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ install_singularity_3: &install_singularity_3 # Install GoLang wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ sudo mkdir -p /usr/local/go - cmhod -R 7777 /usr/local/go + chmod -R 7777 /usr/local/go tar -C /usr/local -xzf go1.11.5.src.tar.gz export PATH=$PATH:/usr/local/go/bin && \ From 0ce4bad74515535cad1586c67c890c5c2f5d55e9 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:14:52 -0500 Subject: [PATCH 19/41] go already instealled? --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 00fcafd4..ec340a37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,8 @@ install_singularity_3: &install_singularity_3 # Install GoLang wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ sudo mkdir -p /usr/local/go - chmod -R 7777 /usr/local/go + sudo rm -rf /usr/local/go + sudo chmod -R 7777 /usr/local/go tar -C /usr/local -xzf go1.11.5.src.tar.gz export PATH=$PATH:/usr/local/go/bin && \ From 22d34f4f7f1a55d5c371c9a4be7d9610c32aa067 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:17:37 -0500 Subject: [PATCH 20/41] go already instealled? --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec340a37..81f28d5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,8 +70,8 @@ install_singularity_3: &install_singularity_3 # Install GoLang wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ sudo mkdir -p /usr/local/go - sudo rm -rf /usr/local/go sudo chmod -R 7777 /usr/local/go + ls /usr/local/go tar -C /usr/local -xzf go1.11.5.src.tar.gz export PATH=$PATH:/usr/local/go/bin && \ From a35babc9e59fd32f1246278b0c6786e01c18c1d8 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:20:17 -0500 Subject: [PATCH 21/41] go already instealled? --- .circleci/config.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81f28d5f..b3003bad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,13 +68,10 @@ install_singularity_3: &install_singularity_3 pkg-config # Install GoLang + cd /tmp wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ - sudo mkdir -p /usr/local/go - sudo chmod -R 7777 /usr/local/go - ls /usr/local/go - tar -C /usr/local -xzf go1.11.5.src.tar.gz - - export PATH=$PATH:/usr/local/go/bin && \ + tar -xzf go1.11.5.src.tar.gz + export PATH=$PATH:/tmp/go/bin && \ sudo mkdir -p /go && \ sudo chmod -R 7777 /go export GOPATH=/go && \ From efc74f558deacbe42bb0b6207c4f735db3a7f657 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:24:35 -0500 Subject: [PATCH 22/41] using an older golang? --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3003bad..47f9f98e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,9 +71,10 @@ install_singularity_3: &install_singularity_3 cd /tmp wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ tar -xzf go1.11.5.src.tar.gz - export PATH=$PATH:/tmp/go/bin && \ + export PATH=/tmp/go/bin:$PATH && \ sudo mkdir -p /go && \ sudo chmod -R 7777 /go + which go export GOPATH=/go && \ go get -u github.com/golang/dep/cmd/dep && \ mkdir -p ${GOPATH}/src/github.com/sylabs && \ From 31a70baef72b3298e00e72e2e1d55925b5bfa0dd Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:28:37 -0500 Subject: [PATCH 23/41] using an older golang? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47f9f98e..e35a1922 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,6 +74,7 @@ install_singularity_3: &install_singularity_3 export PATH=/tmp/go/bin:$PATH && \ sudo mkdir -p /go && \ sudo chmod -R 7777 /go + sudo rm -rf /usr/local/go which go export GOPATH=/go && \ go get -u github.com/golang/dep/cmd/dep && \ From a5f92d9053a9c080f6099b1061b6a7fa7e396034 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:31:42 -0500 Subject: [PATCH 24/41] using an older golang? --- .circleci/config.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e35a1922..e5b2a006 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,11 +69,12 @@ install_singularity_3: &install_singularity_3 # Install GoLang cd /tmp - wget https://dl.google.com/go/go1.11.5.src.tar.gz && \ - tar -xzf go1.11.5.src.tar.gz - export PATH=/tmp/go/bin:$PATH && \ - sudo mkdir -p /go && \ + wget https://dl.google.com/go/go1.11.5.src.tar.gz + tar -xzf go1.11.5.src.tar.gz + export PATH=/tmp/go/bin:$PATH + sudo mkdir -p /go sudo chmod -R 7777 /go + which go sudo rm -rf /usr/local/go which go export GOPATH=/go && \ From ea3832300f1dfb67428e46d204881058a842ee8a Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:34:59 -0500 Subject: [PATCH 25/41] using an older golang? --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e5b2a006..d125a153 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,11 +74,10 @@ install_singularity_3: &install_singularity_3 export PATH=/tmp/go/bin:$PATH sudo mkdir -p /go sudo chmod -R 7777 /go - which go - sudo rm -rf /usr/local/go + echo $PATH which go export GOPATH=/go && \ - go get -u github.com/golang/dep/cmd/dep && \ + /tmp/go/bin/go get -u github.com/golang/dep/cmd/dep && \ mkdir -p ${GOPATH}/src/github.com/sylabs && \ cd ${GOPATH}/src/github.com/sylabs && \ wget https://github.com/sylabs/singularity/releases/download/v3.1.0/singularity-3.1.0.tar.gz && \ From 715abfada965af6c1d1eeb496938e3b9a1b5028b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:38:29 -0500 Subject: [PATCH 26/41] using an older golang? --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d125a153..62911aa6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,8 +69,11 @@ install_singularity_3: &install_singularity_3 # Install GoLang cd /tmp + echo ${PWD} wget https://dl.google.com/go/go1.11.5.src.tar.gz tar -xzf go1.11.5.src.tar.gz + echo $PWD + ls export PATH=/tmp/go/bin:$PATH sudo mkdir -p /go sudo chmod -R 7777 /go From ed9a532e5d57d1764cfddfe69e9eca0e1119937f Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 1 Mar 2019 14:42:23 -0500 Subject: [PATCH 27/41] OMG DOWNLOADING WRONG ONE --- .circleci/config.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62911aa6..9a4a459e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,11 +69,8 @@ install_singularity_3: &install_singularity_3 # Install GoLang cd /tmp - echo ${PWD} - wget https://dl.google.com/go/go1.11.5.src.tar.gz - tar -xzf go1.11.5.src.tar.gz - echo $PWD - ls + wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz + tar -xzf go1.11.5.linux-amd64.tar.gz export PATH=/tmp/go/bin:$PATH sudo mkdir -p /go sudo chmod -R 7777 /go From 9d0c94a9ff6a1c1298a1774c06a0fde93955f49b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 10:16:17 -0500 Subject: [PATCH 28/41] testing orb in workflow --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a4a459e..e6e62eab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,9 @@ # CircleCI build config to test different versions of Singuarity version: 2.1 +orbs: + singularity: singularity/singularity@1.0.2 + workflows: version: 2 test: @@ -135,7 +138,9 @@ jobs: keys: - v1-dependencies - run: *install_python_3 - - run: *install_singularity_3 + - singularity/install_debian_3 + go-version: 11.1.5 + singularity-version: 3.1.0 - run: *install_spython - save_cache: paths: From b9f015c8fde38e4e7af368dfef612e03d9e0a48d Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 10:17:11 -0500 Subject: [PATCH 29/41] testing orb in workflow --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6e62eab..36dc80fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,7 +138,7 @@ jobs: keys: - v1-dependencies - run: *install_python_3 - - singularity/install_debian_3 + - singularity/install_debian_3: go-version: 11.1.5 singularity-version: 3.1.0 - run: *install_spython From b089bc9ab5d33fb123c95d344c08482c5cc80d05 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 10:19:06 -0500 Subject: [PATCH 30/41] testing orb in workflow --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 36dc80fb..554d890a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,8 +138,9 @@ jobs: keys: - v1-dependencies - run: *install_python_3 - - singularity/install_debian_3: + - singularity/install-go: go-version: 11.1.5 + - singularity/debian-install-3: singularity-version: 3.1.0 - run: *install_spython - save_cache: From 9c4aebc1aec4c6b58aa11e7ebb30437ee995776c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 10:21:54 -0500 Subject: [PATCH 31/41] testing orb in workflow --- .circleci/config.yml | 68 ++++++-------------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 554d890a..2778ce73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,60 +37,6 @@ install_spython: &install_spython command: | $HOME/conda/bin/python setup.py install - -install_singularity_2: &install_singularity_2 - name: install Singularity 2.6.1 - command: | - sudo apt-get update && sudo apt-get -y install git \ - build-essential \ - libtool \ - squashfs-tools \ - autotools-dev \ - libarchive-dev \ - automake \ - autoconf \ - debootstrap \ - yum \ - uuid-dev \ - libssl-dev \ - python3-dev - wget https://github.com/sylabs/singularity/releases/download/2.6.1/singularity-2.6.1.tar.gz - tar -xzvf singularity-2.6.1.tar.gz - cd singularity-2.6.1 && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install - -install_singularity_3: &install_singularity_3 - name: install Singularity 3.1 - command: | - sudo apt-get install -y build-essential \ - squashfs-tools \ - libtool \ - uuid-dev \ - libssl-dev \ - libgpgme11-dev \ - libseccomp-dev \ - pkg-config - - # Install GoLang - cd /tmp - wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz - tar -xzf go1.11.5.linux-amd64.tar.gz - export PATH=/tmp/go/bin:$PATH - sudo mkdir -p /go - sudo chmod -R 7777 /go - echo $PATH - which go - export GOPATH=/go && \ - /tmp/go/bin/go get -u github.com/golang/dep/cmd/dep && \ - mkdir -p ${GOPATH}/src/github.com/sylabs && \ - cd ${GOPATH}/src/github.com/sylabs && \ - wget https://github.com/sylabs/singularity/releases/download/v3.1.0/singularity-3.1.0.tar.gz && \ - tar -xzvf singularity-3.1.0.tar.gz && \ - cd singularity && \ - ./mconfig -p /usr/local && \ - make -C builddir && \ - sudo make -C builddir install - - install_python_3: &install_python_3 name: install Python 3.5 dependencies command: | @@ -133,13 +79,12 @@ jobs: working_directory: ~/repo steps: - checkout - - run: *setup - restore_cache: keys: - v1-dependencies - run: *install_python_3 - singularity/install-go: - go-version: 11.1.5 + go-version: 1.11.5 - singularity/debian-install-3: singularity-version: 3.1.0 - run: *install_spython @@ -159,7 +104,10 @@ jobs: keys: - v1-dependencies - run: *install_python_2 - - run: *install_singularity_3 + - singularity/install-go: + go-version: 1.11.5 + - singularity/debian-install-3: + singularity-version: 3.1.0 - run: *install_spython - save_cache: paths: @@ -177,7 +125,8 @@ jobs: keys: - v1-dependencies - run: *install_python_3 - - run: *install_singularity_2 + - singularity/debian-install-2: + singularity-version: 2.6.1 - run: *install_spython - save_cache: paths: @@ -195,7 +144,8 @@ jobs: keys: - v1-dependencies - run: *install_python_2 - - run: *install_singularity_2 + - singularity/debian-install-2: + singularity-version: 2.6.1 - run: *install_spython - save_cache: paths: From 7fe9640e3a188bd40f2664d9ea8ed233bde9b81c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 10:40:00 -0500 Subject: [PATCH 32/41] try updated orb --- .circleci/config.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2778ce73..583d6b5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 orbs: - singularity: singularity/singularity@1.0.2 + singularity: singularity/singularity@dev:alpha workflows: version: 2 @@ -25,13 +25,6 @@ workflows: branches: ignore: master -setup: &setup - name: install setup - command: | - sudo apt-get update && \ - sudo apt-get install -y wget git jq - sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers - install_spython: &install_spython name: install spython command: | @@ -99,7 +92,6 @@ jobs: working_directory: ~/repo steps: - checkout - - run: *setup - restore_cache: keys: - v1-dependencies @@ -120,7 +112,6 @@ jobs: working_directory: ~/repo steps: - checkout - - run: *setup - restore_cache: keys: - v1-dependencies @@ -139,7 +130,6 @@ jobs: working_directory: ~/repo steps: - checkout - - run: *setup - restore_cache: keys: - v1-dependencies From 13a80611ed2f55f398b1c4edc214ca1af325ff7c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 11:13:08 -0500 Subject: [PATCH 33/41] updating version, removing travis dependencies --- .circleci/config.yml | 2 +- .travis.yml | 24 ------------------------ .travis/before_install | 21 --------------------- .travis/script | 4 ---- CHANGELOG.md | 1 + spython/tests/test_client.py | 4 ++-- spython/version.py | 2 +- 7 files changed, 5 insertions(+), 53 deletions(-) delete mode 100644 .travis.yml delete mode 100644 .travis/before_install delete mode 100644 .travis/script diff --git a/.circleci/config.yml b/.circleci/config.yml index 583d6b5e..0d1cdab3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 orbs: - singularity: singularity/singularity@dev:alpha + singularity: singularity/singularity@1.0.3 workflows: version: 2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bfad6304..00000000 --- a/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: python - -sudo: true - -os: linux - -python: - - "2.7" - - "3.5" - -services: - - docker - -matrix: - allow_failures: - - python: "2.6" - - python: "3.5" - -before_install: - - sudo chmod u+x .travis/* - - /bin/bash .travis/before_install - -script: - - /bin/bash .travis/script diff --git a/.travis/before_install b/.travis/before_install deleted file mode 100644 index 9e81eb2d..00000000 --- a/.travis/before_install +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -sudo apt-get update && sudo apt-get install -y wget git build-essential squashfs-tools \ - libtool \ - autotools-dev \ - libarchive-dev \ - automake \ - autoconf \ - uuid-dev \ - libssl-dev - - -sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers - -echo "PYTHON VERSION" -which python - -python setup.py sdist && python setup.py install - -# Install Singularity (development) -cd /tmp && git clone -b vault/release-2.6 https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install diff --git a/.travis/script b/.travis/script deleted file mode 100644 index b16b7ee5..00000000 --- a/.travis/script +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd $TRAVIS_BUILD_DIR/spython -python -m unittest tests.test_client -python -m unittest tests.test_utils diff --git a/CHANGELOG.md b/CHANGELOG.md index b68da4b7..dbdbc1f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The client here will eventually be released as "spython" (and eventually to singularity on pypi), and the versions here will coincide with these releases. ## [master](https://github.com/singularityhub/singularity-cli/tree/master) + - adding testing for 3.1.0 with Singularity Orbs (0.0.53) - instance stop all missing check for Singularity V3.+ (0.0.52) - fixing bug with instances list, name not taken into account (0.0.51) - additional of args to instance start commands (0.0.50) diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py index a1ee5278..f8133c6a 100644 --- a/spython/tests/test_client.py +++ b/spython/tests/test_client.py @@ -31,10 +31,10 @@ def tearDown(self): def test_commands(self): print('Testing client.build command') - container = "%s/container.img" %(self.tmpdir) + container = "%s/container.sif" %(self.tmpdir) print("...Case 1: Build from docker uri") - created_container = self.cli.build('docker://ubuntu', + created_container = self.cli.build('docker://busybox:1.30.1', image=container, sudo=False) self.assertEqual(created_container, container) diff --git a/spython/version.py b/spython/version.py index 45621e95..280ce035 100644 --- a/spython/version.py +++ b/spython/version.py @@ -6,7 +6,7 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -__version__ = "0.0.52" +__version__ = "0.0.53" AUTHOR = 'Vanessa Sochat' AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'spython' From 118f009c2ec00eb860a7da9a14b6382220170ba5 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 15:45:27 -0500 Subject: [PATCH 34/41] testing fixes --- CHANGELOG.md | 1 + spython/main/inspect.py | 37 +++++++++++++++++++++++++++++------- spython/main/pull.py | 15 +++++++++++++-- spython/tests/test_client.py | 10 ++++------ 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbdbc1f0..151ae178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ singularity on pypi), and the versions here will coincide with these releases. ## [master](https://github.com/singularityhub/singularity-cli/tree/master) - adding testing for 3.1.0 with Singularity Orbs (0.0.53) + - inspect returns parsed json on success, or full message / return code otherwise - instance stop all missing check for Singularity V3.+ (0.0.52) - fixing bug with instances list, name not taken into account (0.0.51) - additional of args to instance start commands (0.0.50) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index 48935c50..bc2b02fd 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -5,21 +5,26 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json from spython.logger import bot +from spython.utils import ( + check_install, + get_singularity_version, + run_command +) -def inspect(self,image=None, json=True, app=None): +def inspect(self, image=None, json=True, app=None, quiet=True): '''inspect will show labels, defile, runscript, and tests for an image - Parameters ========== - image_path: path of image to inspect + image: path of image to inspect json: print json instead of raw text (default True) + quiet: Don't print result to the screen (default True) app: if defined, return help in context of an app ''' - from spython.utils import check_install check_install() # No image provided, default to use the client's loaded image @@ -31,12 +36,30 @@ def inspect(self,image=None, json=True, app=None): cmd = cmd + ['--app', app] options = ['e','d','l','r','hf','t'] + + # After Singularity 3.0, helpfile was changed to H from + + if "version 3" in get_singularity_version(): + options = ['e','d','l','r','H','t'] + [cmd.append('-%s' % x) for x in options] if json is True: cmd.append('--json') cmd.append(image) - output = self._run_command(cmd) - #self.println(output,quiet=self.quiet) - return output + result = run_command(cmd, quiet=True) + + if result['return_code'] == 0: + result = json.loads(result['message'][0]) + + # If labels included, try parsing to json + + if 'labels' in result['attributes']: + labels = json.loads(result['attributes']['labels']) + result['attributes']['labels'] = labels + + if not quiet: + print(json.dumps(result, indent=4)) + + return result diff --git a/spython/main/pull.py b/spython/main/pull.py index 12171818..af16ec9c 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -7,7 +7,7 @@ from spython.logger import bot -from spython.utils import stream_command +from spython.utils import ( stream_command, get_singularity_version ) import os import re import shutil @@ -44,6 +44,10 @@ def pull(self, cmd = self._init_command('pull') + # If Singularity version > 3.0, we have sif format + if 'version 3' in get_singularity_version(): + ext = 'sif' + # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() @@ -74,7 +78,14 @@ def pull(self, # Only add name if we aren't naming by hash or commit if not name_by_commit and not name_by_hash: - cmd = cmd + ["--name", name] + + # Regression Singularity 3.* onward, PULLFOLDER not honored + # https://github.com/sylabs/singularity/issues/2788 + if pull_folder and 'version 3' in get_singularity_version(): + pull_folder_name = os.path.join(pull_folder, os.path.basename(name)) + cmd = cmd + ["--name", pull_folder_name] + else: + cmd = cmd + ["--name", name] if force is True: cmd = cmd + ["--force"] diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py index f8133c6a..a2652962 100644 --- a/spython/tests/test_client.py +++ b/spython/tests/test_client.py @@ -56,9 +56,9 @@ def test_commands(self): os.remove(image) print("...Case 2: Testing docker pull") - container = self.cli.pull("docker://ubuntu:14.04", + container = self.cli.pull("docker://busybox:1.30.1", pull_folder=self.tmpdir) - self.assertTrue("ubuntu:14.04" in container) + self.assertTrue("busybox:1.30.1" in container) print(container) self.assertTrue(os.path.exists(container)) @@ -69,11 +69,9 @@ def test_commands(self): self.assertTrue('bin\nboot\ndev' in result) print("Testing client.inspect command") - result = self.cli.inspect(container) - labels = json.loads(result) - self.assertTrue('data' in labels) - os.remove(container) + labels = self.cli.inspect(container) + os.remove(container) if __name__ == '__main__': From 22140b004ee75fdf43093bd9d7ea414746484408 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 15:48:50 -0500 Subject: [PATCH 35/41] container is changed --- spython/tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py index a2652962..4f7bd9e2 100644 --- a/spython/tests/test_client.py +++ b/spython/tests/test_client.py @@ -66,7 +66,7 @@ def test_commands(self): print('Testing client.execute command') result = self.cli.execute(container,'ls /') print(result) - self.assertTrue('bin\nboot\ndev' in result) + self.assertTrue('tmp\nusr\nvar' in result) print("Testing client.inspect command") labels = self.cli.inspect(container) From ae358568c98588525eef31fb048068ffb30d88e8 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 15:54:25 -0500 Subject: [PATCH 36/41] need to import json with different name --- spython/main/inspect.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index bc2b02fd..3231bee0 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -5,7 +5,7 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -import json +import json as jsonp from spython.logger import bot from spython.utils import ( @@ -51,15 +51,15 @@ def inspect(self, image=None, json=True, app=None, quiet=True): result = run_command(cmd, quiet=True) if result['return_code'] == 0: - result = json.loads(result['message'][0]) + result = jsonp.loads(result['message'][0]) # If labels included, try parsing to json if 'labels' in result['attributes']: - labels = json.loads(result['attributes']['labels']) + labels = jsonp.loads(result['attributes']['labels']) result['attributes']['labels'] = labels if not quiet: - print(json.dumps(result, indent=4)) + print(jsonp.dumps(result, indent=4)) return result From fc320d4e6e4705cd33d0a2101591b2f893e0de99 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 16:03:04 -0500 Subject: [PATCH 37/41] fixing labels --- spython/main/inspect.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index 3231bee0..e5e77cc5 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -54,10 +54,13 @@ def inspect(self, image=None, json=True, app=None, quiet=True): result = jsonp.loads(result['message'][0]) # If labels included, try parsing to json + if "data" in result: + labels = result['data']['attributes'].get('labels', {}) - if 'labels' in result['attributes']: - labels = jsonp.loads(result['attributes']['labels']) - result['attributes']['labels'] = labels + elif 'attributes' in result: + labels = result['attributes'].get('labels', {}) + + result['attributes']['labels'] = jsonp.loads(labels) if not quiet: print(jsonp.dumps(result, indent=4)) From ac0c33b784650f4e7d061099eb1dd1043c69d89c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 16:10:04 -0500 Subject: [PATCH 38/41] fixing labels --- spython/main/inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index e5e77cc5..c31cc0ab 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -48,7 +48,7 @@ def inspect(self, image=None, json=True, app=None, quiet=True): cmd.append('--json') cmd.append(image) - result = run_command(cmd, quiet=True) + result = run_command(cmd, quiet=False) if result['return_code'] == 0: result = jsonp.loads(result['message'][0]) From e0590221cc151172e14c1ad9d7026e5cb30a839a Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 16:15:18 -0500 Subject: [PATCH 39/41] fixing labels --- spython/main/inspect.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index c31cc0ab..6c430205 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -50,17 +50,27 @@ def inspect(self, image=None, json=True, app=None, quiet=True): cmd.append(image) result = run_command(cmd, quiet=False) + print('RESULT') + print(result['message']) + print(len(result['message'])) + if result['return_code'] == 0: result = jsonp.loads(result['message'][0]) - # If labels included, try parsing to json if "data" in result: labels = result['data']['attributes'].get('labels', {}) elif 'attributes' in result: labels = result['attributes'].get('labels', {}) - result['attributes']['labels'] = jsonp.loads(labels) + # If labels included, try parsing to json + + try: + labels = jsonp.loads(labels) + except: + pass + + result['attributes']['labels'] = labels if not quiet: print(jsonp.dumps(result, indent=4)) From ceac6ffd2814ea17b1fd27b01f785428326c477c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 16:27:15 -0500 Subject: [PATCH 40/41] None should not return, do {} instead --- spython/main/inspect.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index 6c430205..3838ddab 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -50,18 +50,14 @@ def inspect(self, image=None, json=True, app=None, quiet=True): cmd.append(image) result = run_command(cmd, quiet=False) - print('RESULT') - print(result['message']) - print(len(result['message'])) - if result['return_code'] == 0: result = jsonp.loads(result['message'][0]) if "data" in result: - labels = result['data']['attributes'].get('labels', {}) + labels = result['data']['attributes'].get('labels') or {} elif 'attributes' in result: - labels = result['attributes'].get('labels', {}) + labels = result['attributes'].get('labels') or {} # If labels included, try parsing to json From 5c4ac1de8b9f66b3302184819840d0a26e4024ab Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sat, 2 Mar 2019 16:34:17 -0500 Subject: [PATCH 41/41] put into separate function --- spython/main/inspect.py | 42 +++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index 3838ddab..2a7c499b 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -53,22 +53,40 @@ def inspect(self, image=None, json=True, app=None, quiet=True): if result['return_code'] == 0: result = jsonp.loads(result['message'][0]) - if "data" in result: - labels = result['data']['attributes'].get('labels') or {} + # Fix up labels + labels = parse_labels(result) - elif 'attributes' in result: - labels = result['attributes'].get('labels') or {} + if not quiet: + print(jsonp.dumps(result, indent=4)) - # If labels included, try parsing to json + return result - try: - labels = jsonp.loads(labels) - except: - pass - result['attributes']['labels'] = labels +def parse_labels(result): + '''fix up the labels, meaning parse to json if needed, and return + original updated object - if not quiet: - print(jsonp.dumps(result, indent=4)) + Parameters + ========== + result: the json object to parse from inspect + ''' + + if "data" in result: + labels = result['data']['attributes'].get('labels') or {} + + elif 'attributes' in result: + labels = result['attributes'].get('labels') or {} + + # If labels included, try parsing to json + + try: + labels = jsonp.loads(labels) + except: + pass + + if "data" in result: + result['data']['attributes']['labels'] = labels + else: + result['attributes']['labels'] = labels return result