From 02c5e57e075233b97f6888f57abe1f6eee415045 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 29 Mar 2022 03:40:14 -0600 Subject: [PATCH 1/2] redo to close #187 and to close #188 Signed-off-by: vsoch --- CHANGELOG.md | 1 + setup.py | 1 - spython/main/parse/parsers/base.py | 4 +- spython/main/parse/writers/docker.py | 16 +- .../testdata/docker2singularity/argsub.def | 1575 +++++++++++++++++ .../testdata/docker2singularity/argsub.docker | 1522 ++++++++++++++++ spython/version.py | 2 +- 7 files changed, 3109 insertions(+), 12 deletions(-) create mode 100644 spython/tests/testdata/docker2singularity/argsub.def create mode 100644 spython/tests/testdata/docker2singularity/argsub.docker diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c5a3b9..45794062 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The client here will eventually be released as "spython" (and eventually to singularity on pypi), and the versions here will coincide with these releases. ## [master](https://github.com/singularityhub/singularity-cli/tree/master) + - parser bugfixes, arg from Docker not properly parsed (0.2.1) - version checks removed to support Singularity 3.x and above (0.2.0) - adding support for SIF (oras pull) (0.1.18) - updating CI/tests and fixing deprecations (warnings) (0.1.17) diff --git a/setup.py b/setup.py index c2cdacac..ac66966b 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,6 @@ def get_requirements(lookup=None): "Topic :: Software Development", "Topic :: Scientific/Engineering", "Operating System :: Unix", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", ], entry_points={"console_scripts": ["spython=spython.client:main"]}, diff --git a/spython/main/parse/parsers/base.py b/spython/main/parse/parsers/base.py index bc37b642..cd64a1e0 100644 --- a/spython/main/parse/parsers/base.py +++ b/spython/main/parse/parsers/base.py @@ -155,6 +155,6 @@ def _replace_from_dict(self, string, args): string: the string with replacements made """ for key, value in args.items(): - if re.search(r"\$(" + key + r"|\{[^}]*\})", string): - string = re.sub(r"\$(" + key + r"|\{[^}]*\})", value, string) + if re.search("([$]" + key + "|[$][{]" + key + "[}])", string): + string = re.sub("([$]" + key + "|[$]{" + key + "[}])", value, string) return string diff --git a/spython/main/parse/writers/docker.py b/spython/main/parse/writers/docker.py index 3ede137a..bf59d52f 100644 --- a/spython/main/parse/writers/docker.py +++ b/spython/main/parse/writers/docker.py @@ -71,16 +71,16 @@ def validate_stage(self, parser): if parser.fromHeader is None: bot.exit("Dockerfile requires a fromHeader.") - # Parse the provided name - uri_regexes = [_reduced_uri, _default_uri, _docker_uri] + # Parse the provided name + uri_regexes = [_reduced_uri, _default_uri, _docker_uri] - for r in uri_regexes: - match = r.match(parser.fromHeader) - if match: - break + for r in uri_regexes: + match = r.match(parser.fromHeader) + if match: + break - if not match: - bot.exit("FROM header %s not valid." % parser.fromHeader) + if not match: + bot.exit("FROM header %s not valid." % parser.fromHeader) def convert(self, runscript="/bin/bash", force=False): """convert is called by the parent class to convert the recipe object diff --git a/spython/tests/testdata/docker2singularity/argsub.def b/spython/tests/testdata/docker2singularity/argsub.def new file mode 100644 index 00000000..c2d26546 --- /dev/null +++ b/spython/tests/testdata/docker2singularity/argsub.def @@ -0,0 +1,1575 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + singularity-cli/argsub.def at 99ebaa14c25bc928d1cfef0975dd1a258a1fb6ee · singularityhub/singularity-cli · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + +
+ +
+ + + + + + + +
+ + + +
+ + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + +
+ + + + + + +
+ + +
+ + + + + + +Permalink + +
+ +
+
+ + + 99ebaa14c2 + + + + +
+
+
+ Switch branches/tags + +
+ + + +
+ +
+ +
+ + +
+ +
+ + + + + + + + + + + + + + + + +
+ + + + +
+
+ +
+ +
+ +

+ singularity-cli/spython/tests/testdata/docker2singularity/argsub.def +

+ Go to file + +
+ + + + +
    +
  • + + Go to file + +
    • +
  • +
    • +
  • +
  • + + + Copy path + +
    • +
  • + + + + Copy permalink + + +
    • +
+
+
+ + + + + + + + + +
+ +
+
+
 
+
+ +
+
 
+ Cannot retrieve contributors at this time +
+
+ + + + + + + + +
+ +
+ + +
+ + 38 lines (28 sloc) + + 727 Bytes +
+ +
+ + +
+ Raw + Blame +
+ +
+ + + + + + + + + +
+
+ +
+
+ + + + + +
+
+
+ + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bootstrap: docker
From: nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04
Stage: spython-base
+
%files
./requirements.txt /workspace
%labels
maintainer="Dong Wang"
%post
CUDA_VERSION=11.1.1
OS_VERSION=20.04
+
+
+
+
PATH="/root/miniconda3/bin:${PATH}"
PATH="/root/miniconda3/bin:${PATH}"
DEBIAN_FRONTEND=noninteractive
+
SHELL ["/bin/bash", "-c"]
+
apt-get update && apt-get upgrade -y &&\
apt-get install -y wget python3-pip
+
python3 -m pip install --upgrade pip
+
cd /workspace
python3 -m pip install -r /workspace/requirements.txt and &&\
rm /workspace/requirements.txt
+
%environment
export PATH="/root/miniconda3/bin:${PATH}"
%runscript
cd /workspace
exec /bin/bash /bin/bash "$@"
%startscript
cd /workspace
exec /bin/bash /bin/bash "$@"
+
+ + + +
+ +
+ + + + +
+ + +
+ + +
 +
+ + +
+ +
+ + +
+ +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + diff --git a/spython/tests/testdata/docker2singularity/argsub.docker b/spython/tests/testdata/docker2singularity/argsub.docker new file mode 100644 index 00000000..2bce22d7 --- /dev/null +++ b/spython/tests/testdata/docker2singularity/argsub.docker @@ -0,0 +1,1522 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + singularity-cli/argsub.docker at 99ebaa14c25bc928d1cfef0975dd1a258a1fb6ee · singularityhub/singularity-cli · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + +
+ +
+ + + + + + + +
+ + + +
+ + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + +
+ + + + + + +
+ + +
+ + + + + + +Permalink + +
+ +
+
+ + + 99ebaa14c2 + + + + +
+
+
+ Switch branches/tags + +
+ + + +
+ +
+ +
+ + +
+ +
+ + + + + + + + + + + + + + + + +
+ + + + +
+
+ +
+ +
+ +

+ singularity-cli/spython/tests/testdata/docker2singularity/argsub.docker +

+ Go to file + +
+ + + + +
    +
  • + + Go to file + +
    • +
  • +
    • +
  • +
  • + + + Copy path + +
    • +
  • + + + + Copy permalink + + +
    • +
+
+
+ + + + + + + + + +
+ +
+
+
 
+
+ +
+
 
+ Cannot retrieve contributors at this time +
+
+ + + + + + + + +
+ +
+ + +
+ + 25 lines (16 sloc) + + 580 Bytes +
+ +
+ + +
+ Raw + Blame +
+ +
+ + + + + + + + + +
+
+ +
+
+ + + + + +
+
+
+ + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ARG CUDA_VERSION=11.1.1
ARG OS_VERSION=20.04
+
FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-devel-ubuntu${OS_VERSION}
+
LABEL maintainer="Dong Wang"
+
+
ENV PATH="/root/miniconda3/bin:${PATH}"
ARG PATH="/root/miniconda3/bin:${PATH}"
ARG DEBIAN_FRONTEND=noninteractive
+
SHELL ["/bin/bash", "-c"]
+
RUN apt-get update && apt-get upgrade -y &&\
apt-get install -y wget python3-pip
+
RUN python3 -m pip install --upgrade pip
+
WORKDIR /workspace
ADD ./requirements.txt /workspace
RUN python3 -m pip install -r /workspace/requirements.txt and &&\
rm /workspace/requirements.txt
+
CMD ["/bin/bash"]
+
+ + + +
+ +
+ + + + +
+ + +
+ + +
 +
+ + +
+ +
+ + +
+ +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + diff --git a/spython/version.py b/spython/version.py index 02cba16a..6248fe9b 100644 --- a/spython/version.py +++ b/spython/version.py @@ -5,7 +5,7 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -__version__ = "0.2.0" +__version__ = "0.2.1" AUTHOR = "Vanessa Sochat" AUTHOR_EMAIL = "vsoch@users.noreply.github.com" NAME = "spython" From 479b8b4450e2e42dc9835405064f268327aedccc Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 29 Mar 2022 03:46:40 -0600 Subject: [PATCH 2/2] lol oops do not program in middle of night! Signed-off-by: vsoch --- .../testdata/docker2singularity/argsub.def | 1593 +---------------- .../testdata/docker2singularity/argsub.docker | 1529 +--------------- 2 files changed, 44 insertions(+), 3078 deletions(-) diff --git a/spython/tests/testdata/docker2singularity/argsub.def b/spython/tests/testdata/docker2singularity/argsub.def index c2d26546..2bd0f0dc 100644 --- a/spython/tests/testdata/docker2singularity/argsub.def +++ b/spython/tests/testdata/docker2singularity/argsub.def @@ -1,1575 +1,38 @@ +Bootstrap: docker +From: nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04 +Stage: spython-base +%files +./requirements.txt /workspace +%labels +maintainer="Dong Wang" +%post +CUDA_VERSION=11.1.1 +OS_VERSION=20.04 +PATH="/root/miniconda3/bin:${PATH}" +PATH="/root/miniconda3/bin:${PATH}" +DEBIAN_FRONTEND=noninteractive - - - - - - - - - - +SHELL ["/bin/bash", "-c"] +apt-get update && apt-get upgrade -y &&\ +apt-get install -y wget python3-pip +python3 -m pip install --upgrade pip - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - singularity-cli/argsub.def at 99ebaa14c25bc928d1cfef0975dd1a258a1fb6ee · singularityhub/singularity-cli · GitHub - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Skip to content - - - - - - - - - - -
- -
- - - - - - - -
- - - -
- - - - - - - - - - -
-
-
- - - - - - - - - - - - - - - - -
- - - - - - -
- - -
- - - - - - -Permalink - -
- -
-
- - - 99ebaa14c2 - - - - -
-
-
- Switch branches/tags - -
- - - -
- -
- -
- - -
- -
- - - - - - - - - - - - - - - - -
- - - - -
-
- -
- -
- -

- singularity-cli/spython/tests/testdata/docker2singularity/argsub.def -

- Go to file - -
- - - - -
    -
  • - - Go to file - -
    • -
  • -
    • -
  • -
  • - - - Copy path - -
    • -
  • - - - - Copy permalink - - -
    • -
-
-
- - - - - - - - - -
- -
-
-
 
-
- -
-
 
- Cannot retrieve contributors at this time -
-
- - - - - - - - -
- -
- - -
- - 38 lines (28 sloc) - - 727 Bytes -
- -
- - -
- Raw - Blame -
- -
- - - - - - - - - -
-
- -
-
- - - - - -
-
-
- - - -
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Bootstrap: docker
From: nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04
Stage: spython-base
-
%files
./requirements.txt /workspace
%labels
maintainer="Dong Wang"
%post
CUDA_VERSION=11.1.1
OS_VERSION=20.04
-
-
-
-
PATH="/root/miniconda3/bin:${PATH}"
PATH="/root/miniconda3/bin:${PATH}"
DEBIAN_FRONTEND=noninteractive
-
SHELL ["/bin/bash", "-c"]
-
apt-get update && apt-get upgrade -y &&\
apt-get install -y wget python3-pip
-
python3 -m pip install --upgrade pip
-
cd /workspace
python3 -m pip install -r /workspace/requirements.txt and &&\
rm /workspace/requirements.txt
-
%environment
export PATH="/root/miniconda3/bin:${PATH}"
%runscript
cd /workspace
exec /bin/bash /bin/bash "$@"
%startscript
cd /workspace
exec /bin/bash /bin/bash "$@"
-
- - - -
- -
- - - - -
- - -
- - -
 -
- - -
- -
- - -
- -
-
- -
- - - - - - - - - - - - - - - - - - - - +cd /workspace +python3 -m pip install -r /workspace/requirements.txt and &&\ +rm /workspace/requirements.txt +%environment +export PATH="/root/miniconda3/bin:${PATH}" +%runscript +cd /workspace +exec /bin/bash /bin/bash "$@" +%startscript +cd /workspace +exec /bin/bash /bin/bash "$@" diff --git a/spython/tests/testdata/docker2singularity/argsub.docker b/spython/tests/testdata/docker2singularity/argsub.docker index 2bce22d7..7aef60f1 100644 --- a/spython/tests/testdata/docker2singularity/argsub.docker +++ b/spython/tests/testdata/docker2singularity/argsub.docker @@ -1,1522 +1,25 @@ +ARG CUDA_VERSION=11.1.1 +ARG OS_VERSION=20.04 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-devel-ubuntu${OS_VERSION} +LABEL maintainer="Dong Wang" +ENV PATH="/root/miniconda3/bin:${PATH}" +ARG PATH="/root/miniconda3/bin:${PATH}" +ARG DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-c"] - - - - - - - - - - +RUN apt-get update && apt-get upgrade -y &&\ +apt-get install -y wget python3-pip +RUN python3 -m pip install --upgrade pip +WORKDIR /workspace +ADD ./requirements.txt /workspace +RUN python3 -m pip install -r /workspace/requirements.txt and &&\ +rm /workspace/requirements.txt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - singularity-cli/argsub.docker at 99ebaa14c25bc928d1cfef0975dd1a258a1fb6ee · singularityhub/singularity-cli · GitHub - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Skip to content - - - - - - - - - - -
- -
- - - - - - - -
- - - -
- - - - - - - - - - -
-
-
- - - - - - - - - - - - - - - - -
- - - - - - -
- - -
- - - - - - -Permalink - -
- -
-
- - - 99ebaa14c2 - - - - -
-
-
- Switch branches/tags - -
- - - -
- -
- -
- - -
- -
- - - - - - - - - - - - - - - - -
- - - - -
-
- -
- -
- -

- singularity-cli/spython/tests/testdata/docker2singularity/argsub.docker -

- Go to file - -
- - - - -
    -
  • - - Go to file - -
    • -
  • -
    • -
  • -
  • - - - Copy path - -
    • -
  • - - - - Copy permalink - - -
    • -
-
-
- - - - - - - - - -
- -
-
-
 
-
- -
-
 
- Cannot retrieve contributors at this time -
-
- - - - - - - - -
- -
- - -
- - 25 lines (16 sloc) - - 580 Bytes -
- -
- - -
- Raw - Blame -
- -
- - - - - - - - - -
-
- -
-
- - - - - -
-
-
- - - -
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ARG CUDA_VERSION=11.1.1
ARG OS_VERSION=20.04
-
FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-devel-ubuntu${OS_VERSION}
-
LABEL maintainer="Dong Wang"
-
-
ENV PATH="/root/miniconda3/bin:${PATH}"
ARG PATH="/root/miniconda3/bin:${PATH}"
ARG DEBIAN_FRONTEND=noninteractive
-
SHELL ["/bin/bash", "-c"]
-
RUN apt-get update && apt-get upgrade -y &&\
apt-get install -y wget python3-pip
-
RUN python3 -m pip install --upgrade pip
-
WORKDIR /workspace
ADD ./requirements.txt /workspace
RUN python3 -m pip install -r /workspace/requirements.txt and &&\
rm /workspace/requirements.txt
-
CMD ["/bin/bash"]
-
- - - -
- -
- - - - -
- - -
- - -
 -
- - -
- -
- - -
- -
-
- -
- - - - - - - - - - - - - - - - - - - - - +CMD ["/bin/bash"]