From 111f31f87f92286e7a7414913cb5f7752be4a001 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Wed, 25 Jan 2023 17:47:23 +0000 Subject: [PATCH 01/34] Remove container engine specfic directory name This is in preparation for generalizing container engine support to allow the use of podman. --- Dockerfile.compute_worker | 4 ++-- Dockerfile.compute_worker_gpu | 4 ++-- {docker/compute_worker => compute_worker}/celery_config.py | 0 {docker/compute_worker => compute_worker}/compute_worker.py | 0 .../compute_worker_requirements.txt | 0 docker-compose.yml | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename {docker/compute_worker => compute_worker}/celery_config.py (100%) rename {docker/compute_worker => compute_worker}/compute_worker.py (100%) rename {docker/compute_worker => compute_worker}/compute_worker_requirements.txt (100%) diff --git a/Dockerfile.compute_worker b/Dockerfile.compute_worker index 77e0ef69d..482924931 100644 --- a/Dockerfile.compute_worker +++ b/Dockerfile.compute_worker @@ -6,10 +6,10 @@ ENV PYTHONUNBUFFERED 1 # Install Docker RUN apt-get update && curl -fsSL https://get.docker.com | sh -ADD docker/compute_worker/compute_worker_requirements.txt . +ADD compute_worker/compute_worker_requirements.txt . RUN pip install -r compute_worker_requirements.txt -ADD docker/compute_worker . +ADD compute_worker . CMD celery -A compute_worker worker \ -l info \ diff --git a/Dockerfile.compute_worker_gpu b/Dockerfile.compute_worker_gpu index f2110647c..e559a0667 100644 --- a/Dockerfile.compute_worker_gpu +++ b/Dockerfile.compute_worker_gpu @@ -19,9 +19,9 @@ RUN apt-get update && apt-get install -y nvidia-docker2 ENV NVIDIA_DOCKER 1 # Python reqs and actual worker stuff -ADD docker/compute_worker/compute_worker_requirements.txt . +ADD compute_worker/compute_worker_requirements.txt . RUN pip3 install -r compute_worker_requirements.txt -ADD docker/compute_worker . +ADD compute_worker . CMD celery -A compute_worker worker \ -l info \ diff --git a/docker/compute_worker/celery_config.py b/compute_worker/celery_config.py similarity index 100% rename from docker/compute_worker/celery_config.py rename to compute_worker/celery_config.py diff --git a/docker/compute_worker/compute_worker.py b/compute_worker/compute_worker.py similarity index 100% rename from docker/compute_worker/compute_worker.py rename to compute_worker/compute_worker.py diff --git a/docker/compute_worker/compute_worker_requirements.txt b/compute_worker/compute_worker_requirements.txt similarity index 100% rename from docker/compute_worker/compute_worker_requirements.txt rename to compute_worker/compute_worker_requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 3c6e24a3a..abdbe2e0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -201,7 +201,7 @@ services: - django - rabbit volumes: - - ./docker/compute_worker:/app + - ./compute_worker:/app - ${HOST_DIRECTORY:-/tmp/codabench}:/codabench # Actual connection back to docker parent to run things - /var/run/docker.sock:/var/run/docker.sock From fd3f38b5ccb267ddd2bf063100f8f6929e1e7b33 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Wed, 25 Jan 2023 17:58:09 +0000 Subject: [PATCH 02/34] Allow container engine to be configured Add support for configuring the container engine through an environment variable ( CONTAINER_ENGINE_EXECUTABLE ). --- compute_worker/compute_worker.py | 71 +++++++++++++++++--------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 110010ad3..a45f651cc 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -65,6 +65,14 @@ STATUS_FAILED, ) +# Setup the container engine that we are using +if os.environ.get("CONTAINER_ENGINE_EXECUTABLE"): + CONTAINER_ENGINE_EXECUTABLE = os.environ.get("CONTAINER_ENGINE_EXECUTABLE") +# We could probably depreciate this now that we can specify the executable +elif os.environ.get("NVIDIA_DOCKER"): + CONTAINER_ENGINE_EXECUTABLE = "nvidia-docker" +else: + CONTAINER_ENGINE_EXECUTABLE = "docker" class SubmissionException(Exception): pass @@ -181,7 +189,7 @@ def __init__(self, run_args): self.user_pk = run_args["user_pk"] self.submission_id = run_args["id"] self.submissions_api_url = run_args["submissions_api_url"] - self.docker_image = run_args["docker_image"] + self.container_image = run_args["docker_image"] self.secret = run_args["secret"] self.prediction_result = run_args["prediction_result"] self.scoring_result = run_args.get("scoring_result") @@ -221,7 +229,7 @@ def __init__(self, run_args): self.requests_session.mount('https://', adapter) async def watch_detailed_results(self): - """Watches files alongside scoring + program docker containers, currently only used + """Watches files alongside scoring + program containers, currently only used for detailed_results.html""" if not self.detailed_results_url: return @@ -314,15 +322,15 @@ def _update_status(self, status, extra_information=None): # }) self._update_submission(data) - def _get_docker_image(self, image_name): - logger.info("Running docker pull for image: {}".format(image_name)) + def _get_container_image(self, image_name): + logger.info("Running pull for image: {}".format(image_name)) try: - cmd = ['docker', 'pull', image_name] - docker_pull = check_output(cmd) - logger.info("Docker pull complete for image: {0} with output of {1}".format(image_name, docker_pull)) + cmd = [CONTAINER_ENGINE_EXECUTABLE, 'pull', image_name] + container_engine_pull = check_output(cmd) + logger.info("Pull complete for image: {0} with output of {1}".format(image_name, container_engine_pull)) except CalledProcessError: - logger.info("Docker pull for image: {} returned a non-zero exit code!") - raise SubmissionException(f"Docker pull for {image_name} failed!") + logger.info("Pull for image: {} returned a non-zero exit code!") + raise SubmissionException(f"Pull for {image_name} failed!") def _get_bundle(self, url, destination, cache=True): """Downloads zip from url and unzips into destination. If cache=True then url is hashed and checked @@ -357,17 +365,17 @@ def _get_bundle(self, url, destination, cache=True): # Give back zip file path for other uses, i.e. md5'ing the zip to ID it return bundle_file - async def _run_docker_cmd(self, docker_cmd, kind): + async def _run_container_engine_cmd(self, engine_cmd, kind): """This runs a command and asynchronously writes the data to both a storage file and a socket - :param docker_cmd: the list of docker command arguments + :param engine_cmd: the list of container engine command arguments :param kind: either 'ingestion' or 'program' :return: """ start = time.time() proc = await asyncio.create_subprocess_exec( - *docker_cmd, + *engine_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) @@ -442,15 +450,15 @@ async def _run_docker_cmd(self, docker_cmd, kind): await websocket.close() def _get_host_path(self, *paths): - """Turns an absolute path inside our docker container, into what the path + """Turns an absolute path inside our container, into what the path would be on the host machine""" # Take our list of paths and smash 'em together path = os.path.join(*paths) - # pull front of path, which points to the location inside docker + # pull front of path, which points to the location inside the container path = path[len(BASE_DIR):] - # add host to front, so when we run commands in docker on the host they + # add host to front, so when we run commands in the container on the host they # can be seen properly path = os.path.join(HOST_DIRECTORY, path) return path @@ -494,13 +502,8 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): ) return - if os.environ.get("NVIDIA_DOCKER"): - docker_process_name = "nvidia-docker" - else: - docker_process_name = "docker" - - docker_cmd = [ - docker_process_name, + engine_cmd = [ + CONTAINER_ENGINE_EXECUTABLE, 'run', # Remove it after run '--rm', @@ -528,21 +531,21 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): else: ingested_program_location = "program" - docker_cmd += ['-v', f'{self._get_host_path(self.root_dir, ingested_program_location)}:/app/ingested_program'] + engine_cmd += ['-v', f'{self._get_host_path(self.root_dir, ingested_program_location)}:/app/ingested_program'] if self.input_data: - docker_cmd += ['-v', f'{self._get_host_path(self.root_dir, "input_data")}:/app/input_data'] + engine_cmd += ['-v', f'{self._get_host_path(self.root_dir, "input_data")}:/app/input_data'] if self.is_scoring: # For scoring programs, we want to have a shared directory just in case we have an ingestion program. # This will add the share dir regardless of ingestion or scoring, as long as we're `is_scoring` - docker_cmd += ['-v', f'{self._get_host_path(self.root_dir, "shared")}:/app/shared'] + engine_cmd += ['-v', f'{self._get_host_path(self.root_dir, "shared")}:/app/shared'] # Input from submission (or submission + ingestion combo) - docker_cmd += ['-v', f'{self._get_host_path(self.input_dir)}:/app/input'] + engine_cmd += ['-v', f'{self._get_host_path(self.input_dir)}:/app/input'] # Set the image name (i.e. "codalab/codalab-legacy") for the container - docker_cmd += [self.docker_image] + engine_cmd += [self.container_image] # Handle Legacy competitions by replacing anything in the run command command = replace_legacy_metadata_command( @@ -553,12 +556,12 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): ) # Append the actual program to run - docker_cmd += command.split(' ') + engine_cmd += command.split(' ') - logger.info(f"Running program = {' '.join(docker_cmd)}") + logger.info(f"Running program = {' '.join(engine_cmd)}") - # This runs the docker command and asynchronously passes data back via websocket - return await self._run_docker_cmd(docker_cmd, kind=kind) + # This runs the container engine command and asynchronously passes data back via websocket + return await self._run_container_engine_cmd(engine_cmd, kind=kind) def _put_dir(self, url, directory): logger.info("Putting dir %s in %s" % (directory, url)) @@ -649,9 +652,9 @@ def prepare(self): for filename in glob.iglob(self.root_dir + '**/*.*', recursive=True): logger.info(filename) - # Before the run starts we want to download docker images, they may take a while to download + # Before the run starts we want to download images, they may take a while to download # and to do this during the run would subtract from the participants time. - self._get_docker_image(self.docker_image) + self._get_container_image(self.container_image) def start(self): if not self.is_scoring: @@ -690,7 +693,7 @@ def start(self): else: program_to_kill = self.program_container_name # Try and stop the program. If stop does not succeed - kill_code = subprocess.call(['docker', 'stop', str(program_to_kill)]) + kill_code = subprocess.call([CONTAINER_ENGINE_EXECUTABLE, 'stop', str(program_to_kill)]) logger.info(f'Kill process returned {kill_code}') if kind == 'program': self.program_exit_code = return_code From 36e3327c720c701ed6f464b2fd6ae87acf75d697 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Mon, 30 Jan 2023 17:11:09 +0000 Subject: [PATCH 03/34] Ensure host directories exist Docker will create them, but other container engines like podman may not. --- compute_worker/compute_worker.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index a45f651cc..629d003b0 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -451,7 +451,9 @@ async def _run_container_engine_cmd(self, engine_cmd, kind): def _get_host_path(self, *paths): """Turns an absolute path inside our container, into what the path - would be on the host machine""" + would be on the host machine. We also ensure that the directory exists, + docker will create if necessary, but other container engines such as + podman may not.""" # Take our list of paths and smash 'em together path = os.path.join(*paths) @@ -461,6 +463,10 @@ def _get_host_path(self, *paths): # add host to front, so when we run commands in the container on the host they # can be seen properly path = os.path.join(HOST_DIRECTORY, path) + + # Create if necessary + os.makedirs(path, exist_ok=True) + return path async def _run_program_directory(self, program_dir, kind, can_be_output=False): From 63709e582b8749ff28d749c959a50b57fc417053 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Wed, 25 Jan 2023 18:36:22 +0000 Subject: [PATCH 04/34] Add Containerfile for rootless podman container This Containerfile allows rootless Podman in Podman (PINP). --- Containerfile.compute_worker_podman | 63 +++++++++++++++++++++++++++++ podman/containers.conf | 12 ++++++ podman/worker-containers.conf | 5 +++ podman/worker-storage.conf | 5 +++ 4 files changed, 85 insertions(+) create mode 100644 Containerfile.compute_worker_podman create mode 100644 podman/containers.conf create mode 100644 podman/worker-containers.conf create mode 100644 podman/worker-storage.conf diff --git a/Containerfile.compute_worker_podman b/Containerfile.compute_worker_podman new file mode 100644 index 000000000..67efbed76 --- /dev/null +++ b/Containerfile.compute_worker_podman @@ -0,0 +1,63 @@ +FROM fedora:37 + +# Include deps +RUN dnf -y update && \ + dnf -y install podman fuse-overlayfs python3.8 python3-pip \ + --exclude container-selinux && \ + dnf clean all && \ + rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Setup user +RUN useradd worker; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; + +# Copy over the podman container configuration +COPY podman/containers.conf /etc/containers/containers.conf +COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf + +# Copy over the podman storage configuration +COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf + +RUN mkdir -p /home/worker/.local/share/containers && \ + chown worker:worker -R /home/worker && \ + chmod 644 /etc/containers/containers.conf + +# Copy & modify the defaults to provide reference if runtime changes needed. +# Changes here are required for running with fuse-overlay storage inside container. +RUN sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf \ + > /etc/containers/storage.conf + +# Add volume for containers +VOLUME /home/worker/.local/share/containers + +# Create directory for tmp space +RUN mkdir /codabench && \ + chown worker:worker /codabench + +# Set up podman registry for dockerhub +RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf + +# This makes output not buffer and return immediately, nice for seeing results in stdout +ENV PYTHONUNBUFFERED 1 +ENV CONTAINER_ENGINE_EXECUTABLE podman + +# Get pip for 3.8 +RUN python3.8 -m ensurepip --upgrade + +WORKDIR /home/worker/compute_worker + +ADD compute_worker/ /home/worker/compute_worker + +RUN chown worker:worker -R /home/worker/compute_worker + +RUN pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt + +CMD celery -A compute_worker worker \ + -l info \ + -Q compute-worker \ + -n compute-worker@%n \ + --concurrency=1 diff --git a/podman/containers.conf b/podman/containers.conf new file mode 100644 index 000000000..220c1f850 --- /dev/null +++ b/podman/containers.conf @@ -0,0 +1,12 @@ +[containers] +netns="host" +userns="host" +ipcns="host" +utsns="host" +cgroupns="host" +cgroups="disabled" +log_driver = "k8s-file" +[engine] +cgroup_manager = "cgroupfs" +events_logger="file" +runtime="crun" diff --git a/podman/worker-containers.conf b/podman/worker-containers.conf new file mode 100644 index 000000000..2bdd95a3b --- /dev/null +++ b/podman/worker-containers.conf @@ -0,0 +1,5 @@ +[containers] +volumes = [ + "/proc:/proc", +] +default_sysctls = [] diff --git a/podman/worker-storage.conf b/podman/worker-storage.conf new file mode 100644 index 000000000..0b1afb08b --- /dev/null +++ b/podman/worker-storage.conf @@ -0,0 +1,5 @@ +[storage] +driver = "overlay" + +[storage.options.overlay] +mount_program = "/usr/bin/fuse-overlayfs" From 43e01d4bc3de26e8339ddb1463eef7d960ddb3af Mon Sep 17 00:00:00 2001 From: dinh_tuan tran Date: Mon, 6 Feb 2023 14:40:39 +0100 Subject: [PATCH 05/34] Adding Containerfile for building podman gpu compute_worker image --- Containerfile.compute_worker_podman_gpu | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 Containerfile.compute_worker_podman_gpu diff --git a/Containerfile.compute_worker_podman_gpu b/Containerfile.compute_worker_podman_gpu new file mode 100644 index 000000000..e8a9c8e99 --- /dev/null +++ b/Containerfile.compute_worker_podman_gpu @@ -0,0 +1,65 @@ +FROM fedora:37 + +# Include deps +RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | sudo tee /etc/yum.repos.d/cuda.repo && \ + curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo && \ + rpm -Uvh http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm && \ + rpm -Uvh http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm && \ + dnf -y update && \ + dnf module install -y nvidia-driver:latest-dkms && \ + dnf -y install podman fuse-overlayfs python3.8 python3-pip nvidia-container-runtime nvidia-container-toolkit \ + cuda --exclude container-selinux && \ + dnf clean all && \ + rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Setup user +RUN useradd worker; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; + +# Copy over the podman container configuration +COPY podman/containers.conf /etc/containers/containers.conf +COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf + +# Copy over the podman storage configuration +COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf + +RUN mkdir -p /home/worker/.local/share/containers && \ + chown worker:worker -R /home/worker && \ + chmod 644 /etc/containers/containers.conf + +# Copy & modify the defaults to provide reference if runtime changes needed. +# Changes here are required for running with fuse-overlay storage inside container. +RUN sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf \ + > /etc/containers/storage.conf + +# Add volume for containers +VOLUME /home/worker/.local/share/containers + +# This makes output not buffer and return immediately, nice for seeing results in stdout +ENV PYTHONUNBUFFERED 1 +ENV CONTAINER_ENGINE_EXECUTABLE podman + +# Create directory for tmp space +RUN mkdir /codabench && \ + chown worker:worker /codabench && \ +# Set up podman registry for dockerhub + echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \ +# Get pip for 3.8 + python3.8 -m ensurepip --upgrade + +WORKDIR /home/worker/compute_worker + +ADD compute_worker/ /home/worker/compute_worker + +RUN chown worker:worker -R /home/worker/compute_worker && \ + pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt + +CMD nvidia-smi && celery -A compute_worker worker \ + -l info \ + -Q compute-worker \ + -n compute-worker@%n \ + --concurrency=1 From fd7d38fec91d88df823af8be9cabd1eba02672f2 Mon Sep 17 00:00:00 2001 From: dinh_tuan tran Date: Mon, 6 Feb 2023 19:10:12 +0100 Subject: [PATCH 06/34] Adding nvidia runtime config --- Containerfile.compute_worker_podman_gpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Containerfile.compute_worker_podman_gpu b/Containerfile.compute_worker_podman_gpu index e8a9c8e99..f3eba9454 100644 --- a/Containerfile.compute_worker_podman_gpu +++ b/Containerfile.compute_worker_podman_gpu @@ -34,7 +34,8 @@ RUN sed -e 's|^#mount_program|mount_program|g' \ -e '/additionalimage.*/a "/var/lib/shared",' \ -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ /usr/share/containers/storage.conf \ - > /etc/containers/storage.conf + > /etc/containers/storage.conf; sed -i 's/^#no-cgroups = false/no-cgroups = true/;' /etc/nvidia-container-runtime/config.toml + # Add volume for containers VOLUME /home/worker/.local/share/containers From 3c27d79478aa493b2b58c5b95c551283a6eeed9b Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Tue, 14 Feb 2023 11:56:09 -0800 Subject: [PATCH 07/34] Fix issue with shadow-utils --- Containerfile.compute_worker_podman | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Containerfile.compute_worker_podman b/Containerfile.compute_worker_podman index 67efbed76..9c96ead37 100644 --- a/Containerfile.compute_worker_podman +++ b/Containerfile.compute_worker_podman @@ -2,6 +2,8 @@ FROM fedora:37 # Include deps RUN dnf -y update && \ + # https://bugzilla.redhat.com/show_bug.cgi?id=1995337#c3 + rpm --setcaps shadow-utils 2>/dev/null && \ dnf -y install podman fuse-overlayfs python3.8 python3-pip \ --exclude container-selinux && \ dnf clean all && \ From d134c1e727ec402a065cefd00e0d4452f5282c9d Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Tue, 14 Feb 2023 12:21:45 -0800 Subject: [PATCH 08/34] Remove sudo The podman container is built as root --- Containerfile.compute_worker_podman_gpu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Containerfile.compute_worker_podman_gpu b/Containerfile.compute_worker_podman_gpu index f3eba9454..0ef68e1a2 100644 --- a/Containerfile.compute_worker_podman_gpu +++ b/Containerfile.compute_worker_podman_gpu @@ -1,8 +1,8 @@ FROM fedora:37 # Include deps -RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | sudo tee /etc/yum.repos.d/cuda.repo && \ - curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo && \ +RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | tee /etc/yum.repos.d/cuda.repo && \ + curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | tee /etc/yum.repos.d/nvidia-docker.repo && \ rpm -Uvh http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm && \ rpm -Uvh http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm && \ dnf -y update && \ From c83e043e6041746447e81925951c435310602b48 Mon Sep 17 00:00:00 2001 From: dinh_tuan tran Date: Wed, 29 Mar 2023 10:09:06 +0200 Subject: [PATCH 09/34] Optimize Containerfile in order to reduce image size --- Containerfile.compute_worker_podman | 59 ++++++++++++----------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/Containerfile.compute_worker_podman b/Containerfile.compute_worker_podman index 9c96ead37..3b1673a85 100644 --- a/Containerfile.compute_worker_podman +++ b/Containerfile.compute_worker_podman @@ -7,56 +7,45 @@ RUN dnf -y update && \ dnf -y install podman fuse-overlayfs python3.8 python3-pip \ --exclude container-selinux && \ dnf clean all && \ - rm -rf /var/cache /var/log/dnf* /var/log/yum.* + rm -rf /var/cache /var/log/dnf* /var/log/yum.* && \ + # Setup user + useradd worker; \ + echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ + echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; -# Setup user -RUN useradd worker; \ -echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ -echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; - -# Copy over the podman container configuration +# Copy over the podman container and storage configurations COPY podman/containers.conf /etc/containers/containers.conf -COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf - -# Copy over the podman storage configuration -COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf - -RUN mkdir -p /home/worker/.local/share/containers && \ - chown worker:worker -R /home/worker && \ - chmod 644 /etc/containers/containers.conf +COPY podman/worker-*.conf /home/worker/.config/containers/ # Copy & modify the defaults to provide reference if runtime changes needed. # Changes here are required for running with fuse-overlay storage inside container. -RUN sed -e 's|^#mount_program|mount_program|g' \ - -e '/additionalimage.*/a "/var/lib/shared",' \ - -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ - /usr/share/containers/storage.conf \ - > /etc/containers/storage.conf +RUN mkdir -p /home/worker/.local/share/containers && \ + chown worker:worker -R /home/worker && \ + chmod 644 /etc/containers/containers.conf && \ + sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf > /etc/containers/storage.conf # Add volume for containers VOLUME /home/worker/.local/share/containers -# Create directory for tmp space -RUN mkdir /codabench && \ - chown worker:worker /codabench - -# Set up podman registry for dockerhub -RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf - # This makes output not buffer and return immediately, nice for seeing results in stdout -ENV PYTHONUNBUFFERED 1 -ENV CONTAINER_ENGINE_EXECUTABLE podman - -# Get pip for 3.8 -RUN python3.8 -m ensurepip --upgrade +ENV PYTHONUNBUFFERED=1 CONTAINER_ENGINE_EXECUTABLE=podman WORKDIR /home/worker/compute_worker ADD compute_worker/ /home/worker/compute_worker -RUN chown worker:worker -R /home/worker/compute_worker - -RUN pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt +# Create directory for tmp space +# Set up podman registry for dockerhub +RUN mkdir /codabench && \ + chown worker:worker /codabench && \ + echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \ + # Get pip for 3.8 + python3.8 -m ensurepip --upgrade && \ + chown worker:worker -R /home/worker/compute_worker && \ + pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt CMD celery -A compute_worker worker \ -l info \ From abf66f1437118c29a7b64011ffb77bdd123d4fe3 Mon Sep 17 00:00:00 2001 From: bbearce Date: Wed, 29 Mar 2023 12:47:46 +0000 Subject: [PATCH 10/34] podman import error during tests --- src/apps/competitions/tests/test_legacy_command_replacement.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/apps/competitions/tests/test_legacy_command_replacement.py b/src/apps/competitions/tests/test_legacy_command_replacement.py index 52b031353..d7dc7dec8 100644 --- a/src/apps/competitions/tests/test_legacy_command_replacement.py +++ b/src/apps/competitions/tests/test_legacy_command_replacement.py @@ -1,6 +1,5 @@ from django.test import TestCase -from docker.compute_worker.compute_worker import replace_legacy_metadata_command - +from compute_worker.compute_worker import replace_legacy_metadata_command class LegacyConverterCommandTests(TestCase): def test_ingestion_command_is_converted_correctly(self): From 8327ff6ddb1197417b13939d074543622b292c3d Mon Sep 17 00:00:00 2001 From: didayolo Date: Wed, 29 Mar 2023 14:47:53 +0200 Subject: [PATCH 11/34] Add checkbox for terms and conditions --- src/templates/registration/signup.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/templates/registration/signup.html b/src/templates/registration/signup.html index f5d5ca78b..426d05bcc 100644 --- a/src/templates/registration/signup.html +++ b/src/templates/registration/signup.html @@ -87,6 +87,11 @@

{% endif %} +
+ + +
+
From a463a0ff85153c194ff03a05d7bd6e43a78af7f8 Mon Sep 17 00:00:00 2001 From: bbearce Date: Wed, 29 Mar 2023 12:53:51 +0000 Subject: [PATCH 12/34] flake error --- src/apps/competitions/tests/test_legacy_command_replacement.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/competitions/tests/test_legacy_command_replacement.py b/src/apps/competitions/tests/test_legacy_command_replacement.py index d7dc7dec8..18a1fbfeb 100644 --- a/src/apps/competitions/tests/test_legacy_command_replacement.py +++ b/src/apps/competitions/tests/test_legacy_command_replacement.py @@ -1,6 +1,7 @@ from django.test import TestCase from compute_worker.compute_worker import replace_legacy_metadata_command + class LegacyConverterCommandTests(TestCase): def test_ingestion_command_is_converted_correctly(self): v15 = 'python $ingestion_program/ingestion.py $input $output $ingestion_program $submission_program' From 32249a34c958ce14843c958209b61da8c81a8c53 Mon Sep 17 00:00:00 2001 From: dinh_tuan tran Date: Wed, 29 Mar 2023 15:19:26 +0200 Subject: [PATCH 13/34] Revert file --- Containerfile.compute_worker_podman | 59 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/Containerfile.compute_worker_podman b/Containerfile.compute_worker_podman index 3b1673a85..9c96ead37 100644 --- a/Containerfile.compute_worker_podman +++ b/Containerfile.compute_worker_podman @@ -7,45 +7,56 @@ RUN dnf -y update && \ dnf -y install podman fuse-overlayfs python3.8 python3-pip \ --exclude container-selinux && \ dnf clean all && \ - rm -rf /var/cache /var/log/dnf* /var/log/yum.* && \ - # Setup user - useradd worker; \ - echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ - echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; + rm -rf /var/cache /var/log/dnf* /var/log/yum.* -# Copy over the podman container and storage configurations +# Setup user +RUN useradd worker; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; + +# Copy over the podman container configuration COPY podman/containers.conf /etc/containers/containers.conf -COPY podman/worker-*.conf /home/worker/.config/containers/ +COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf + +# Copy over the podman storage configuration +COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf -# Copy & modify the defaults to provide reference if runtime changes needed. -# Changes here are required for running with fuse-overlay storage inside container. RUN mkdir -p /home/worker/.local/share/containers && \ chown worker:worker -R /home/worker && \ - chmod 644 /etc/containers/containers.conf && \ - sed -e 's|^#mount_program|mount_program|g' \ - -e '/additionalimage.*/a "/var/lib/shared",' \ - -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ - /usr/share/containers/storage.conf > /etc/containers/storage.conf + chmod 644 /etc/containers/containers.conf + +# Copy & modify the defaults to provide reference if runtime changes needed. +# Changes here are required for running with fuse-overlay storage inside container. +RUN sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf \ + > /etc/containers/storage.conf # Add volume for containers VOLUME /home/worker/.local/share/containers +# Create directory for tmp space +RUN mkdir /codabench && \ + chown worker:worker /codabench + +# Set up podman registry for dockerhub +RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf + # This makes output not buffer and return immediately, nice for seeing results in stdout -ENV PYTHONUNBUFFERED=1 CONTAINER_ENGINE_EXECUTABLE=podman +ENV PYTHONUNBUFFERED 1 +ENV CONTAINER_ENGINE_EXECUTABLE podman + +# Get pip for 3.8 +RUN python3.8 -m ensurepip --upgrade WORKDIR /home/worker/compute_worker ADD compute_worker/ /home/worker/compute_worker -# Create directory for tmp space -# Set up podman registry for dockerhub -RUN mkdir /codabench && \ - chown worker:worker /codabench && \ - echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \ - # Get pip for 3.8 - python3.8 -m ensurepip --upgrade && \ - chown worker:worker -R /home/worker/compute_worker && \ - pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt +RUN chown worker:worker -R /home/worker/compute_worker + +RUN pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt CMD celery -A compute_worker worker \ -l info \ From 64d371d98ee2ef4d8497d0d8b3c3ee560a946fc9 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 3 Apr 2023 17:55:36 +0500 Subject: [PATCH 14/34] detailed competition results added --- .gitignore | 3 +++ docker-compose.yml | 3 ++- src/apps/api/views/competitions.py | 1 + src/apps/competitions/urls.py | 1 + src/apps/competitions/views.py | 4 ++++ .../competitions/detail/_detailed_results.tag | 16 ++++++++++++++++ .../riot/competitions/detail/leaderboards.tag | 8 ++++++++ src/templates/competitions/detailed_results.html | 4 ++++ 8 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 src/static/riot/competitions/detail/_detailed_results.tag create mode 100644 src/templates/competitions/detailed_results.html diff --git a/.gitignore b/.gitignore index ad8e29a2f..b5b34f56a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ docker-compose.override.yml server_config.yaml /graphs/ /codabench/ + +.DS_Store +.DS_Store? diff --git a/docker-compose.yml b/docker-compose.yml index b421f6341..ac91386a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -192,7 +192,8 @@ services: memory: 256M compute_worker: - command: bash -c "watchmedo auto-restart -p '*.py' --recursive -- celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n" + # command: bash -c "watchmedo auto-restart -p '*.py' --recursive -- celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n" + command: bash -c "celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n" working_dir: /app build: context: . diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index a8cbf7546..b6c4de12b 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -474,6 +474,7 @@ def get_leaderboard(self, request, pk): if submission_key not in submissions_keys: submissions_keys[submission_key] = len(response['submissions']) response['submissions'].append({ + 'id' : submission['id'], 'owner': submission['display_name'] or submission['owner'], 'scores': [], 'fact_sheet_answers': submission['fact_sheet_answers'], diff --git a/src/apps/competitions/urls.py b/src/apps/competitions/urls.py index 6f9e29ba0..a3d4419a9 100644 --- a/src/apps/competitions/urls.py +++ b/src/apps/competitions/urls.py @@ -12,4 +12,5 @@ path('edit//', views.CompetitionForm.as_view(), name="edit"), path('upload/', views.CompetitionUpload.as_view(), name="upload"), path('public/', views.CompetitionPublic.as_view(), name="public"), + path('/detailed_results//', views.CompetitionDetailedResults.as_view(), name="detailed_results"), ] diff --git a/src/apps/competitions/views.py b/src/apps/competitions/views.py index 09e30e822..c236236c9 100644 --- a/src/apps/competitions/views.py +++ b/src/apps/competitions/views.py @@ -17,6 +17,7 @@ class CompetitionForm(LoginRequiredMixin, TemplateView): template_name = 'competitions/form.html' + class CompetitionUpload(LoginRequiredMixin, TemplateView): template_name = 'competitions/upload.html' @@ -33,3 +34,6 @@ def get_object(self, *args, **kwargs): if is_creator or is_collaborator or competition.published or valid_secret_key: return competition raise Http404() + +class CompetitionDetailedResults(LoginRequiredMixin, TemplateView): + template_name = 'competitions/detailed_results.html' diff --git a/src/static/riot/competitions/detail/_detailed_results.tag b/src/static/riot/competitions/detail/_detailed_results.tag new file mode 100644 index 000000000..946657dc3 --- /dev/null +++ b/src/static/riot/competitions/detail/_detailed_results.tag @@ -0,0 +1,16 @@ + +

Detailed Results

+ + + + \ No newline at end of file diff --git a/src/static/riot/competitions/detail/leaderboards.tag b/src/static/riot/competitions/detail/leaderboards.tag index 05dc2c3dd..f23d59a01 100644 --- a/src/static/riot/competitions/detail/leaderboards.tag +++ b/src/static/riot/competitions/detail/leaderboards.tag @@ -30,11 +30,13 @@ Task: { task.name } + # Participant {column.title} + Detailed Results @@ -55,6 +57,7 @@ { submission.owner } { submission.organization.name } { get_score(column, submission) } + Show detailed results @@ -67,6 +70,7 @@ self.filtered_columns = [] self.phase_id = null self.competition_id = null + self.enable_detailed_results = false self.get_score = function(column, submission) { if(column.task_id === -1){ @@ -117,6 +121,8 @@ CODALAB.api.get_leaderboard_for_render(self.phase_id) .done(responseData => { self.selected_leaderboard = responseData + + self.columns = [] // Make fake task and columns for Metadata so it can be filtered like columns if(self.selected_leaderboard.fact_sheet_keys){ @@ -154,6 +160,8 @@ CODALAB.events.on('competition_loaded', (competition) => { self.competition_id = competition.id self.opts.is_admin ? self.show_download = "visible": self.show_download = "hidden" + self.enable_detailed_results = competition.enable_detailed_results + }) CODALAB.events.on('submission_changed_on_leaderboard', self.update_leaderboard) diff --git a/src/templates/competitions/detailed_results.html b/src/templates/competitions/detailed_results.html new file mode 100644 index 000000000..e7c14cfa6 --- /dev/null +++ b/src/templates/competitions/detailed_results.html @@ -0,0 +1,4 @@ +{% extends "base.html" %} +{% block content %} + +{% endblock %} \ No newline at end of file From c7995481349e646d93d06da8adcf626ba0393077 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 4 Apr 2023 18:06:25 +0500 Subject: [PATCH 15/34] circle ci feedabck addressed, docker-compose file reverted to original form --- docker-compose.yml | 3 +-- src/apps/competitions/views.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ac91386a3..b421f6341 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -192,8 +192,7 @@ services: memory: 256M compute_worker: - # command: bash -c "watchmedo auto-restart -p '*.py' --recursive -- celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n" - command: bash -c "celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n" + command: bash -c "watchmedo auto-restart -p '*.py' --recursive -- celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n" working_dir: /app build: context: . diff --git a/src/apps/competitions/views.py b/src/apps/competitions/views.py index c236236c9..7a4045f7f 100644 --- a/src/apps/competitions/views.py +++ b/src/apps/competitions/views.py @@ -17,7 +17,6 @@ class CompetitionForm(LoginRequiredMixin, TemplateView): template_name = 'competitions/form.html' - class CompetitionUpload(LoginRequiredMixin, TemplateView): template_name = 'competitions/upload.html' @@ -35,5 +34,6 @@ def get_object(self, *args, **kwargs): return competition raise Http404() + class CompetitionDetailedResults(LoginRequiredMixin, TemplateView): template_name = 'competitions/detailed_results.html' From 1038d55d376789ef93156aaaff64319ab99c25ed Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 4 Apr 2023 18:18:58 +0500 Subject: [PATCH 16/34] white spaces removed to pass circle ci tests --- src/apps/api/views/competitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index b6c4de12b..fde3aedf9 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -474,7 +474,7 @@ def get_leaderboard(self, request, pk): if submission_key not in submissions_keys: submissions_keys[submission_key] = len(response['submissions']) response['submissions'].append({ - 'id' : submission['id'], + 'id': submission['id'], 'owner': submission['display_name'] or submission['owner'], 'scores': [], 'fact_sheet_answers': submission['fact_sheet_answers'], From 2a4c6328903d4de8c09c707f4e56ed1cc85b2923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Tue, 4 Apr 2023 18:03:16 +0200 Subject: [PATCH 17/34] Update test_competitions.py --- src/tests/functional/test_competitions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/functional/test_competitions.py b/src/tests/functional/test_competitions.py index cdbebe8a2..3db8bd355 100644 --- a/src/tests/functional/test_competitions.py +++ b/src/tests/functional/test_competitions.py @@ -11,8 +11,8 @@ from tasks.models import Task from ..utils import SeleniumTestCase -SHORT_WAIT = 0.1 -LONG_WAIT = 2 +SHORT_WAIT = 0.2 +LONG_WAIT = 4 class TestCompetitions(SeleniumTestCase): From 2d736c3709161a9df5abf9fa48fa79f6aa767cdd Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 4 Apr 2023 12:05:46 -0400 Subject: [PATCH 18/34] change cdn from unpkg to cdn.jsdeliver --- src/templates/base.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/base.html b/src/templates/base.html index 850123411..127c812c8 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -11,7 +11,7 @@ - + @@ -227,7 +227,7 @@

CodaBench

- + From b400b5d0296ee77dd11d803632aee0e2b86bd465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Tue, 4 Apr 2023 18:21:13 +0200 Subject: [PATCH 19/34] Update pull_request_template.md --- .github/pull_request_template.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 599f87459..87a6160da 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,9 +14,6 @@ more information is better. ... -# Known issues to be addressed in a separate PR -... - # A checklist for hand testing - [ ] add checklist here @@ -26,15 +23,12 @@ more information is better. [link]('#') to any relevant files (or drag and drop into github) -# Misc. comments -... - - # Checklist - [ ] Code review by me - [ ] Hand tested by me - [ ] I'm proud of my work - [ ] Code review by reviewer - [ ] Hand tested by reviewer +- [ ] CircleCi tests are passing - [ ] Ready to merge From 217ad05423e604571c57208fbdeff78f9d841915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Wed, 5 Apr 2023 16:14:22 +0200 Subject: [PATCH 20/34] Update test_submissions.py --- src/tests/functional/test_submissions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/functional/test_submissions.py b/src/tests/functional/test_submissions.py index e6e094ba2..c33d33a4b 100644 --- a/src/tests/functional/test_submissions.py +++ b/src/tests/functional/test_submissions.py @@ -9,6 +9,8 @@ from utils.storage import md5 from ..utils import SeleniumTestCase +LONG_WAIT = 4 + class TestSubmissions(SeleniumTestCase): def setUp(self): @@ -36,7 +38,7 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio self.assert_current_url(comp_url) # This clicks the page before it loads fully, delay it a bit... - self.wait(1) + self.wait(LONG_WAIT) self.find('.item[data-tab="participate-tab"]').click() self.circleci_screenshot("set_submission_file_name.png") From 23b9ade026e4a9cac307bdb92c797fbe3ad7c116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Wed, 5 Apr 2023 16:23:35 +0200 Subject: [PATCH 21/34] Update test_submissions.py --- src/tests/functional/test_submissions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/functional/test_submissions.py b/src/tests/functional/test_submissions.py index c33d33a4b..a8451797c 100644 --- a/src/tests/functional/test_submissions.py +++ b/src/tests/functional/test_submissions.py @@ -10,6 +10,7 @@ from ..utils import SeleniumTestCase LONG_WAIT = 4 +SHORT_WAIT = 0.2 class TestSubmissions(SeleniumTestCase): @@ -50,6 +51,7 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio # Inside the accordion the output is being streamed self.find('.submission-output-container .title').click() + self.wait(SHORT_WAIT) assert self.find_text_in_class('.submission_output', expected_submission_output, timeout=timeout) # The submission table lists our submission! From 8f33e791e933cb3124559f69588390bddb4e56c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Wed, 5 Apr 2023 17:17:06 +0200 Subject: [PATCH 22/34] Add condition --- docker-compose.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b421f6341..19e38809a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,11 +69,15 @@ services: entrypoint: > /bin/sh -c " set -x - while ! nc -z minio 9000; echo 'Waiting for minio to startup...' && sleep 0.1; sleep 3; - until (/usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 3; done; - /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME; - /usr/bin/mc mb minio_docker/$AWS_STORAGE_PRIVATE_BUCKET_NAME; - /usr/bin/mc anonymous set download minio_docker/$AWS_STORAGE_BUCKET_NAME; + while ! nc -z minio 9000; echo 'Waiting for minio to startup...' && sleep 5; + if [ -n \"$MINIO_ACCESS_KEY\" ] && [ -n \"$MINIO_SECRET_KEY\" ] && [ -n \"$MINIO_PORT\" ]; then + until /usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do echo '...waiting...' && sleep 5; done; + /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME; + /usr/bin/mc mb minio_docker/$AWS_STORAGE_PRIVATE_BUCKET_NAME; + /usr/bin/mc anonymous set download minio_docker/$AWS_STORAGE_BUCKET_NAME; + else + echo 'MINIO_ACCESS_KEY, MINIO_SECRET_KEY, or MINIO_PORT are not defined. Skipping buckets creation.'; + fi exit 0; " From c499a8c5fbb6e46cf64c2e0ae89b662006198a11 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 7 Apr 2023 20:21:45 +0500 Subject: [PATCH 23/34] detailed results iframe width increased to full page width --- .../riot/competitions/detail/_detailed_results.tag | 11 ++++++++--- src/templates/competitions/detailed_results.html | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/static/riot/competitions/detail/_detailed_results.tag b/src/static/riot/competitions/detail/_detailed_results.tag index 946657dc3..ae5617fdb 100644 --- a/src/static/riot/competitions/detail/_detailed_results.tag +++ b/src/static/riot/competitions/detail/_detailed_results.tag @@ -9,8 +9,13 @@ .done(function (data) { self.detailed_result = data.detailed_result self.update() - }) - - + }) + + + + \ No newline at end of file diff --git a/src/templates/competitions/detailed_results.html b/src/templates/competitions/detailed_results.html index e7c14cfa6..17d8b9be5 100644 --- a/src/templates/competitions/detailed_results.html +++ b/src/templates/competitions/detailed_results.html @@ -1,4 +1,4 @@ {% extends "base.html" %} {% block content %} - + {% endblock %} \ No newline at end of file From f9ea7a4620f20f4fb0e5393752c1f73b3a194eaa Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 10 Apr 2023 22:34:38 -0400 Subject: [PATCH 24/34] if else fi not terminated with ; --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 19e38809a..d33e029b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,7 @@ services: /usr/bin/mc anonymous set download minio_docker/$AWS_STORAGE_BUCKET_NAME; else echo 'MINIO_ACCESS_KEY, MINIO_SECRET_KEY, or MINIO_PORT are not defined. Skipping buckets creation.'; - fi + fi; exit 0; " From b02a81e1803fd27edda67ee026ac462d1ead4b6d Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 10 Apr 2023 22:37:12 -0400 Subject: [PATCH 25/34] password_reset --- src/apps/profiles/urls_accounts.py | 6 +++- src/apps/profiles/views.py | 28 +++++++++++++++++++ src/templates/base.html | 4 +++ .../registration/password_reset_complete.html | 10 +++++++ .../registration/password_reset_confirm.html | 14 ++++++++++ .../registration/password_reset_done.html | 18 ++++++++++++ .../registration/password_reset_email.html | 2 ++ .../registration/password_reset_form.html | 24 ++++++++++++++++ .../registration/registration_base.html | 1 + 9 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 src/templates/registration/password_reset_complete.html create mode 100644 src/templates/registration/password_reset_confirm.html create mode 100644 src/templates/registration/password_reset_done.html create mode 100644 src/templates/registration/password_reset_email.html create mode 100644 src/templates/registration/password_reset_form.html create mode 100644 src/templates/registration/registration_base.html diff --git a/src/apps/profiles/urls_accounts.py b/src/apps/profiles/urls_accounts.py index 6266d5216..fbc290f46 100644 --- a/src/apps/profiles/urls_accounts.py +++ b/src/apps/profiles/urls_accounts.py @@ -1,6 +1,6 @@ from django.conf.urls import url from django.urls import path - +from django.contrib.auth import views as auth_views from . import views app_name = "accounts" @@ -12,4 +12,8 @@ path('login/', views.LoginView.as_view(), name='login'), # path('logout/', auth_views.LogoutView.as_view(), name='logout'), path('logout/', views.LogoutView.as_view(), name='logout'), + path('password_reset/', views.CustomPasswordResetView.as_view(), name='password_reset'), + path('password_reset/done/', views.CustomPasswordResetDoneView.as_view(), name='password_reset_done'), + path('reset///', views.CustomPasswordResetConfirmView.as_view(), name='password_reset_confirm'), + path('reset/done/', views.CustomPasswordResetCompleteView.as_view(), name='password_reset_complete'), ] diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 50e5cf66a..4cce77162 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -1,4 +1,5 @@ import json +import django from django.conf import settings from django.contrib import messages @@ -125,6 +126,33 @@ def sign_up(request): context['form'] = SignUpForm() return render(request, 'registration/signup.html', context) +# Password Reset views below +# https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView +# Search for PasswordResetView +class CustomPasswordResetView(auth_views.PasswordResetView): + # form_class = auth_forms.PasswordResetForm + # template_name = 'registration/password_reset_form.html' + # email_template_name = '' # Defaults to registration/password_reset_email.html if not supplied. + # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. + # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. + success_url = django.urls.reverse_lazy("accounts:password_reset_done") + from_email = "codabench@codabench.org" + +class CustomPasswordResetDoneView(auth_views.PasswordResetDoneView): + pass + # template_name = '' # Defaults to registration/password_reset_done.html if not supplied. + +class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): + success_url = django.urls.reverse_lazy("accounts:password_reset_complete") + # template_name = '' # Default value is registration/password_reset_confirm.html. + # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. + # post_reset_login = '' # Defaults to False. + # form_class = '' # Defaults to django.contrib.auth.forms.SetPasswordForm. + # success_url = '' # Defaults to 'password_reset_complete' + +class CustomPasswordResetCompleteView(auth_views.PasswordResetCompleteView): + pass +# Password Reset views above class UserNotificationEdit(LoginRequiredMixin, DetailView): queryset = User.objects.all() diff --git a/src/templates/base.html b/src/templates/base.html index 127c812c8..5e96e8e1b 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -145,6 +145,10 @@ Notifications + + + Change Password + Logout diff --git a/src/templates/registration/password_reset_complete.html b/src/templates/registration/password_reset_complete.html new file mode 100644 index 000000000..e69a6e197 --- /dev/null +++ b/src/templates/registration/password_reset_complete.html @@ -0,0 +1,10 @@ +{% extends 'base.html' %} + +{% block content %} +
+

+ Password Reset Complete +

+

Your password has been successfully reset.

+
+{% endblock %} diff --git a/src/templates/registration/password_reset_confirm.html b/src/templates/registration/password_reset_confirm.html new file mode 100644 index 000000000..873f819bb --- /dev/null +++ b/src/templates/registration/password_reset_confirm.html @@ -0,0 +1,14 @@ +{% extends 'base.html' %} + +{% block content %} +
+

+ Change Password +

+
+ {% csrf_token %} + {{ form.as_p }} + +
+
+{% endblock %} diff --git a/src/templates/registration/password_reset_done.html b/src/templates/registration/password_reset_done.html new file mode 100644 index 000000000..c36f7c225 --- /dev/null +++ b/src/templates/registration/password_reset_done.html @@ -0,0 +1,18 @@ +{% extends "registration/registration_base.html" %} + +{% block title %}Password reset{% endblock %} + +{% block content %} +
+

+ Password Reset Complete +

+

+ We have sent you an email with a link to reset your password. Please check + your email and click the link to continue. +

+
+{% endblock %} + + +{# This is used by django.contrib.auth #} \ No newline at end of file diff --git a/src/templates/registration/password_reset_email.html b/src/templates/registration/password_reset_email.html new file mode 100644 index 000000000..249023bb2 --- /dev/null +++ b/src/templates/registration/password_reset_email.html @@ -0,0 +1,2 @@ +Someone asked for password reset for email {{ email }}. Follow the link below: +{{ protocol}}://{{ domain }}{% url 'accounts:password_reset_confirm' uidb64=uid token=token %} diff --git a/src/templates/registration/password_reset_form.html b/src/templates/registration/password_reset_form.html new file mode 100644 index 000000000..5d225765d --- /dev/null +++ b/src/templates/registration/password_reset_form.html @@ -0,0 +1,24 @@ +{% extends "registration/registration_base.html" %} + + +{% block title %}Reset password{% endblock %} + +{% block content %} +
+

+ Reset password +

+ {% if user.is_authenticated %} +

Note: you are already logged in as {{ user.username }}.

+ {% endif %} +

Forgot your password? Enter your email in the form below and we'll send you instructions for creating a new one.

+
+ {% csrf_token %} + {{ form.as_p }} + +
+
+{% endblock %} + + +{# This is used by django.contrib.auth #} \ No newline at end of file diff --git a/src/templates/registration/registration_base.html b/src/templates/registration/registration_base.html new file mode 100644 index 000000000..63913c188 --- /dev/null +++ b/src/templates/registration/registration_base.html @@ -0,0 +1 @@ +{% extends "base.html" %} \ No newline at end of file From d8b1c0dabcffda7c7791b9e7a9925454e06ee4cb Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 10 Apr 2023 22:39:40 -0400 Subject: [PATCH 26/34] forgot to add the forgot password link --- src/templates/registration/login.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/templates/registration/login.html b/src/templates/registration/login.html index 4ffd5fcb3..f4b802cb2 100644 --- a/src/templates/registration/login.html +++ b/src/templates/registration/login.html @@ -53,6 +53,7 @@

New to us? Sign Up

+

Forgot your password?

From 80a4feb37d98666a0336b504b30babc6ea9c7f20 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 10 Apr 2023 22:57:00 -0400 Subject: [PATCH 27/34] flask error --- src/apps/profiles/urls_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/profiles/urls_accounts.py b/src/apps/profiles/urls_accounts.py index fbc290f46..e7f94ffb6 100644 --- a/src/apps/profiles/urls_accounts.py +++ b/src/apps/profiles/urls_accounts.py @@ -1,7 +1,7 @@ from django.conf.urls import url from django.urls import path -from django.contrib.auth import views as auth_views from . import views +# from django.contrib.auth import views as auth_views app_name = "accounts" From c8de3ae828566a1e5b05ea8abaf60881bd409109 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 10 Apr 2023 23:01:06 -0400 Subject: [PATCH 28/34] more flake errors --- src/apps/profiles/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 4cce77162..76782ba29 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -126,6 +126,7 @@ def sign_up(request): context['form'] = SignUpForm() return render(request, 'registration/signup.html', context) + # Password Reset views below # https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView # Search for PasswordResetView @@ -138,10 +139,12 @@ class CustomPasswordResetView(auth_views.PasswordResetView): success_url = django.urls.reverse_lazy("accounts:password_reset_done") from_email = "codabench@codabench.org" + class CustomPasswordResetDoneView(auth_views.PasswordResetDoneView): pass # template_name = '' # Defaults to registration/password_reset_done.html if not supplied. + class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): success_url = django.urls.reverse_lazy("accounts:password_reset_complete") # template_name = '' # Default value is registration/password_reset_confirm.html. @@ -150,10 +153,12 @@ class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): # form_class = '' # Defaults to django.contrib.auth.forms.SetPasswordForm. # success_url = '' # Defaults to 'password_reset_complete' + class CustomPasswordResetCompleteView(auth_views.PasswordResetCompleteView): pass # Password Reset views above + class UserNotificationEdit(LoginRequiredMixin, DetailView): queryset = User.objects.all() template_name = 'profiles/user_notifications.html' From 953083b1384be265498a1724740b2e1625609ea3 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 11 Apr 2023 11:32:08 -0400 Subject: [PATCH 29/34] correct email --- src/apps/profiles/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 76782ba29..16f6a6243 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -137,7 +137,7 @@ class CustomPasswordResetView(auth_views.PasswordResetView): # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. success_url = django.urls.reverse_lazy("accounts:password_reset_done") - from_email = "codabench@codabench.org" + from_email = "info@codalab.org" class CustomPasswordResetDoneView(auth_views.PasswordResetDoneView): From 1732a3cb9a97e2332b43dea6d017cd5cda902d8e Mon Sep 17 00:00:00 2001 From: didayolo Date: Thu, 13 Apr 2023 16:33:25 +0200 Subject: [PATCH 30/34] Change default docker image to py37 --- compute_worker/compute_worker.py | 2 +- src/apps/competitions/models.py | 2 +- src/apps/competitions/tests/unpacker_test_data.py | 2 +- src/apps/competitions/unpackers/v1.py | 2 +- src/apps/competitions/unpackers/v2.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 629d003b0..fb0d66b30 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -550,7 +550,7 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): # Input from submission (or submission + ingestion combo) engine_cmd += ['-v', f'{self._get_host_path(self.input_dir)}:/app/input'] - # Set the image name (i.e. "codalab/codalab-legacy") for the container + # Set the image name (i.e. "codalab/codalab-legacy:py37") for the container engine_cmd += [self.container_image] # Handle Legacy competitions by replacing anything in the run command diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index dccb4e77f..9c5be88ef 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -42,7 +42,7 @@ class Competition(ChaHubSaveMixin, models.Model): terms = models.TextField(null=True, blank=True) is_migrating = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) - docker_image = models.CharField(max_length=128, default="codalab/codalab-legacy:py3") + docker_image = models.CharField(max_length=128, default="codalab/codalab-legacy:py37") enable_detailed_results = models.BooleanField(default=False) queue = models.ForeignKey('queues.Queue', on_delete=models.SET_NULL, null=True, blank=True, diff --git a/src/apps/competitions/tests/unpacker_test_data.py b/src/apps/competitions/tests/unpacker_test_data.py index a061b86fc..5172e261b 100644 --- a/src/apps/competitions/tests/unpacker_test_data.py +++ b/src/apps/competitions/tests/unpacker_test_data.py @@ -6,7 +6,7 @@ "title": "Sample time series competition", "description": "Sample competition for time series prediction", "image": "logo.jpg", - "competition_docker_image": "codalab/codalab-legacy:py3", + "competition_docker_image": "codalab/codalab-legacy:py37", "end_date": None, "html": { "data": "data.txt", diff --git a/src/apps/competitions/unpackers/v1.py b/src/apps/competitions/unpackers/v1.py index 7ac01885b..897f6be02 100644 --- a/src/apps/competitions/unpackers/v1.py +++ b/src/apps/competitions/unpackers/v1.py @@ -11,7 +11,7 @@ def __init__(self, *args, **kwargs): # Just in case docker image is blank (""), replace with default value docker_image = self.competition_yaml.get('competition_docker_image') if not docker_image: - docker_image = "codalab/codalab-legacy:py3" + docker_image = "codalab/codalab-legacy:py37" self.competition = { "title": self.competition_yaml.get('title'), diff --git a/src/apps/competitions/unpackers/v2.py b/src/apps/competitions/unpackers/v2.py index 237441e69..f6bd941fb 100644 --- a/src/apps/competitions/unpackers/v2.py +++ b/src/apps/competitions/unpackers/v2.py @@ -13,7 +13,7 @@ def __init__(self, *args, **kwargs): "title": self.competition_yaml.get('title'), "logo": None, "registration_auto_approve": self.competition_yaml.get('registration_auto_approve', False), - "docker_image": self.competition_yaml.get('docker_image', 'codalab/codalab-legacy:py3'), + "docker_image": self.competition_yaml.get('docker_image', 'codalab/codalab-legacy:py37'), "enable_detailed_results": self.competition_yaml.get('enable_detailed_results', False), "description": self.competition_yaml.get("description", ""), "competition_type": self.competition_yaml.get("competition_type", "competition"), From 6e3d8e898d14c93c497144315c67311ae422dcec Mon Sep 17 00:00:00 2001 From: Benjamin Date: Thu, 13 Apr 2023 10:33:29 -0400 Subject: [PATCH 31/34] saving my place --- src/apps/profiles/views.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 16f6a6243..c91f80138 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -9,6 +9,7 @@ from django.http import Http404 from django.shortcuts import render, redirect from django.contrib.auth import views as auth_views +# from django.contrib.auth import forms as auth_forms ## from django.contrib.auth.mixins import LoginRequiredMixin from django.template.loader import render_to_string from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode @@ -130,10 +131,29 @@ def sign_up(request): # Password Reset views below # https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView # Search for PasswordResetView + +# class PasswordResetForm(forms.Form): + + +# auth_forms class CustomPasswordResetView(auth_views.PasswordResetView): + # def get_context_data(self, *args, **kwargs): + # context = super(LoginView, self).get_context_data(*args, **kwargs) + # # "http://localhost:8888/profiles/signup?next=http://localhost/social/login/chahub" + # context['chahub_signup_url'] = "{}/profiles/signup?next={}/social/login/chahub".format(settings.SOCIAL_AUTH_CHAHUB_BASE_URL, settings.SITE_DOMAIN) + # return context + # import pdb; pdb.set_trace() + # def send_mail(self, subject_template_name, email_template_name, + # context, from_email, to_email, html_email_template_name=None): + # # Render the email message + # email_message = render_to_string(email_template_name, context) + + # # Print the email message to the console + # print(email_message) # form_class = auth_forms.PasswordResetForm # template_name = 'registration/password_reset_form.html' # email_template_name = '' # Defaults to registration/password_reset_email.html if not supplied. + print("CustomPasswordResetView") # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. success_url = django.urls.reverse_lazy("accounts:password_reset_done") From 67e43958237ba380ba381b9ee157fc4f8bb66fc0 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 18 Apr 2023 11:49:34 -0400 Subject: [PATCH 32/34] print to django logs --- src/apps/profiles/urls_accounts.py | 9 ++- src/apps/profiles/views.py | 92 +++++++++++++++++++----------- 2 files changed, 66 insertions(+), 35 deletions(-) diff --git a/src/apps/profiles/urls_accounts.py b/src/apps/profiles/urls_accounts.py index e7f94ffb6..43d4411c3 100644 --- a/src/apps/profiles/urls_accounts.py +++ b/src/apps/profiles/urls_accounts.py @@ -1,7 +1,7 @@ from django.conf.urls import url from django.urls import path from . import views -# from django.contrib.auth import views as auth_views +from django.contrib.auth import views as auth_views app_name = "accounts" @@ -12,8 +12,11 @@ path('login/', views.LoginView.as_view(), name='login'), # path('logout/', auth_views.LogoutView.as_view(), name='logout'), path('logout/', views.LogoutView.as_view(), name='logout'), + path('password_reset/', views.CustomPasswordResetView.as_view(), name='password_reset'), - path('password_reset/done/', views.CustomPasswordResetDoneView.as_view(), name='password_reset_done'), + # path('password_reset/done/', views.CustomPasswordResetDoneView.as_view(), name='password_reset_done'), + path('password_reset/done/', auth_views.PasswordResetDoneView.as_view(), name='password_reset_done'), path('reset///', views.CustomPasswordResetConfirmView.as_view(), name='password_reset_confirm'), - path('reset/done/', views.CustomPasswordResetCompleteView.as_view(), name='password_reset_complete'), + # path('reset/done/', views.CustomPasswordResetCompleteView.as_view(), name='password_reset_complete'), + path('reset/done/', auth_views.PasswordResetCompleteView.as_view(), name='password_reset_complete'), ] diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index c91f80138..17d6e9565 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -5,11 +5,11 @@ from django.contrib import messages from django.contrib.auth import authenticate from django.contrib.sites.shortcuts import get_current_site -from django.core.mail import EmailMessage +from django.core.mail import EmailMessage, EmailMultiAlternatives from django.http import Http404 from django.shortcuts import render, redirect from django.contrib.auth import views as auth_views -# from django.contrib.auth import forms as auth_forms ## +from django.contrib.auth import forms as auth_forms from django.contrib.auth.mixins import LoginRequiredMixin from django.template.loader import render_to_string from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode @@ -128,55 +128,83 @@ def sign_up(request): return render(request, 'registration/signup.html', context) -# Password Reset views below -# https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView -# Search for PasswordResetView - -# class PasswordResetForm(forms.Form): - - +# Password Reset views/forms below # auth_forms +class CustomPasswordResetForm(auth_forms.PasswordResetForm): + """ + Subclassed auth_forms.PasswordResetForm in order to add a print statement + to see the email in the logs. + Source: https://github.com/django/django/blob/8b1ff0da4b162e87edebd94e61f2cd153e9e159d/django/contrib/auth/forms.py#L287 + """ + def send_mail( + self, + subject_template_name, + email_template_name, + context, + from_email, + to_email, + html_email_template_name=None, + ): + """ + Send a django.core.mail.EmailMultiAlternatives to `to_email`. + """ + subject = render_to_string(subject_template_name, context) + # Email subject *must not* contain newlines + subject = "".join(subject.splitlines()) + body = render_to_string(email_template_name, context) + + email_message = EmailMultiAlternatives(subject, body, from_email, [to_email]) + print(email_message.message()) + if html_email_template_name is not None: + html_email = render_to_string(html_email_template_name, context) + email_message.attach_alternative(html_email, "text/html") + + email_message.send() + +# auth_views +# https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView # Search for PasswordResetView class CustomPasswordResetView(auth_views.PasswordResetView): - # def get_context_data(self, *args, **kwargs): - # context = super(LoginView, self).get_context_data(*args, **kwargs) - # # "http://localhost:8888/profiles/signup?next=http://localhost/social/login/chahub" - # context['chahub_signup_url'] = "{}/profiles/signup?next={}/social/login/chahub".format(settings.SOCIAL_AUTH_CHAHUB_BASE_URL, settings.SITE_DOMAIN) - # return context - # import pdb; pdb.set_trace() - # def send_mail(self, subject_template_name, email_template_name, - # context, from_email, to_email, html_email_template_name=None): - # # Render the email message - # email_message = render_to_string(email_template_name, context) - - # # Print the email message to the console - # print(email_message) - # form_class = auth_forms.PasswordResetForm + """ + 1. form_class: subclassing auth_views.PasswordResetView to use a custom form "CustomPasswordResetForm" above + 2. success_url: Our src/apps/profiles/urls_accounts.py has become an "app" with the use of "app_name". + We have to use app:view_name syntax in templates like " {% url 'accounts:password_reset_confirm'%} " + Therefore we need to tell this view to find the right success_url with that syntax or django won't be + able to find the view. + 3. from_email: We want to set the from_email to info@codalab.org - may eventually put in .env file. + # The other commented sections are the defaults for other attributes in auth_views.PasswordResetView. + They are in here in case someone wants to customize in the future. All attributes show up in the order + shown in the docs. + """ # template_name = 'registration/password_reset_form.html' + form_class = CustomPasswordResetForm # auth_forms.PasswordResetForm # email_template_name = '' # Defaults to registration/password_reset_email.html if not supplied. - print("CustomPasswordResetView") # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. success_url = django.urls.reverse_lazy("accounts:password_reset_done") from_email = "info@codalab.org" -class CustomPasswordResetDoneView(auth_views.PasswordResetDoneView): - pass +# class CustomPasswordResetDoneView(auth_views.PasswordResetDoneView): +# pass # template_name = '' # Defaults to registration/password_reset_done.html if not supplied. class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): - success_url = django.urls.reverse_lazy("accounts:password_reset_complete") + """ + 1. success_url: Our src/apps/profiles/urls_accounts.py has become an "app" with the use of "app_name". + We have to use app:view_name syntax in templates like " {% url 'accounts:password_reset_confirm'%} " + Therefore we need to tell this view to find the right success_url with that syntax or django won't be + able to find the view. + """ # template_name = '' # Default value is registration/password_reset_confirm.html. + # form_class = '' # Defaults to django.contrib.auth.forms.SetPasswordForm. # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. # post_reset_login = '' # Defaults to False. - # form_class = '' # Defaults to django.contrib.auth.forms.SetPasswordForm. - # success_url = '' # Defaults to 'password_reset_complete' + success_url = django.urls.reverse_lazy("accounts:password_reset_complete") -class CustomPasswordResetCompleteView(auth_views.PasswordResetCompleteView): - pass -# Password Reset views above +# class CustomPasswordResetCompleteView(auth_views.PasswordResetCompleteView): +# pass class UserNotificationEdit(LoginRequiredMixin, DetailView): From 8abc387af8c53b82fdb4ada50d26a7d9755d4291 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 18 Apr 2023 12:15:01 -0400 Subject: [PATCH 33/34] final design change --- src/apps/profiles/urls_accounts.py | 3 --- src/apps/profiles/views.py | 9 --------- 2 files changed, 12 deletions(-) diff --git a/src/apps/profiles/urls_accounts.py b/src/apps/profiles/urls_accounts.py index 43d4411c3..86321d24e 100644 --- a/src/apps/profiles/urls_accounts.py +++ b/src/apps/profiles/urls_accounts.py @@ -12,11 +12,8 @@ path('login/', views.LoginView.as_view(), name='login'), # path('logout/', auth_views.LogoutView.as_view(), name='logout'), path('logout/', views.LogoutView.as_view(), name='logout'), - path('password_reset/', views.CustomPasswordResetView.as_view(), name='password_reset'), - # path('password_reset/done/', views.CustomPasswordResetDoneView.as_view(), name='password_reset_done'), path('password_reset/done/', auth_views.PasswordResetDoneView.as_view(), name='password_reset_done'), path('reset///', views.CustomPasswordResetConfirmView.as_view(), name='password_reset_confirm'), - # path('reset/done/', views.CustomPasswordResetCompleteView.as_view(), name='password_reset_complete'), path('reset/done/', auth_views.PasswordResetCompleteView.as_view(), name='password_reset_complete'), ] diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 17d6e9565..74f5884b3 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -184,11 +184,6 @@ class CustomPasswordResetView(auth_views.PasswordResetView): from_email = "info@codalab.org" -# class CustomPasswordResetDoneView(auth_views.PasswordResetDoneView): -# pass - # template_name = '' # Defaults to registration/password_reset_done.html if not supplied. - - class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): """ 1. success_url: Our src/apps/profiles/urls_accounts.py has become an "app" with the use of "app_name". @@ -203,10 +198,6 @@ class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): success_url = django.urls.reverse_lazy("accounts:password_reset_complete") -# class CustomPasswordResetCompleteView(auth_views.PasswordResetCompleteView): -# pass - - class UserNotificationEdit(LoginRequiredMixin, DetailView): queryset = User.objects.all() template_name = 'profiles/user_notifications.html' From 6dad0c42895a02d5ad0fbfda836a98bb6b8c38ec Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 18 Apr 2023 12:18:54 -0400 Subject: [PATCH 34/34] flake formatting errors --- src/apps/profiles/views.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 74f5884b3..fe02bdee9 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -160,7 +160,8 @@ def send_mail( email_message.attach_alternative(html_email, "text/html") email_message.send() - + + # auth_views # https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView # Search for PasswordResetView class CustomPasswordResetView(auth_views.PasswordResetView): @@ -172,14 +173,14 @@ class CustomPasswordResetView(auth_views.PasswordResetView): able to find the view. 3. from_email: We want to set the from_email to info@codalab.org - may eventually put in .env file. # The other commented sections are the defaults for other attributes in auth_views.PasswordResetView. - They are in here in case someone wants to customize in the future. All attributes show up in the order + They are in here in case someone wants to customize in the future. All attributes show up in the order shown in the docs. """ # template_name = 'registration/password_reset_form.html' - form_class = CustomPasswordResetForm # auth_forms.PasswordResetForm - # email_template_name = '' # Defaults to registration/password_reset_email.html if not supplied. - # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. - # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. + form_class = CustomPasswordResetForm # auth_forms.PasswordResetForm + # email_template_name = '' # Defaults to registration/password_reset_email.html if not supplied. + # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. + # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. success_url = django.urls.reverse_lazy("accounts:password_reset_done") from_email = "info@codalab.org"