diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 599f87459..87a6160da 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,9 +14,6 @@ more information is better. ... -# Known issues to be addressed in a separate PR -... - # A checklist for hand testing - [ ] add checklist here @@ -26,15 +23,12 @@ more information is better. [link]('#') to any relevant files (or drag and drop into github) -# Misc. comments -... - - # Checklist - [ ] Code review by me - [ ] Hand tested by me - [ ] I'm proud of my work - [ ] Code review by reviewer - [ ] Hand tested by reviewer +- [ ] CircleCi tests are passing - [ ] Ready to merge diff --git a/.gitignore b/.gitignore index ad8e29a2f..b5b34f56a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ docker-compose.override.yml server_config.yaml /graphs/ /codabench/ + +.DS_Store +.DS_Store? diff --git a/Containerfile.compute_worker_podman b/Containerfile.compute_worker_podman new file mode 100644 index 000000000..9c96ead37 --- /dev/null +++ b/Containerfile.compute_worker_podman @@ -0,0 +1,65 @@ +FROM fedora:37 + +# Include deps +RUN dnf -y update && \ + # https://bugzilla.redhat.com/show_bug.cgi?id=1995337#c3 + rpm --setcaps shadow-utils 2>/dev/null && \ + dnf -y install podman fuse-overlayfs python3.8 python3-pip \ + --exclude container-selinux && \ + dnf clean all && \ + rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Setup user +RUN useradd worker; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; + +# Copy over the podman container configuration +COPY podman/containers.conf /etc/containers/containers.conf +COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf + +# Copy over the podman storage configuration +COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf + +RUN mkdir -p /home/worker/.local/share/containers && \ + chown worker:worker -R /home/worker && \ + chmod 644 /etc/containers/containers.conf + +# Copy & modify the defaults to provide reference if runtime changes needed. +# Changes here are required for running with fuse-overlay storage inside container. +RUN sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf \ + > /etc/containers/storage.conf + +# Add volume for containers +VOLUME /home/worker/.local/share/containers + +# Create directory for tmp space +RUN mkdir /codabench && \ + chown worker:worker /codabench + +# Set up podman registry for dockerhub +RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf + +# This makes output not buffer and return immediately, nice for seeing results in stdout +ENV PYTHONUNBUFFERED 1 +ENV CONTAINER_ENGINE_EXECUTABLE podman + +# Get pip for 3.8 +RUN python3.8 -m ensurepip --upgrade + +WORKDIR /home/worker/compute_worker + +ADD compute_worker/ /home/worker/compute_worker + +RUN chown worker:worker -R /home/worker/compute_worker + +RUN pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt + +CMD celery -A compute_worker worker \ + -l info \ + -Q compute-worker \ + -n compute-worker@%n \ + --concurrency=1 diff --git a/Containerfile.compute_worker_podman_gpu b/Containerfile.compute_worker_podman_gpu new file mode 100644 index 000000000..0ef68e1a2 --- /dev/null +++ b/Containerfile.compute_worker_podman_gpu @@ -0,0 +1,66 @@ +FROM fedora:37 + +# Include deps +RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | tee /etc/yum.repos.d/cuda.repo && \ + curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | tee /etc/yum.repos.d/nvidia-docker.repo && \ + rpm -Uvh http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm && \ + rpm -Uvh http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm && \ + dnf -y update && \ + dnf module install -y nvidia-driver:latest-dkms && \ + dnf -y install podman fuse-overlayfs python3.8 python3-pip nvidia-container-runtime nvidia-container-toolkit \ + cuda --exclude container-selinux && \ + dnf clean all && \ + rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Setup user +RUN useradd worker; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \ +echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid; + +# Copy over the podman container configuration +COPY podman/containers.conf /etc/containers/containers.conf +COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf + +# Copy over the podman storage configuration +COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf + +RUN mkdir -p /home/worker/.local/share/containers && \ + chown worker:worker -R /home/worker && \ + chmod 644 /etc/containers/containers.conf + +# Copy & modify the defaults to provide reference if runtime changes needed. +# Changes here are required for running with fuse-overlay storage inside container. +RUN sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf \ + > /etc/containers/storage.conf; sed -i 's/^#no-cgroups = false/no-cgroups = true/;' /etc/nvidia-container-runtime/config.toml + + +# Add volume for containers +VOLUME /home/worker/.local/share/containers + +# This makes output not buffer and return immediately, nice for seeing results in stdout +ENV PYTHONUNBUFFERED 1 +ENV CONTAINER_ENGINE_EXECUTABLE podman + +# Create directory for tmp space +RUN mkdir /codabench && \ + chown worker:worker /codabench && \ +# Set up podman registry for dockerhub + echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \ +# Get pip for 3.8 + python3.8 -m ensurepip --upgrade + +WORKDIR /home/worker/compute_worker + +ADD compute_worker/ /home/worker/compute_worker + +RUN chown worker:worker -R /home/worker/compute_worker && \ + pip3.8 install -r /home/worker/compute_worker/compute_worker_requirements.txt + +CMD nvidia-smi && celery -A compute_worker worker \ + -l info \ + -Q compute-worker \ + -n compute-worker@%n \ + --concurrency=1 diff --git a/Dockerfile.compute_worker b/Dockerfile.compute_worker index 77e0ef69d..482924931 100644 --- a/Dockerfile.compute_worker +++ b/Dockerfile.compute_worker @@ -6,10 +6,10 @@ ENV PYTHONUNBUFFERED 1 # Install Docker RUN apt-get update && curl -fsSL https://get.docker.com | sh -ADD docker/compute_worker/compute_worker_requirements.txt . +ADD compute_worker/compute_worker_requirements.txt . RUN pip install -r compute_worker_requirements.txt -ADD docker/compute_worker . +ADD compute_worker . CMD celery -A compute_worker worker \ -l info \ diff --git a/Dockerfile.compute_worker_gpu b/Dockerfile.compute_worker_gpu index f2110647c..e559a0667 100644 --- a/Dockerfile.compute_worker_gpu +++ b/Dockerfile.compute_worker_gpu @@ -19,9 +19,9 @@ RUN apt-get update && apt-get install -y nvidia-docker2 ENV NVIDIA_DOCKER 1 # Python reqs and actual worker stuff -ADD docker/compute_worker/compute_worker_requirements.txt . +ADD compute_worker/compute_worker_requirements.txt . RUN pip3 install -r compute_worker_requirements.txt -ADD docker/compute_worker . +ADD compute_worker . CMD celery -A compute_worker worker \ -l info \ diff --git a/docker/compute_worker/celery_config.py b/compute_worker/celery_config.py similarity index 100% rename from docker/compute_worker/celery_config.py rename to compute_worker/celery_config.py diff --git a/docker/compute_worker/compute_worker.py b/compute_worker/compute_worker.py similarity index 92% rename from docker/compute_worker/compute_worker.py rename to compute_worker/compute_worker.py index 110010ad3..fb0d66b30 100644 --- a/docker/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -65,6 +65,14 @@ STATUS_FAILED, ) +# Setup the container engine that we are using +if os.environ.get("CONTAINER_ENGINE_EXECUTABLE"): + CONTAINER_ENGINE_EXECUTABLE = os.environ.get("CONTAINER_ENGINE_EXECUTABLE") +# We could probably depreciate this now that we can specify the executable +elif os.environ.get("NVIDIA_DOCKER"): + CONTAINER_ENGINE_EXECUTABLE = "nvidia-docker" +else: + CONTAINER_ENGINE_EXECUTABLE = "docker" class SubmissionException(Exception): pass @@ -181,7 +189,7 @@ def __init__(self, run_args): self.user_pk = run_args["user_pk"] self.submission_id = run_args["id"] self.submissions_api_url = run_args["submissions_api_url"] - self.docker_image = run_args["docker_image"] + self.container_image = run_args["docker_image"] self.secret = run_args["secret"] self.prediction_result = run_args["prediction_result"] self.scoring_result = run_args.get("scoring_result") @@ -221,7 +229,7 @@ def __init__(self, run_args): self.requests_session.mount('https://', adapter) async def watch_detailed_results(self): - """Watches files alongside scoring + program docker containers, currently only used + """Watches files alongside scoring + program containers, currently only used for detailed_results.html""" if not self.detailed_results_url: return @@ -314,15 +322,15 @@ def _update_status(self, status, extra_information=None): # }) self._update_submission(data) - def _get_docker_image(self, image_name): - logger.info("Running docker pull for image: {}".format(image_name)) + def _get_container_image(self, image_name): + logger.info("Running pull for image: {}".format(image_name)) try: - cmd = ['docker', 'pull', image_name] - docker_pull = check_output(cmd) - logger.info("Docker pull complete for image: {0} with output of {1}".format(image_name, docker_pull)) + cmd = [CONTAINER_ENGINE_EXECUTABLE, 'pull', image_name] + container_engine_pull = check_output(cmd) + logger.info("Pull complete for image: {0} with output of {1}".format(image_name, container_engine_pull)) except CalledProcessError: - logger.info("Docker pull for image: {} returned a non-zero exit code!") - raise SubmissionException(f"Docker pull for {image_name} failed!") + logger.info("Pull for image: {} returned a non-zero exit code!") + raise SubmissionException(f"Pull for {image_name} failed!") def _get_bundle(self, url, destination, cache=True): """Downloads zip from url and unzips into destination. If cache=True then url is hashed and checked @@ -357,17 +365,17 @@ def _get_bundle(self, url, destination, cache=True): # Give back zip file path for other uses, i.e. md5'ing the zip to ID it return bundle_file - async def _run_docker_cmd(self, docker_cmd, kind): + async def _run_container_engine_cmd(self, engine_cmd, kind): """This runs a command and asynchronously writes the data to both a storage file and a socket - :param docker_cmd: the list of docker command arguments + :param engine_cmd: the list of container engine command arguments :param kind: either 'ingestion' or 'program' :return: """ start = time.time() proc = await asyncio.create_subprocess_exec( - *docker_cmd, + *engine_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) @@ -442,17 +450,23 @@ async def _run_docker_cmd(self, docker_cmd, kind): await websocket.close() def _get_host_path(self, *paths): - """Turns an absolute path inside our docker container, into what the path - would be on the host machine""" + """Turns an absolute path inside our container, into what the path + would be on the host machine. We also ensure that the directory exists, + docker will create if necessary, but other container engines such as + podman may not.""" # Take our list of paths and smash 'em together path = os.path.join(*paths) - # pull front of path, which points to the location inside docker + # pull front of path, which points to the location inside the container path = path[len(BASE_DIR):] - # add host to front, so when we run commands in docker on the host they + # add host to front, so when we run commands in the container on the host they # can be seen properly path = os.path.join(HOST_DIRECTORY, path) + + # Create if necessary + os.makedirs(path, exist_ok=True) + return path async def _run_program_directory(self, program_dir, kind, can_be_output=False): @@ -494,13 +508,8 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): ) return - if os.environ.get("NVIDIA_DOCKER"): - docker_process_name = "nvidia-docker" - else: - docker_process_name = "docker" - - docker_cmd = [ - docker_process_name, + engine_cmd = [ + CONTAINER_ENGINE_EXECUTABLE, 'run', # Remove it after run '--rm', @@ -528,21 +537,21 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): else: ingested_program_location = "program" - docker_cmd += ['-v', f'{self._get_host_path(self.root_dir, ingested_program_location)}:/app/ingested_program'] + engine_cmd += ['-v', f'{self._get_host_path(self.root_dir, ingested_program_location)}:/app/ingested_program'] if self.input_data: - docker_cmd += ['-v', f'{self._get_host_path(self.root_dir, "input_data")}:/app/input_data'] + engine_cmd += ['-v', f'{self._get_host_path(self.root_dir, "input_data")}:/app/input_data'] if self.is_scoring: # For scoring programs, we want to have a shared directory just in case we have an ingestion program. # This will add the share dir regardless of ingestion or scoring, as long as we're `is_scoring` - docker_cmd += ['-v', f'{self._get_host_path(self.root_dir, "shared")}:/app/shared'] + engine_cmd += ['-v', f'{self._get_host_path(self.root_dir, "shared")}:/app/shared'] # Input from submission (or submission + ingestion combo) - docker_cmd += ['-v', f'{self._get_host_path(self.input_dir)}:/app/input'] + engine_cmd += ['-v', f'{self._get_host_path(self.input_dir)}:/app/input'] - # Set the image name (i.e. "codalab/codalab-legacy") for the container - docker_cmd += [self.docker_image] + # Set the image name (i.e. "codalab/codalab-legacy:py37") for the container + engine_cmd += [self.container_image] # Handle Legacy competitions by replacing anything in the run command command = replace_legacy_metadata_command( @@ -553,12 +562,12 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False): ) # Append the actual program to run - docker_cmd += command.split(' ') + engine_cmd += command.split(' ') - logger.info(f"Running program = {' '.join(docker_cmd)}") + logger.info(f"Running program = {' '.join(engine_cmd)}") - # This runs the docker command and asynchronously passes data back via websocket - return await self._run_docker_cmd(docker_cmd, kind=kind) + # This runs the container engine command and asynchronously passes data back via websocket + return await self._run_container_engine_cmd(engine_cmd, kind=kind) def _put_dir(self, url, directory): logger.info("Putting dir %s in %s" % (directory, url)) @@ -649,9 +658,9 @@ def prepare(self): for filename in glob.iglob(self.root_dir + '**/*.*', recursive=True): logger.info(filename) - # Before the run starts we want to download docker images, they may take a while to download + # Before the run starts we want to download images, they may take a while to download # and to do this during the run would subtract from the participants time. - self._get_docker_image(self.docker_image) + self._get_container_image(self.container_image) def start(self): if not self.is_scoring: @@ -690,7 +699,7 @@ def start(self): else: program_to_kill = self.program_container_name # Try and stop the program. If stop does not succeed - kill_code = subprocess.call(['docker', 'stop', str(program_to_kill)]) + kill_code = subprocess.call([CONTAINER_ENGINE_EXECUTABLE, 'stop', str(program_to_kill)]) logger.info(f'Kill process returned {kill_code}') if kind == 'program': self.program_exit_code = return_code diff --git a/docker/compute_worker/compute_worker_requirements.txt b/compute_worker/compute_worker_requirements.txt similarity index 100% rename from docker/compute_worker/compute_worker_requirements.txt rename to compute_worker/compute_worker_requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index fbe4e502e..d33e029b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,11 +69,15 @@ services: entrypoint: > /bin/sh -c " set -x - while ! nc -z minio 9000; echo 'Waiting for minio to startup...' && sleep 0.1; sleep 3; - until (/usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 3; done; - /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME; - /usr/bin/mc mb minio_docker/$AWS_STORAGE_PRIVATE_BUCKET_NAME; - /usr/bin/mc anonymous set download minio_docker/$AWS_STORAGE_BUCKET_NAME; + while ! nc -z minio 9000; echo 'Waiting for minio to startup...' && sleep 5; + if [ -n \"$MINIO_ACCESS_KEY\" ] && [ -n \"$MINIO_SECRET_KEY\" ] && [ -n \"$MINIO_PORT\" ]; then + until /usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do echo '...waiting...' && sleep 5; done; + /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME; + /usr/bin/mc mb minio_docker/$AWS_STORAGE_PRIVATE_BUCKET_NAME; + /usr/bin/mc anonymous set download minio_docker/$AWS_STORAGE_BUCKET_NAME; + else + echo 'MINIO_ACCESS_KEY, MINIO_SECRET_KEY, or MINIO_PORT are not defined. Skipping buckets creation.'; + fi; exit 0; " @@ -201,7 +205,7 @@ services: - django - rabbit volumes: - - ./docker/compute_worker:/app + - ./compute_worker:/app - ${HOST_DIRECTORY:-/tmp/codabench}:/codabench # Actual connection back to docker parent to run things - /var/run/docker.sock:/var/run/docker.sock diff --git a/podman/containers.conf b/podman/containers.conf new file mode 100644 index 000000000..220c1f850 --- /dev/null +++ b/podman/containers.conf @@ -0,0 +1,12 @@ +[containers] +netns="host" +userns="host" +ipcns="host" +utsns="host" +cgroupns="host" +cgroups="disabled" +log_driver = "k8s-file" +[engine] +cgroup_manager = "cgroupfs" +events_logger="file" +runtime="crun" diff --git a/podman/worker-containers.conf b/podman/worker-containers.conf new file mode 100644 index 000000000..2bdd95a3b --- /dev/null +++ b/podman/worker-containers.conf @@ -0,0 +1,5 @@ +[containers] +volumes = [ + "/proc:/proc", +] +default_sysctls = [] diff --git a/podman/worker-storage.conf b/podman/worker-storage.conf new file mode 100644 index 000000000..0b1afb08b --- /dev/null +++ b/podman/worker-storage.conf @@ -0,0 +1,5 @@ +[storage] +driver = "overlay" + +[storage.options.overlay] +mount_program = "/usr/bin/fuse-overlayfs" diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index a8cbf7546..fde3aedf9 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -474,6 +474,7 @@ def get_leaderboard(self, request, pk): if submission_key not in submissions_keys: submissions_keys[submission_key] = len(response['submissions']) response['submissions'].append({ + 'id': submission['id'], 'owner': submission['display_name'] or submission['owner'], 'scores': [], 'fact_sheet_answers': submission['fact_sheet_answers'], diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index dccb4e77f..9c5be88ef 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -42,7 +42,7 @@ class Competition(ChaHubSaveMixin, models.Model): terms = models.TextField(null=True, blank=True) is_migrating = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) - docker_image = models.CharField(max_length=128, default="codalab/codalab-legacy:py3") + docker_image = models.CharField(max_length=128, default="codalab/codalab-legacy:py37") enable_detailed_results = models.BooleanField(default=False) queue = models.ForeignKey('queues.Queue', on_delete=models.SET_NULL, null=True, blank=True, diff --git a/src/apps/competitions/tests/test_legacy_command_replacement.py b/src/apps/competitions/tests/test_legacy_command_replacement.py index 52b031353..18a1fbfeb 100644 --- a/src/apps/competitions/tests/test_legacy_command_replacement.py +++ b/src/apps/competitions/tests/test_legacy_command_replacement.py @@ -1,5 +1,5 @@ from django.test import TestCase -from docker.compute_worker.compute_worker import replace_legacy_metadata_command +from compute_worker.compute_worker import replace_legacy_metadata_command class LegacyConverterCommandTests(TestCase): diff --git a/src/apps/competitions/tests/unpacker_test_data.py b/src/apps/competitions/tests/unpacker_test_data.py index a061b86fc..5172e261b 100644 --- a/src/apps/competitions/tests/unpacker_test_data.py +++ b/src/apps/competitions/tests/unpacker_test_data.py @@ -6,7 +6,7 @@ "title": "Sample time series competition", "description": "Sample competition for time series prediction", "image": "logo.jpg", - "competition_docker_image": "codalab/codalab-legacy:py3", + "competition_docker_image": "codalab/codalab-legacy:py37", "end_date": None, "html": { "data": "data.txt", diff --git a/src/apps/competitions/unpackers/v1.py b/src/apps/competitions/unpackers/v1.py index 7ac01885b..897f6be02 100644 --- a/src/apps/competitions/unpackers/v1.py +++ b/src/apps/competitions/unpackers/v1.py @@ -11,7 +11,7 @@ def __init__(self, *args, **kwargs): # Just in case docker image is blank (""), replace with default value docker_image = self.competition_yaml.get('competition_docker_image') if not docker_image: - docker_image = "codalab/codalab-legacy:py3" + docker_image = "codalab/codalab-legacy:py37" self.competition = { "title": self.competition_yaml.get('title'), diff --git a/src/apps/competitions/unpackers/v2.py b/src/apps/competitions/unpackers/v2.py index 237441e69..f6bd941fb 100644 --- a/src/apps/competitions/unpackers/v2.py +++ b/src/apps/competitions/unpackers/v2.py @@ -13,7 +13,7 @@ def __init__(self, *args, **kwargs): "title": self.competition_yaml.get('title'), "logo": None, "registration_auto_approve": self.competition_yaml.get('registration_auto_approve', False), - "docker_image": self.competition_yaml.get('docker_image', 'codalab/codalab-legacy:py3'), + "docker_image": self.competition_yaml.get('docker_image', 'codalab/codalab-legacy:py37'), "enable_detailed_results": self.competition_yaml.get('enable_detailed_results', False), "description": self.competition_yaml.get("description", ""), "competition_type": self.competition_yaml.get("competition_type", "competition"), diff --git a/src/apps/competitions/urls.py b/src/apps/competitions/urls.py index 6f9e29ba0..a3d4419a9 100644 --- a/src/apps/competitions/urls.py +++ b/src/apps/competitions/urls.py @@ -12,4 +12,5 @@ path('edit//', views.CompetitionForm.as_view(), name="edit"), path('upload/', views.CompetitionUpload.as_view(), name="upload"), path('public/', views.CompetitionPublic.as_view(), name="public"), + path('/detailed_results//', views.CompetitionDetailedResults.as_view(), name="detailed_results"), ] diff --git a/src/apps/competitions/views.py b/src/apps/competitions/views.py index 09e30e822..7a4045f7f 100644 --- a/src/apps/competitions/views.py +++ b/src/apps/competitions/views.py @@ -33,3 +33,7 @@ def get_object(self, *args, **kwargs): if is_creator or is_collaborator or competition.published or valid_secret_key: return competition raise Http404() + + +class CompetitionDetailedResults(LoginRequiredMixin, TemplateView): + template_name = 'competitions/detailed_results.html' diff --git a/src/apps/profiles/urls_accounts.py b/src/apps/profiles/urls_accounts.py index 6266d5216..86321d24e 100644 --- a/src/apps/profiles/urls_accounts.py +++ b/src/apps/profiles/urls_accounts.py @@ -1,7 +1,7 @@ from django.conf.urls import url from django.urls import path - from . import views +from django.contrib.auth import views as auth_views app_name = "accounts" @@ -12,4 +12,8 @@ path('login/', views.LoginView.as_view(), name='login'), # path('logout/', auth_views.LogoutView.as_view(), name='logout'), path('logout/', views.LogoutView.as_view(), name='logout'), + path('password_reset/', views.CustomPasswordResetView.as_view(), name='password_reset'), + path('password_reset/done/', auth_views.PasswordResetDoneView.as_view(), name='password_reset_done'), + path('reset///', views.CustomPasswordResetConfirmView.as_view(), name='password_reset_confirm'), + path('reset/done/', auth_views.PasswordResetCompleteView.as_view(), name='password_reset_complete'), ] diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 50e5cf66a..fe02bdee9 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -1,13 +1,15 @@ import json +import django from django.conf import settings from django.contrib import messages from django.contrib.auth import authenticate from django.contrib.sites.shortcuts import get_current_site -from django.core.mail import EmailMessage +from django.core.mail import EmailMessage, EmailMultiAlternatives from django.http import Http404 from django.shortcuts import render, redirect from django.contrib.auth import views as auth_views +from django.contrib.auth import forms as auth_forms from django.contrib.auth.mixins import LoginRequiredMixin from django.template.loader import render_to_string from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode @@ -126,6 +128,77 @@ def sign_up(request): return render(request, 'registration/signup.html', context) +# Password Reset views/forms below +# auth_forms +class CustomPasswordResetForm(auth_forms.PasswordResetForm): + """ + Subclassed auth_forms.PasswordResetForm in order to add a print statement + to see the email in the logs. + Source: https://github.com/django/django/blob/8b1ff0da4b162e87edebd94e61f2cd153e9e159d/django/contrib/auth/forms.py#L287 + """ + def send_mail( + self, + subject_template_name, + email_template_name, + context, + from_email, + to_email, + html_email_template_name=None, + ): + """ + Send a django.core.mail.EmailMultiAlternatives to `to_email`. + """ + subject = render_to_string(subject_template_name, context) + # Email subject *must not* contain newlines + subject = "".join(subject.splitlines()) + body = render_to_string(email_template_name, context) + + email_message = EmailMultiAlternatives(subject, body, from_email, [to_email]) + print(email_message.message()) + if html_email_template_name is not None: + html_email = render_to_string(html_email_template_name, context) + email_message.attach_alternative(html_email, "text/html") + + email_message.send() + + +# auth_views +# https://devdocs.io/django~2.2/topics/auth/default#django.contrib.auth.views.PasswordChangeView # Search for PasswordResetView +class CustomPasswordResetView(auth_views.PasswordResetView): + """ + 1. form_class: subclassing auth_views.PasswordResetView to use a custom form "CustomPasswordResetForm" above + 2. success_url: Our src/apps/profiles/urls_accounts.py has become an "app" with the use of "app_name". + We have to use app:view_name syntax in templates like " {% url 'accounts:password_reset_confirm'%} " + Therefore we need to tell this view to find the right success_url with that syntax or django won't be + able to find the view. + 3. from_email: We want to set the from_email to info@codalab.org - may eventually put in .env file. + # The other commented sections are the defaults for other attributes in auth_views.PasswordResetView. + They are in here in case someone wants to customize in the future. All attributes show up in the order + shown in the docs. + """ + # template_name = 'registration/password_reset_form.html' + form_class = CustomPasswordResetForm # auth_forms.PasswordResetForm + # email_template_name = '' # Defaults to registration/password_reset_email.html if not supplied. + # subject_template_name = '' # Defaults to registration/password_reset_subject.txt if not supplied. + # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. + success_url = django.urls.reverse_lazy("accounts:password_reset_done") + from_email = "info@codalab.org" + + +class CustomPasswordResetConfirmView(auth_views.PasswordResetConfirmView): + """ + 1. success_url: Our src/apps/profiles/urls_accounts.py has become an "app" with the use of "app_name". + We have to use app:view_name syntax in templates like " {% url 'accounts:password_reset_confirm'%} " + Therefore we need to tell this view to find the right success_url with that syntax or django won't be + able to find the view. + """ + # template_name = '' # Default value is registration/password_reset_confirm.html. + # form_class = '' # Defaults to django.contrib.auth.forms.SetPasswordForm. + # token_generator = '' # This will default to default_token_generator, it’s an instance of django.contrib.auth.tokens.PasswordResetTokenGenerator. + # post_reset_login = '' # Defaults to False. + success_url = django.urls.reverse_lazy("accounts:password_reset_complete") + + class UserNotificationEdit(LoginRequiredMixin, DetailView): queryset = User.objects.all() template_name = 'profiles/user_notifications.html' diff --git a/src/static/riot/competitions/detail/_detailed_results.tag b/src/static/riot/competitions/detail/_detailed_results.tag new file mode 100644 index 000000000..ae5617fdb --- /dev/null +++ b/src/static/riot/competitions/detail/_detailed_results.tag @@ -0,0 +1,21 @@ + +

Detailed Results

+ + + + + + \ No newline at end of file diff --git a/src/static/riot/competitions/detail/leaderboards.tag b/src/static/riot/competitions/detail/leaderboards.tag index 05dc2c3dd..f23d59a01 100644 --- a/src/static/riot/competitions/detail/leaderboards.tag +++ b/src/static/riot/competitions/detail/leaderboards.tag @@ -30,11 +30,13 @@ Task: { task.name } + # Participant {column.title} + Detailed Results @@ -55,6 +57,7 @@ { submission.owner } { submission.organization.name } { get_score(column, submission) } + Show detailed results @@ -67,6 +70,7 @@ self.filtered_columns = [] self.phase_id = null self.competition_id = null + self.enable_detailed_results = false self.get_score = function(column, submission) { if(column.task_id === -1){ @@ -117,6 +121,8 @@ CODALAB.api.get_leaderboard_for_render(self.phase_id) .done(responseData => { self.selected_leaderboard = responseData + + self.columns = [] // Make fake task and columns for Metadata so it can be filtered like columns if(self.selected_leaderboard.fact_sheet_keys){ @@ -154,6 +160,8 @@ CODALAB.events.on('competition_loaded', (competition) => { self.competition_id = competition.id self.opts.is_admin ? self.show_download = "visible": self.show_download = "hidden" + self.enable_detailed_results = competition.enable_detailed_results + }) CODALAB.events.on('submission_changed_on_leaderboard', self.update_leaderboard) diff --git a/src/templates/base.html b/src/templates/base.html index 850123411..5e96e8e1b 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -11,7 +11,7 @@ - + @@ -145,6 +145,10 @@ Notifications + + + Change Password + Logout @@ -227,7 +231,7 @@

CodaBench

- + diff --git a/src/templates/competitions/detailed_results.html b/src/templates/competitions/detailed_results.html new file mode 100644 index 000000000..17d8b9be5 --- /dev/null +++ b/src/templates/competitions/detailed_results.html @@ -0,0 +1,4 @@ +{% extends "base.html" %} +{% block content %} + +{% endblock %} \ No newline at end of file diff --git a/src/templates/registration/login.html b/src/templates/registration/login.html index 4ffd5fcb3..f4b802cb2 100644 --- a/src/templates/registration/login.html +++ b/src/templates/registration/login.html @@ -53,6 +53,7 @@

New to us? Sign Up

+

Forgot your password?

diff --git a/src/templates/registration/password_reset_complete.html b/src/templates/registration/password_reset_complete.html new file mode 100644 index 000000000..e69a6e197 --- /dev/null +++ b/src/templates/registration/password_reset_complete.html @@ -0,0 +1,10 @@ +{% extends 'base.html' %} + +{% block content %} +
+

+ Password Reset Complete +

+

Your password has been successfully reset.

+
+{% endblock %} diff --git a/src/templates/registration/password_reset_confirm.html b/src/templates/registration/password_reset_confirm.html new file mode 100644 index 000000000..873f819bb --- /dev/null +++ b/src/templates/registration/password_reset_confirm.html @@ -0,0 +1,14 @@ +{% extends 'base.html' %} + +{% block content %} +
+

+ Change Password +

+
+ {% csrf_token %} + {{ form.as_p }} + +
+
+{% endblock %} diff --git a/src/templates/registration/password_reset_done.html b/src/templates/registration/password_reset_done.html new file mode 100644 index 000000000..c36f7c225 --- /dev/null +++ b/src/templates/registration/password_reset_done.html @@ -0,0 +1,18 @@ +{% extends "registration/registration_base.html" %} + +{% block title %}Password reset{% endblock %} + +{% block content %} +
+

+ Password Reset Complete +

+

+ We have sent you an email with a link to reset your password. Please check + your email and click the link to continue. +

+
+{% endblock %} + + +{# This is used by django.contrib.auth #} \ No newline at end of file diff --git a/src/templates/registration/password_reset_email.html b/src/templates/registration/password_reset_email.html new file mode 100644 index 000000000..249023bb2 --- /dev/null +++ b/src/templates/registration/password_reset_email.html @@ -0,0 +1,2 @@ +Someone asked for password reset for email {{ email }}. Follow the link below: +{{ protocol}}://{{ domain }}{% url 'accounts:password_reset_confirm' uidb64=uid token=token %} diff --git a/src/templates/registration/password_reset_form.html b/src/templates/registration/password_reset_form.html new file mode 100644 index 000000000..5d225765d --- /dev/null +++ b/src/templates/registration/password_reset_form.html @@ -0,0 +1,24 @@ +{% extends "registration/registration_base.html" %} + + +{% block title %}Reset password{% endblock %} + +{% block content %} +
+

+ Reset password +

+ {% if user.is_authenticated %} +

Note: you are already logged in as {{ user.username }}.

+ {% endif %} +

Forgot your password? Enter your email in the form below and we'll send you instructions for creating a new one.

+
+ {% csrf_token %} + {{ form.as_p }} + +
+
+{% endblock %} + + +{# This is used by django.contrib.auth #} \ No newline at end of file diff --git a/src/templates/registration/registration_base.html b/src/templates/registration/registration_base.html new file mode 100644 index 000000000..63913c188 --- /dev/null +++ b/src/templates/registration/registration_base.html @@ -0,0 +1 @@ +{% extends "base.html" %} \ No newline at end of file diff --git a/src/templates/registration/signup.html b/src/templates/registration/signup.html index f5d5ca78b..426d05bcc 100644 --- a/src/templates/registration/signup.html +++ b/src/templates/registration/signup.html @@ -87,6 +87,11 @@

{% endif %} +
+ + +
+
diff --git a/src/tests/functional/test_competitions.py b/src/tests/functional/test_competitions.py index cdbebe8a2..3db8bd355 100644 --- a/src/tests/functional/test_competitions.py +++ b/src/tests/functional/test_competitions.py @@ -11,8 +11,8 @@ from tasks.models import Task from ..utils import SeleniumTestCase -SHORT_WAIT = 0.1 -LONG_WAIT = 2 +SHORT_WAIT = 0.2 +LONG_WAIT = 4 class TestCompetitions(SeleniumTestCase): diff --git a/src/tests/functional/test_submissions.py b/src/tests/functional/test_submissions.py index e6e094ba2..a8451797c 100644 --- a/src/tests/functional/test_submissions.py +++ b/src/tests/functional/test_submissions.py @@ -9,6 +9,9 @@ from utils.storage import md5 from ..utils import SeleniumTestCase +LONG_WAIT = 4 +SHORT_WAIT = 0.2 + class TestSubmissions(SeleniumTestCase): def setUp(self): @@ -36,7 +39,7 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio self.assert_current_url(comp_url) # This clicks the page before it loads fully, delay it a bit... - self.wait(1) + self.wait(LONG_WAIT) self.find('.item[data-tab="participate-tab"]').click() self.circleci_screenshot("set_submission_file_name.png") @@ -48,6 +51,7 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio # Inside the accordion the output is being streamed self.find('.submission-output-container .title').click() + self.wait(SHORT_WAIT) assert self.find_text_in_class('.submission_output', expected_submission_output, timeout=timeout) # The submission table lists our submission!