Merge main into develop

.dockerignore

@@ -0,0 +1,7 @@
+**/*.pyc
+**/.pytest_cache/
+**/__pycache__/
+.git
+.github
+.idea
+node_modules

.github/workflows/build.yml

@@ -1,41 +1,47 @@
 name: Python build
 
-on: [push, pull_request]
+on: [ push, pull_request ]
 
 jobs:
 
   build:
     runs-on: ubuntu-latest
-    # Consistent with Version.md
-    container: nastyabirillo/hyperstyle:1.2.1
+    # Consistent with base image in Dockerfile
+    container: stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3
 
     steps:
-
       - name: Checkout
         uses: actions/checkout@v1
 
+      - name: Install requirements
+        run: |
+          pip install --no-cache-dir -r requirements-test.txt -r requirements.txt
+
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
           # TODO: change max-complexity into 10 after refactoring
           # TODO: remove R504, A003, E800, E402, WPS1, WPS2, WPS3, WPS4, WPS5, WPS6, H601
           flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS1,WPS2,WPS3,WPS4,WPS5,WPS6,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
-   
+
       - name: Set up Eslint
         run: |
-          npm install eslint --save-dev
-          ./node_modules/.bin/eslint --init
+          # Consistent with eslint version in Dockerfile
+          npm install eslint@7.5.0 -g && eslint --init
 
       - name: Test with pytest
         run: |
-            pytest
-
-      # We should have only INFO errors      
+          pytest
+
       - name: Check installed module can run python linters
         run: |
-            python src/python/review/run_tool.py setup.py 
-             
+            python src/python/review/run_tool.py setup.py
+
       - name: Check installed module can run java linters
         run: |
             python src/python/review/run_tool.py test/resources/inspectors/java/test_algorithm_with_scanner.java
+
+      - name: Check installed module can run js linters
+        run: |
+          python src/python/review/run_tool.py test/resources/inspectors/js/case0_no_issues.js

Dockerfile

@@ -1,35 +1,12 @@
-# This Dockerfile is used only for production
+FROM stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3
 
-FROM python:3.8.2-alpine3.11
-
-RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
-    && apk add --update nodejs npm
-
-RUN npm i -g eslint@7.5.0
-
-RUN java -version
-RUN ls /usr/lib/jvm
-
-# Other dependencies
-RUN apk add bash
-
-# Set up Eslint
-RUN npm install eslint --save-dev && ./node_modules/.bin/eslint --init
-
-# Dependencies and package installation
-WORKDIR /
-
-COPY requirements-test.txt review/requirements-test.txt
-RUN pip3 install --no-cache-dir -r review/requirements-test.txt
-
-COPY requirements.txt review/requirements.txt
-RUN pip3 install --no-cache-dir -r review/requirements.txt
+RUN npm install eslint@7.5.0 -g \
+    && eslint --init
 
 COPY . review
-RUN pip3 install --no-cache-dir ./review
-
-# Container's enviroment variables
-ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk
-ENV PATH="$JAVA_HOME/bin:${PATH}"
+RUN pip install --no-cache-dir \
+    -r review/requirements-test.txt \
+    -r review/requirements.txt \
+    ./review
 
-CMD ["/bin/bash"]
+CMD ["/bin/bash"]

Dockerfile.base

@@ -0,0 +1,170 @@
+FROM python:3.8.11-slim
+
+#########
+# Taken from https://github.com/docker-library/openjdk/blob/608f26c5ea63ca34070b439c904cb94a30f6b0c1/11/jdk/slim-buster/Dockerfile
+#########
+
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+# utilities for keeping Debian and OpenJDK CA certificates in sync
+		ca-certificates p11-kit \
+	; \
+	rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME /usr/local/openjdk-11
+RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] # backwards compatibility
+ENV PATH $JAVA_HOME/bin:$PATH
+
+# Default to UTF-8 file.encoding
+ENV LANG C.UTF-8
+
+# https://adoptopenjdk.net/upstream.html
+# >
+# > What are these binaries?
+# >
+# > These binaries are built by Red Hat on their infrastructure on behalf of the OpenJDK jdk8u and jdk11u projects. The binaries are created from the unmodified source code at OpenJDK. Although no formal support agreement is provided, please report any bugs you may find to https://bugs.java.com/.
+# >
+ENV JAVA_VERSION 11.0.11+9
+# https://github.com/docker-library/openjdk/issues/320#issuecomment-494050246
+# >
+# > I am the OpenJDK 8 and 11 Updates OpenJDK project lead.
+# > ...
+# > While it is true that the OpenJDK Governing Board has not sanctioned those releases, they (or rather we, since I am a member) didn't sanction Oracle's OpenJDK releases either. As far as I am aware, the lead of an OpenJDK project is entitled to release binary builds, and there is clearly a need for them.
+# >
+
+RUN set -eux; \
+	\
+	arch="$(dpkg --print-architecture)"; \
+	case "$arch" in \
+		'amd64') \
+			downloadUrl='https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_x64_linux_11.0.11_9.tar.gz'; \
+			;; \
+		'arm64') \
+			downloadUrl='https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_aarch64_linux_11.0.11_9.tar.gz'; \
+			;; \
+		*) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \
+	esac; \
+	\
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		dirmngr \
+		gnupg \
+		wget \
+	; \
+	rm -rf /var/lib/apt/lists/*; \
+	\
+	wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \
+	wget --progress=dot:giga -O openjdk.tgz.asc "$downloadUrl.sign"; \
+	\
+	export GNUPGHOME="$(mktemp -d)"; \
+# pre-fetch Andrew Haley's (the OpenJDK 8 and 11 Updates OpenJDK project lead) key so we can verify that the OpenJDK key was signed by it
+# (https://github.com/docker-library/openjdk/pull/322#discussion_r286839190)
+# we pre-fetch this so that the signature it makes on the OpenJDK key can survive "import-clean" in gpg
+	gpg --batch --keyserver keyserver.ubuntu.com --recv-keys EAC843EBD3EFDB98CC772FADA5CD6035332FA671; \
+# TODO find a good link for users to verify this key is right (https://mail.openjdk.java.net/pipermail/jdk-updates-dev/2019-April/000951.html is one of the only mentions of it I can find); perhaps a note added to https://adoptopenjdk.net/upstream.html would make sense?
+# no-self-sigs-only: https://salsa.debian.org/debian/gnupg2/commit/c93ca04a53569916308b369c8b218dad5ae8fe07
+	gpg --batch --keyserver keyserver.ubuntu.com --keyserver-options no-self-sigs-only --recv-keys CA5F11C6CE22644D42C6AC4492EF8D39DC13168F; \
+	gpg --batch --list-sigs --keyid-format 0xLONG CA5F11C6CE22644D42C6AC4492EF8D39DC13168F \
+		| tee /dev/stderr \
+		| grep '0xA5CD6035332FA671' \
+		| grep 'Andrew Haley'; \
+	gpg --batch --verify openjdk.tgz.asc openjdk.tgz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME"; \
+	\
+	mkdir -p "$JAVA_HOME"; \
+	tar --extract \
+		--file openjdk.tgz \
+		--directory "$JAVA_HOME" \
+		--strip-components 1 \
+		--no-same-owner \
+	; \
+	rm openjdk.tgz*; \
+	\
+	apt-mark auto '.*' > /dev/null; \
+	[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	\
+# update "cacerts" bundle to use Debian's CA certificates (and make sure it stays up-to-date with changes to Debian's store)
+# see https://github.com/docker-library/openjdk/issues/327
+#     http://rabexc.org/posts/certificates-not-working-java#comment-4099504075
+#     https://salsa.debian.org/java-team/ca-certificates-java/blob/3e51a84e9104823319abeb31f880580e46f45a98/debian/jks-keystore.hook.in
+#     https://git.alpinelinux.org/aports/tree/community/java-cacerts/APKBUILD?id=761af65f38b4570093461e6546dcf6b179d2b624#n29
+	{ \
+		echo '#!/usr/bin/env bash'; \
+		echo 'set -Eeuo pipefail'; \
+		echo 'trust extract --overwrite --format=java-cacerts --filter=ca-anchors --purpose=server-auth "$JAVA_HOME/lib/security/cacerts"'; \
+	} > /etc/ca-certificates/update.d/docker-openjdk; \
+	chmod +x /etc/ca-certificates/update.d/docker-openjdk; \
+	/etc/ca-certificates/update.d/docker-openjdk; \
+	\
+# https://github.com/docker-library/openjdk/issues/331#issuecomment-498834472
+	find "$JAVA_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf; \
+	ldconfig; \
+	\
+# https://github.com/docker-library/openjdk/issues/212#issuecomment-420979840
+# https://openjdk.java.net/jeps/341
+	java -Xshare:dump; \
+	\
+# basic smoke test
+	fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java; \
+	javac --version; \
+	java --version
+
+#########
+# Taken from  https://github.com/nodejs/docker-node/blob/fd130acf063b312355a5d88d51716db3ff34ae49/14/buster-slim/Dockerfile
+#########
+
+ENV NODE_VERSION 14.17.3
+
+RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
+    && case "${dpkgArch##*-}" in \
+      amd64) ARCH='x64';; \
+      ppc64el) ARCH='ppc64le';; \
+      s390x) ARCH='s390x';; \
+      arm64) ARCH='arm64';; \
+      armhf) ARCH='armv7l';; \
+      i386) ARCH='x86';; \
+      *) echo "unsupported architecture"; exit 1 ;; \
+    esac \
+    && set -ex \
+    # libatomic1 for arm
+    && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/* \
+    && for key in \
+      4ED778F539E3634C779C87C6D7062848A1AB005C \
+      94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
+      74F12602B6F1C4E913FAA37AD3A89613643B6201 \
+      71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
+      8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
+      C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
+      C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \
+      DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
+      A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
+      108F52B48DB57BB0CC439B2997B01419BD92F80A \
+      B9E2F5981AA6E0CD28160D9FF13993A75599653C \
+    ; do \
+      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \
+      gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \
+    done \
+    && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
+    && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
+    && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
+    && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
+    && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
+    && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
+    && apt-mark auto '.*' > /dev/null \
+    && find /usr/local -type f -executable -exec ldd '{}' ';' \
+      | awk '/=>/ { print $(NF-1) }' \
+      | sort -u \
+      | xargs -r dpkg-query --search \
+      | cut -d: -f1 \
+      | sort -u \
+      | xargs -r apt-mark manual \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+    && ln -s /usr/local/bin/node /usr/local/bin/nodejs \
+    # smoke tests
+    && node --version \
+    && npm --version

README.md

@@ -201,7 +201,8 @@ __Note__: If you have `ModuleNotFoundError` while you try to run tests, please c
 
 __Note__: We use [eslint](https://eslint.org/) and [open-jdk 11](https://openjdk.java.net/projects/jdk/11/)
 in the tests. Please, set up the environment before running the tests. 
-You can see en example of the environment configuration in 
-the [Dockerfile](./docker/dev/Dockerfile) file.
+You can see en example of the environment configuration in
+the [Dockerfile](Dockerfile) file.
 
 Use `pytest` from the root directory to run __ALL__ tests.
+

requirements-evaluation.txt

@@ -1,3 +1 @@
-openpyxl==3.0.7
-pandarallel
-plotly
+plotly==4.14.3

requirements-roberta.txt

@@ -1,5 +1,5 @@
 tqdm==4.49.0
-scikit-learn~=0.24.2
+scikit-learn==0.24.2
 transformers==4.6.1
 tokenizers==0.10.2
 torch==1.8.1

requirements-test.txt

@@ -1,9 +1,11 @@
-pytest~=6.2.3
-pytest-runner
-pytest-subtests
-jsonschema~=3.2.0
-django~=3.2
-pylint~=2.7.4
-requests~=2.25.1
-setuptools~=56.0.0
-openpyxl==3.0.7
+pytest==6.2.3
+pytest-runner==5.2
+pytest-subtests==0.4.0
+jsonschema==3.2.0
+pandas==1.2.3
+django==3.2
+pylint==2.7.4
+requests==2.25.1
+setuptools==56.0.0
+openpyxl==3.0.7
+pandarallel==1.5.2

src/python/review/inspectors/detekt/detekt.py

@@ -41,7 +41,7 @@ def _create_command(cls, path: Path, output_path: Path):
             '--input', str(path),
         ]
 
-    def inspect(self, path: Path, config) -> List[BaseIssue]:
+    def inspect(self, path: Path, config: dict) -> List[BaseIssue]:
         with new_temp_dir() as temp_dir:
             output_path = temp_dir / 'output.xml'
             command = self._create_command(path, output_path)

src/python/review/inspectors/eslint/eslint.py

@@ -21,8 +21,9 @@ class ESLintInspector(BaseInspector):
     }
 
     @classmethod
-    def _create_command(cls, path: Path, output_path: Path, is_local: bool = False) -> List[str]:
-        eslint_command = 'eslint' if not is_local else './node_modules/.bin/eslint'
+    def _create_command(cls, path: Path, output_path: Path) -> List[str]:
+        local_path = 'node_modules/.bin/eslint'  # used only in local dev environment
+        eslint_command = local_path if Path(local_path).exists() else 'eslint'
         return [
             eslint_command,
             '-c', PATH_ESLINT_CONFIG,
@@ -31,10 +32,10 @@ def _create_command(cls, path: Path, output_path: Path, is_local: bool = False)
             path,
         ]
 
-    def inspect(self, path: Path, config: dict, is_local: bool = False) -> List[BaseIssue]:
+    def inspect(self, path: Path, config: dict) -> List[BaseIssue]:
         with new_temp_dir() as temp_dir:
             output_path = temp_dir / 'output.xml'
-            command = self._create_command(path, output_path, is_local)
+            command = self._create_command(path, output_path)
             run_in_subprocess(command)
 
             issues = parse_checkstyle_file_result(output_path,

test/python/inspectors/test_eslint_inspector.py

@@ -19,7 +19,7 @@ def test_file_with_issues(file_name: str, n_issues: int):
 
     path_to_file = JS_DATA_FOLDER / file_name
     with use_file_metadata(path_to_file) as file_metadata:
-        issues = inspector.inspect(file_metadata.path, {}, True)
+        issues = inspector.inspect(file_metadata.path, {})
         issues = filter_low_measure_issues(issues, Language.JS)
 
     assert len(issues) == n_issues