diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3f78c748 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +**/*.pyc +**/.pytest_cache/ +**/__pycache__/ +.git +.github +.idea +node_modules diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 507108db..c0fa5ce3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,19 +1,22 @@ name: Python build -on: [push, pull_request] +on: [ push, pull_request ] jobs: build: runs-on: ubuntu-latest - # Consistent with Version.md - container: nastyabirillo/hyperstyle:1.2.1 + # Consistent with base image in Dockerfile + container: stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3 steps: - - name: Checkout uses: actions/checkout@v1 + - name: Install requirements + run: | + pip install --no-cache-dir -r requirements-test.txt -r requirements.txt + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -21,21 +24,24 @@ jobs: # TODO: change max-complexity into 10 after refactoring # TODO: remove R504, A003, E800, E402, WPS1, WPS2, WPS3, WPS4, WPS5, WPS6, H601 flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS1,WPS2,WPS3,WPS4,WPS5,WPS6,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules - + - name: Set up Eslint run: | - npm install eslint --save-dev - ./node_modules/.bin/eslint --init + # Consistent with eslint version in Dockerfile + npm install eslint@7.5.0 -g && eslint --init - name: Test with pytest run: | - pytest - - # We should have only INFO errors + pytest + - name: Check installed module can run python linters run: | - python src/python/review/run_tool.py setup.py - + python src/python/review/run_tool.py setup.py + - name: Check installed module can run java linters run: | python src/python/review/run_tool.py test/resources/inspectors/java/test_algorithm_with_scanner.java + + - name: Check installed module can run js linters + run: | + python src/python/review/run_tool.py test/resources/inspectors/js/case0_no_issues.js \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 41f14cdb..b33c5f72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,12 @@ -# This Dockerfile is used only for production +FROM stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3 -FROM python:3.8.2-alpine3.11 - -RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \ - && apk add --update nodejs npm - -RUN npm i -g eslint@7.5.0 - -RUN java -version -RUN ls /usr/lib/jvm - -# Other dependencies -RUN apk add bash - -# Set up Eslint -RUN npm install eslint --save-dev && ./node_modules/.bin/eslint --init - -# Dependencies and package installation -WORKDIR / - -COPY requirements-test.txt review/requirements-test.txt -RUN pip3 install --no-cache-dir -r review/requirements-test.txt - -COPY requirements.txt review/requirements.txt -RUN pip3 install --no-cache-dir -r review/requirements.txt +RUN npm install eslint@7.5.0 -g \ + && eslint --init COPY . review -RUN pip3 install --no-cache-dir ./review - -# Container's enviroment variables -ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk -ENV PATH="$JAVA_HOME/bin:${PATH}" +RUN pip install --no-cache-dir \ + -r review/requirements-test.txt \ + -r review/requirements.txt \ + ./review -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 00000000..c6869724 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,170 @@ +FROM python:3.8.11-slim + +######### +# Taken from https://github.com/docker-library/openjdk/blob/608f26c5ea63ca34070b439c904cb94a30f6b0c1/11/jdk/slim-buster/Dockerfile +######### + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ +# utilities for keeping Debian and OpenJDK CA certificates in sync + ca-certificates p11-kit \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME /usr/local/openjdk-11 +RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] # backwards compatibility +ENV PATH $JAVA_HOME/bin:$PATH + +# Default to UTF-8 file.encoding +ENV LANG C.UTF-8 + +# https://adoptopenjdk.net/upstream.html +# > +# > What are these binaries? +# > +# > These binaries are built by Red Hat on their infrastructure on behalf of the OpenJDK jdk8u and jdk11u projects. The binaries are created from the unmodified source code at OpenJDK. Although no formal support agreement is provided, please report any bugs you may find to https://bugs.java.com/. +# > +ENV JAVA_VERSION 11.0.11+9 +# https://github.com/docker-library/openjdk/issues/320#issuecomment-494050246 +# > +# > I am the OpenJDK 8 and 11 Updates OpenJDK project lead. +# > ... +# > While it is true that the OpenJDK Governing Board has not sanctioned those releases, they (or rather we, since I am a member) didn't sanction Oracle's OpenJDK releases either. As far as I am aware, the lead of an OpenJDK project is entitled to release binary builds, and there is clearly a need for them. +# > + +RUN set -eux; \ + \ + arch="$(dpkg --print-architecture)"; \ + case "$arch" in \ + 'amd64') \ + downloadUrl='https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_x64_linux_11.0.11_9.tar.gz'; \ + ;; \ + 'arm64') \ + downloadUrl='https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_aarch64_linux_11.0.11_9.tar.gz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ + esac; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + dirmngr \ + gnupg \ + wget \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + \ + wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \ + wget --progress=dot:giga -O openjdk.tgz.asc "$downloadUrl.sign"; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ +# pre-fetch Andrew Haley's (the OpenJDK 8 and 11 Updates OpenJDK project lead) key so we can verify that the OpenJDK key was signed by it +# (https://github.com/docker-library/openjdk/pull/322#discussion_r286839190) +# we pre-fetch this so that the signature it makes on the OpenJDK key can survive "import-clean" in gpg + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys EAC843EBD3EFDB98CC772FADA5CD6035332FA671; \ +# TODO find a good link for users to verify this key is right (https://mail.openjdk.java.net/pipermail/jdk-updates-dev/2019-April/000951.html is one of the only mentions of it I can find); perhaps a note added to https://adoptopenjdk.net/upstream.html would make sense? +# no-self-sigs-only: https://salsa.debian.org/debian/gnupg2/commit/c93ca04a53569916308b369c8b218dad5ae8fe07 + gpg --batch --keyserver keyserver.ubuntu.com --keyserver-options no-self-sigs-only --recv-keys CA5F11C6CE22644D42C6AC4492EF8D39DC13168F; \ + gpg --batch --list-sigs --keyid-format 0xLONG CA5F11C6CE22644D42C6AC4492EF8D39DC13168F \ + | tee /dev/stderr \ + | grep '0xA5CD6035332FA671' \ + | grep 'Andrew Haley'; \ + gpg --batch --verify openjdk.tgz.asc openjdk.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + \ + mkdir -p "$JAVA_HOME"; \ + tar --extract \ + --file openjdk.tgz \ + --directory "$JAVA_HOME" \ + --strip-components 1 \ + --no-same-owner \ + ; \ + rm openjdk.tgz*; \ + \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + \ +# update "cacerts" bundle to use Debian's CA certificates (and make sure it stays up-to-date with changes to Debian's store) +# see https://github.com/docker-library/openjdk/issues/327 +# http://rabexc.org/posts/certificates-not-working-java#comment-4099504075 +# https://salsa.debian.org/java-team/ca-certificates-java/blob/3e51a84e9104823319abeb31f880580e46f45a98/debian/jks-keystore.hook.in +# https://git.alpinelinux.org/aports/tree/community/java-cacerts/APKBUILD?id=761af65f38b4570093461e6546dcf6b179d2b624#n29 + { \ + echo '#!/usr/bin/env bash'; \ + echo 'set -Eeuo pipefail'; \ + echo 'trust extract --overwrite --format=java-cacerts --filter=ca-anchors --purpose=server-auth "$JAVA_HOME/lib/security/cacerts"'; \ + } > /etc/ca-certificates/update.d/docker-openjdk; \ + chmod +x /etc/ca-certificates/update.d/docker-openjdk; \ + /etc/ca-certificates/update.d/docker-openjdk; \ + \ +# https://github.com/docker-library/openjdk/issues/331#issuecomment-498834472 + find "$JAVA_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf; \ + ldconfig; \ + \ +# https://github.com/docker-library/openjdk/issues/212#issuecomment-420979840 +# https://openjdk.java.net/jeps/341 + java -Xshare:dump; \ + \ +# basic smoke test + fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java; \ + javac --version; \ + java --version + +######### +# Taken from https://github.com/nodejs/docker-node/blob/fd130acf063b312355a5d88d51716db3ff34ae49/14/buster-slim/Dockerfile +######### + +ENV NODE_VERSION 14.17.3 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 74F12602B6F1C4E913FAA37AD3A89613643B6201 \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version diff --git a/README.md b/README.md index ca364037..bd656c79 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,8 @@ __Note__: If you have `ModuleNotFoundError` while you try to run tests, please c __Note__: We use [eslint](https://eslint.org/) and [open-jdk 11](https://openjdk.java.net/projects/jdk/11/) in the tests. Please, set up the environment before running the tests. -You can see en example of the environment configuration in -the [Dockerfile](./docker/dev/Dockerfile) file. +You can see en example of the environment configuration in +the [Dockerfile](Dockerfile) file. Use `pytest` from the root directory to run __ALL__ tests. + diff --git a/requirements-evaluation.txt b/requirements-evaluation.txt index 014f2f6c..046f8026 100644 --- a/requirements-evaluation.txt +++ b/requirements-evaluation.txt @@ -1,3 +1 @@ -openpyxl==3.0.7 -pandarallel -plotly +plotly==4.14.3 diff --git a/requirements-roberta.txt b/requirements-roberta.txt index 287d83b5..15421cf1 100644 --- a/requirements-roberta.txt +++ b/requirements-roberta.txt @@ -1,5 +1,5 @@ tqdm==4.49.0 -scikit-learn~=0.24.2 +scikit-learn==0.24.2 transformers==4.6.1 tokenizers==0.10.2 torch==1.8.1 diff --git a/requirements-test.txt b/requirements-test.txt index cb0543a3..dc0f70dc 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,11 @@ -pytest~=6.2.3 -pytest-runner -pytest-subtests -jsonschema~=3.2.0 -django~=3.2 -pylint~=2.7.4 -requests~=2.25.1 -setuptools~=56.0.0 -openpyxl==3.0.7 \ No newline at end of file +pytest==6.2.3 +pytest-runner==5.2 +pytest-subtests==0.4.0 +jsonschema==3.2.0 +pandas==1.2.3 +django==3.2 +pylint==2.7.4 +requests==2.25.1 +setuptools==56.0.0 +openpyxl==3.0.7 +pandarallel==1.5.2 \ No newline at end of file diff --git a/src/python/review/inspectors/detekt/detekt.py b/src/python/review/inspectors/detekt/detekt.py index 8f5f0946..041c6b2d 100644 --- a/src/python/review/inspectors/detekt/detekt.py +++ b/src/python/review/inspectors/detekt/detekt.py @@ -41,7 +41,7 @@ def _create_command(cls, path: Path, output_path: Path): '--input', str(path), ] - def inspect(self, path: Path, config) -> List[BaseIssue]: + def inspect(self, path: Path, config: dict) -> List[BaseIssue]: with new_temp_dir() as temp_dir: output_path = temp_dir / 'output.xml' command = self._create_command(path, output_path) diff --git a/src/python/review/inspectors/eslint/eslint.py b/src/python/review/inspectors/eslint/eslint.py index 839a166c..c7d0b1b4 100644 --- a/src/python/review/inspectors/eslint/eslint.py +++ b/src/python/review/inspectors/eslint/eslint.py @@ -21,8 +21,9 @@ class ESLintInspector(BaseInspector): } @classmethod - def _create_command(cls, path: Path, output_path: Path, is_local: bool = False) -> List[str]: - eslint_command = 'eslint' if not is_local else './node_modules/.bin/eslint' + def _create_command(cls, path: Path, output_path: Path) -> List[str]: + local_path = 'node_modules/.bin/eslint' # used only in local dev environment + eslint_command = local_path if Path(local_path).exists() else 'eslint' return [ eslint_command, '-c', PATH_ESLINT_CONFIG, @@ -31,10 +32,10 @@ def _create_command(cls, path: Path, output_path: Path, is_local: bool = False) path, ] - def inspect(self, path: Path, config: dict, is_local: bool = False) -> List[BaseIssue]: + def inspect(self, path: Path, config: dict) -> List[BaseIssue]: with new_temp_dir() as temp_dir: output_path = temp_dir / 'output.xml' - command = self._create_command(path, output_path, is_local) + command = self._create_command(path, output_path) run_in_subprocess(command) issues = parse_checkstyle_file_result(output_path, diff --git a/test/python/inspectors/test_eslint_inspector.py b/test/python/inspectors/test_eslint_inspector.py index a69f30e5..d7aceeac 100644 --- a/test/python/inspectors/test_eslint_inspector.py +++ b/test/python/inspectors/test_eslint_inspector.py @@ -19,7 +19,7 @@ def test_file_with_issues(file_name: str, n_issues: int): path_to_file = JS_DATA_FOLDER / file_name with use_file_metadata(path_to_file) as file_metadata: - issues = inspector.inspect(file_metadata.path, {}, True) + issues = inspector.inspect(file_metadata.path, {}) issues = filter_low_measure_issues(issues, Language.JS) assert len(issues) == n_issues