Release new version

.github/workflows/build.yml

@@ -7,11 +7,23 @@ jobs:
   build:
     runs-on: ubuntu-latest
     # Consistent with base image in Dockerfile
-    container: stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3
+    # container: stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3
+    container: nastyabirillo/hyperstyle:1.2.1
 
     steps:
+      - name: Install git
+        run: |
+            apt-get update
+            apt-get -y install git
+
+      - name: Check env variables
+        run: |
+            echo $DETEKT_DIRECTORY && echo $DETEKT_VERSION
+            echo $CHECKSTYLE_DIRECTORY && echo CHECKSTYLE_VERSION
+            echo $PMD_DIRECTORY && echo PMD_VERSION
+
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Install requirements
         run: |
@@ -22,8 +34,22 @@ jobs:
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
           # TODO: change max-complexity into 10 after refactoring
+          # TODO: remove R504, A003, E800, E402, WPS1, WPS2, WPS3, WPS4, WPS5, WPS6, H601
           flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
 
+      - name: Sort whitelists
+        run: |
+            for file in "whitelist.txt" "src/python/review/inspectors/flake8/whitelist.txt"
+            do
+            LC_ALL=C sort $file -o $file
+            done
+
+      - name: Commit sorted whitelists
+        uses: EndBug/add-and-commit@v7.2.1
+        with:
+          add: "['whitelist.txt', 'src/python/review/inspectors/flake8/whitelist.txt']"
+          message: 'Sort whitelists (Github Actions)'
+
       - name: Set up Eslint
         run: |
           # Consistent with eslint version in Dockerfile

Dockerfile

@@ -9,4 +9,34 @@ RUN pip install --no-cache-dir \
     -r review/requirements.txt \
     ./review
 
+ENV LINTERS_DIRECTORY      /opt/linters
+
+ENV CHECKSTYLE_VERSION  8.44
+ENV CHECKSTYLE_DIRECTORY  ${LINTERS_DIRECTORY}/checkstyle
+
+ENV DETEKT_VERSION  1.14.2
+ENV DETEKT_DIRECTORY  ${LINTERS_DIRECTORY}/detekt
+
+ENV PMD_VERSION  6.36.0
+ENV PMD_DIRECTORY  ${LINTERS_DIRECTORY}/pmd
+
+RUN mkdir -p ${CHECKSTYLE_DIRECTORY} && mkdir -p ${DETEKT_DIRECTORY} && mkdir -p ${PMD_DIRECTORY}
+
+RUN apt -y update && apt -y upgrade
+
+# Install Curl and Unzip
+RUN apt -y install curl unzip
+
+# Install Detekt and Detekt-formatting
+RUN curl -sSLO https://github.com/detekt/detekt/releases/download/v${DETEKT_VERSION}/detekt-cli-${DETEKT_VERSION}.zip \
+    && unzip detekt-cli-${DETEKT_VERSION}.zip -d ${DETEKT_DIRECTORY} \
+    &&  curl -H "Accept: application/zip" https://repo.maven.apache.org/maven2/io/gitlab/arturbosch/detekt/detekt-formatting/${DETEKT_VERSION}/detekt-formatting-${DETEKT_VERSION}.jar -o ${DETEKT_DIRECTORY}/detekt-formatting-${DETEKT_VERSION}.jar
+
+# Install Checkstyle
+RUN curl -L https://github.com/checkstyle/checkstyle/releases/download/checkstyle-${CHECKSTYLE_VERSION}/checkstyle-${CHECKSTYLE_VERSION}-all.jar > ${CHECKSTYLE_DIRECTORY}/checkstyle-${CHECKSTYLE_VERSION}-all.jar
+
+# Install PMD
+RUN curl -sSLO https://github.com/pmd/pmd/releases/download/pmd_releases/${PMD_VERSION}/pmd-bin-${PMD_VERSION}.zip \
+    && unzip pmd-bin-${PMD_VERSION}.zip -d ${PMD_DIRECTORY}
+
 CMD ["/bin/bash"]

README.md

@@ -24,7 +24,7 @@ The source code of **hyperstyle** is distributed under the Apache 2.0 License.
 
 The 3rd party software we use in this project has its own licenses.
 
-Python language:
+Python language (all versions can be found in the (requirements.txt)[requirements.txt] file):
 
 - [x]  flake8 [MIT]
     * [Site and docs](https://flake8.pycqa.org/en/latest/)
@@ -40,34 +40,27 @@ Python language:
 
 Java language:
 
-- [x]  PMD [BSD]
+- [x]  PMD [BSD] (Version: 6.36.0)
     * [Site and docs](https://pmd.github.io/)
     * [Repository](https://github.com/pmd/pmd)
 
-- [x]  Checkstyle [GNU LGPL v2.1]
+- [x]  Checkstyle [GNU LGPL v2.1] (Version: 8.44)
     * [Site and docs](https://checkstyle.sourceforge.io/)
     * [Repository](https://github.com/checkstyle/checkstyle)
-
-- [ ]  SpotBugs [GNU LGPL v2.1]
-    * [Site and docs](https://spotbugs.github.io/)
-    * [Repository](https://github.com/spotbugs/spotbugs)
-
-- [ ]  SpringLint
-    * [Repository](https://github.com/mauricioaniche/springlint)
 
 
 
 Kotlin language:
 
-- [x]  Detekt [Apache 2.0]
+- [x]  Detekt [Apache 2.0] (Version: 1.14.2)
     * [Site and docs](https://detekt.github.io/detekt/)
     * [Repository](https://github.com/detekt/detekt)
 
 
 
 JavaScript language:
 
-- [x]  ESlint [MIT]
+- [x]  ESlint [MIT] (Version: 7.5.0)
     * [Site and docs](https://eslint.org/)
     * [Repository](https://github.com/eslint/eslint)
 
@@ -80,6 +73,37 @@ Simply clone the repository and run the following commands:
 1. `pip install -r requirements.txt`
 2. `pip install -r requirements-test.txt` for tests
 
+Also, you should set up the environment with Java and Kotlin linters sources.
+You have to create set of environment variables:
+- `CHECKSTYLE_VERSION` (the value of the variable must be the same with its value in [Dockerfile](Dockerfile))
+- `CHECKSTYLE_DIRECTORY` (the directory with `CHECKSTYLE` linter sources)
+- `DETEKT_VERSION` (the value of the variable must be the same with its value in [Dockerfile](Dockerfile))
+- `DETEKT_DIRECTORY` (the directory with `DETEKT` linter sources)
+- `PMD_VERSION` (the value of the variable must be the same with its value in [Dockerfile](Dockerfile))
+- `PMD_DIRECTORY` (the directory with `PMD` linter sources)
+
+You can download all sources manually or by the following commands:
+- `CHECKSTYLE`: 
+```bash
+curl -L https://github.com/checkstyle/checkstyle/releases/download/checkstyle-${CHECKSTYLE_VERSION}/checkstyle-${CHECKSTYLE_VERSION}-all.jar > ${CHECKSTYLE_DIRECTORY}/checkstyle-${CHECKSTYLE_VERSION}-all.jar
+```
+- `DETEKT`: 
+```bash
+curl -sSLO https://github.com/detekt/detekt/releases/download/v${DETEKT_VERSION}/detekt-cli-${DETEKT_VERSION}.zip \
+&& unzip detekt-cli-${DETEKT_VERSION}.zip -d ${DETEKT_DIRECTORY} \
+&&  curl -H "Accept: application/zip" https://repo.maven.apache.org/maven2/io/gitlab/arturbosch/detekt/detekt-formatting/${DETEKT_VERSION}/detekt-formatting-${DETEKT_VERSION}.jar -o ${DETEKT_DIRECTORY}/detekt-formatting-${DETEKT_VERSION}.jar
+```
+- `PMD`: 
+```bash
+curl -sSLO https://github.com/pmd/pmd/releases/download/pmd_releases/${PMD_VERSION}/pmd-bin-${PMD_VERSION}.zip \
+&& unzip pmd-bin-${PMD_VERSION}.zip -d ${PMD_DIRECTORY}
+```
+
+_In the future we will add a bash script that will do it automatically._ 
+
+**Alternatively**, you can build a docker image by [Dockerfile](Dockerfile) and run the tool inside this image.
+Or use the public docker image, that we use in the [build.yml](.github/workflows/build.yml) file.
+
 ## Usage
 
 Run the [run_tool.py](./src/python/review/run_tool.py) with the arguments.
@@ -97,12 +121,15 @@ Argument | Description
 **‑v**, **‑‑verbosity** |  choose logging level according [this](https://docs.python.org/3/library/logging.html#levels) list: `1` - **ERROR**; `2` - **INFO**; `3` - **DEBUG**; `0` - disable logging (**CRITICAL** value); default value is `0` (**CRITICAL**).
 **‑d**, **‑‑disable**   |  disable inspectors. Available values: for **Python** language: `pylint` for [Pylint](https://github.com/PyCQA/pylint), `flake8` for [flake8](https://flake8.pycqa.org/en/latest/), `radon` for [Radon](https://radon.readthedocs.io/en/latest/), `python_ast` to check different measures providing by AST; for **Java** language: `checkstyle` for the [Checkstyle](https://checkstyle.sourceforge.io/), `pmd` for [PMD](https://pmd.github.io/); for `Kotlin` language: detekt for [Detekt](https://detekt.github.io/detekt/); for **JavaScript** language: `eslint` for [ESlint](https://eslint.org/). Example: `-d pylint,flake8`.
 **‑‑allow-duplicates**        |  allow duplicate issues found by different linters. By default, duplicates are skipped.
-**‑‑language-version**, **‑‑language_version**  |  specify the language version for JAVA inspectors. Available values: `java7`, `java8`, `java9`, `java11`. **Note**: **‑‑language_version** is deprecated. Will be deleted in the future.
+**‑‑language-version**, **‑‑language_version**  |  specify the language version for JAVA inspectors. Available values: `java7`, `java8`, `java9`, `java11`, `java15`. **Note**: **‑‑language_version** is deprecated and will be deleted in the future.
 **‑‑n-cpu**, **‑‑n_cpu**  |  specify number of _cpu_ that can be used to run inspectors. **Note**: **‑‑n_cpu** is deprecated. Will be deleted in the future.
 **‑f**, **‑‑format**    |  the output format. Available values: `json`, `text`. Default value is `json`.
 **‑s**, **‑‑start-line**|  the first line to be analyzed. By default it starts from `1`.
 **‑e**, **‑‑end-line**  |  the end line to be analyzed. The default value is `None`, which meant to handle file by the end.
 **‑‑new-format**              |  the argument determines whether the tool should use the _new format_. _New format_ means separating the result by the files to allow getting quality and observed issues for each file separately. The default value is `False`.
+**‑‑history**                 |  JSON string with a list of issues for each language. For each issue its class and quantity are specified. Example: `--history "{\"python\": [{\"origin_class\": \"SC200\", \"number\": 20}, {\"origin_class\": \"WPS314\", \"number\": 3}]}"`
+**‑‑with‑all‑categories** | Without this flag, all issues will be categorized into 5 main categories: `CODE_STYLE`, `BEST_PRACTICES`, `ERROR_PRONE`, `COMPLEXITY`, `INFO`.
+**‑‑group‑by‑difficulty** | With this flag, the final grade and influence on penalty will be grouped by the issue difficulty.
 
 The output examples:
 
@@ -123,7 +150,9 @@ The output examples:
       "line": "<the code line>",
       "line_number": 54,
       "column_number": 0,
-      "category": "FUNC_LEN"
+      "category": "FUNC_LEN",
+      "difficulty": "EASY",
+      "influence_on_penalty": 0 
     },
     ...
   ]
@@ -152,7 +181,104 @@ The output examples:
           "line": "<the code line>",
           "line_number": 174,
           "column_number": 12,
-          "category": "BEST_PRACTICES"
+          "category": "BEST_PRACTICES", 
+          "difficulty": "MEDIUM",
+          "influence_on_penalty": 0 
+        },
+        ...
+      ]
+    }
+  ]
+}
+```
+
+3. Json `old format` (with **&#8209;&#8209;group&#8209;by&#8209;difficulty** argument):
+
+```json
+{
+  "quality": {
+    "EASY": {
+      "code": "BAD",
+      "text": "Code quality (beta): BAD"
+    },
+    "MEDIUM": {
+      "code": "BAD",
+      "text": "Code quality (beta): BAD"
+    },
+    "HARD": {
+      "code": "BAD",
+      "text": "Code quality (beta): BAD"
+    }
+  },
+  "issues": [
+    {
+      "code": "C002",
+      "text": "Too long function. Try to split it into smaller functions / methods.It will make your code easy to understand and less error prone.",
+      "line": "<the code line>",
+      "line_number": 54,
+      "column_number": 0,
+      "category": "FUNC_LEN",
+      "difficulty": "EASY",
+      "influence_on_penalty": {
+        "EASY": 0,
+        "MEDIUM": 0,
+        "HARD": 0
+      }
+    },
+    ...
+  ]
+}
+```
+
+4. Json `new format` (with **&#8209;&#8209;group&#8209;by&#8209;difficulty** argument)
+
+```json
+{
+  "quality": {
+    "EASY": {
+      "code": "BAD",
+      "text": "Code quality (beta): BAD"
+    },
+    "MEDIUM": {
+      "code": "BAD",
+      "text": "Code quality (beta): BAD"
+    },
+    "HARD": {
+      "code": "BAD",
+      "text": "Code quality (beta): BAD"
+    }
+  },
+  "file_review_results": [
+    {
+      "file_name": "<your file>",
+      "quality": {
+        "EASY": {
+          "code": "BAD",
+          "text": "Code quality (beta): BAD"
+        },
+        "MEDIUM": {
+          "code": "BAD",
+          "text": "Code quality (beta): BAD"
+        },
+        "HARD": {
+          "code": "BAD",
+          "text": "Code quality (beta): BAD"
+        }
+      },
+      "issues": [
+        {
+          "code": "W0703",
+          "text": "Catching too general exception Exception",
+          "line": "<the code line>",
+          "line_number": 174,
+          "column_number": 12,
+          "category": "BEST_PRACTICES",
+          "difficulty": "MEDIUM",
+          "influence_on_penalty": {
+            "EASY": 0,
+            "MEDIUM": 0,
+            "HARD": 0
+          }
         },
         ...
       ]
@@ -161,7 +287,7 @@ The output examples:
 }
 ```
 
-3. Text format:
+5. Text format:
 
 ```text
 Review of <path to your file or project> (N violations)

VERSION.md

@@ -1 +1 @@
-1.2.0
+1.2.1

requirements-test.txt

@@ -2,8 +2,3 @@ pytest==6.2.3
 pytest-runner==5.2
 pytest-subtests==0.4.0
 jsonschema==3.2.0
-pandas==1.2.3
-django==3.2
-pylint==2.7.4
-requests==2.25.1
-setuptools==56.0.0

setup.py

@@ -28,10 +28,15 @@ def get_inspectors_additional_files() -> List[str]:
     return result
 
 
+def get_requires() -> List[str]:
+    with open(current_dir / 'requirements.txt') as requirements_file:
+        return requirements_file.read().split('\n')
+
+
 setup(
-    name='review',
+    name='hyperstyle',
     version=get_version(),
-    description='review',
+    description='A tool for running a set of pre-configured linters and evaluating code quality.',
     long_description=get_long_description(),
     long_description_content_type='text/markdown',
     url='https://github.com/hyperskill/hyperstyle',
@@ -47,7 +52,7 @@ def get_inspectors_additional_files() -> List[str]:
     ],
     keywords='code review',
     python_requires='>=3.8, <4',
-    install_requires=['upsourceapi'],
+    install_requires=get_requires(),
     packages=find_packages(exclude=[
         '*.unit_tests',
         '*.unit_tests.*',

src/python/common/tool_arguments.py

@@ -72,3 +72,14 @@ class RunToolArgument(Enum):
     NEW_FORMAT = ArgumentsInfo(None, '--new-format',
                                'The argument determines whether the tool '
                                'should use the new format')
+
+    HISTORY = ArgumentsInfo(None, '--history',
+                            'Json string, which contains lists of issues in the previous submissions '
+                            'for other tasks for one user.')
+
+    WITH_ALL_CATEGORIES = ArgumentsInfo(None, '--with-all-categories',
+                                        'Without this flag, all issues will be categorized into 5 main categories: '
+                                        'CODE_STYLE, BEST_PRACTICES, ERROR_PRONE, COMPLEXITY, INFO.')
+
+    GROUP_BY_DIFFICULTY = ArgumentsInfo(None, '--group-by-difficulty',
+                                        'With this flag, the final grade will be grouped by the issue difficulty.')