Add Python inspections and a script for tool evaluation

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
           # TODO: change max-complexity into 10 after refactoring
-          flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=I201,I202,I101,I100,R504,A003,E800,SC200,SC100,E402 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
+          flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
 
       - name: Set up Eslint
         run: |
@@ -32,3 +32,19 @@ jobs:
       - name: Test with pytest
         run: |
           pytest
+
+      - name: Install review module
+        run: |
+          pip install .
+
+      - name: Check installed module can run python linters
+        run: |
+            review setup.py
+
+      - name: Check installed module can run java linters
+        run: |
+            review test/resources/inspectors/java/test_algorithm_with_scanner.java
+
+      - name: Check installed module can run js linters
+        run: |
+          review test/resources/inspectors/js/case0_no_issues.js

README.md

@@ -33,7 +33,10 @@ Python language:
 - [x]  Pylint [GNU LGPL v2]
     * [Site and docs](https://www.pylint.org/)
     * [Repository](https://github.com/PyCQA/pylint)
-
+
+- [x] Radon [MIT]
+    * [Site and docs](https://radon.readthedocs.io/en/latest/)
+    * [Repository](https://github.com/rubik/radon)
 
 Java language:
 
@@ -92,7 +95,7 @@ Argument | Description
 --- | ---
 **‑h**, **‑‑help**      |  show the help message and exit.
 **‑v**, **‑‑verbosity** |  choose logging level according [this](https://docs.python.org/3/library/logging.html#levels) list: `1` - **ERROR**; `2` - **INFO**; `3` - **DEBUG**; `0` - disable logging (**CRITICAL** value); default value is `0` (**CRITICAL**).
-**‑d**, **‑‑disable**   |  disable inspectors. Available values: for **Python** language: `pylint` for [Pylint](https://github.com/PyCQA/pylint), `flake8` for [flake8](https://flake8.pycqa.org/en/latest/), `python_ast` to check different measures providing by AST; for **Java** language: `checkstyle` for the [Checkstyle](https://checkstyle.sourceforge.io/), `pmd` for [PMD](https://pmd.github.io/); for `Kotlin` language: detekt for [Detekt](https://detekt.github.io/detekt/); for **JavaScript** language: `eslint` for [ESlint](https://eslint.org/). Example: `-d pylint,flake8`.
+**‑d**, **‑‑disable**   |  disable inspectors. Available values: for **Python** language: `pylint` for [Pylint](https://github.com/PyCQA/pylint), `flake8` for [flake8](https://flake8.pycqa.org/en/latest/), `radon` for [Radon](https://radon.readthedocs.io/en/latest/), `python_ast` to check different measures providing by AST; for **Java** language: `checkstyle` for the [Checkstyle](https://checkstyle.sourceforge.io/), `pmd` for [PMD](https://pmd.github.io/); for `Kotlin` language: detekt for [Detekt](https://detekt.github.io/detekt/); for **JavaScript** language: `eslint` for [ESlint](https://eslint.org/). Example: `-d pylint,flake8`.
 **‑‑allow-duplicates**        |  allow duplicate issues found by different linters. By default, duplicates are skipped.
 **‑‑language-version**, **‑‑language_version**  |  specify the language version for JAVA inspectors. Available values: `java7`, `java8`, `java9`, `java11`. **Note**: **‑‑language_version** is deprecated. Will be deleted in the future.
 **‑‑n-cpu**, **‑‑n_cpu**  |  specify number of _cpu_ that can be used to run inspectors. **Note**: **‑‑n_cpu** is deprecated. Will be deleted in the future.
@@ -193,7 +196,7 @@ __Note__: If you have `ModuleNotFoundError` while you try to run tests, please c
 
 __Note__: We use [eslint](https://eslint.org/) and [open-jdk 11](https://openjdk.java.net/projects/jdk/11/)
 in the tests. Please, set up the environment before running the tests. 
-You can see en example of the environment configuration in 
+You can see en example of the environment configuration in
 the [Dockerfile](Dockerfile) file.
 
 Use `pytest` from the root directory to run __ALL__ tests.

VERSION.md

@@ -1 +1 @@
-1.1.1
+1.2.0

requirements-test.txt

@@ -2,4 +2,8 @@ pytest==6.2.3
 pytest-runner==5.2
 pytest-subtests==0.4.0
 jsonschema==3.2.0
-pandas==1.2.3
+pandas==1.2.3
+django==3.2
+pylint==2.7.4
+requests==2.25.1
+setuptools==56.0.0

requirements.txt

@@ -17,6 +17,12 @@ flake8-return==1.1.2
 flake8-spellcheck==0.24.0
 mccabe==0.6.1
 pep8-naming==0.11.1
+wps-light==0.15.2
+flake8-broken-line==0.3.0
+flake8-string-format==0.3.0
+flake8-commas==2.0.0
+cohesion==1.0.0
+radon==4.5.0
 
 # extra libraries and frameworks
 django==3.2

setup.py

@@ -19,13 +19,11 @@ def get_version() -> str:
 
 def get_inspectors_additional_files() -> List[str]:
     inspectors_path = current_dir / 'src' / 'python' / 'review' / 'inspectors'
-
     result = []
     for root, _, files in os.walk(inspectors_path):
         for file in files:
-            file_path = Path(root) / file
-            if not file_path.name.endswith('.py'):
-                result.append(str(file_path))
+            if not file.endswith('.py'):
+                result.append(str(Path(root) / file))
 
     return result
 
@@ -45,22 +43,28 @@ def get_inspectors_additional_files() -> List[str]:
         'Topic :: Software Development :: Build Tools',
         'License :: OSI Approved :: MIT License',
         'Programming Language :: Python :: 3',
-        'Operating System :: OS Independent'
+        'Operating System :: OS Independent',
     ],
     keywords='code review',
     python_requires='>=3.8, <4',
     install_requires=['upsourceapi'],
     packages=find_packages(exclude=[
-        '*.unit_tests', '*.unit_tests.*', 'unit_tests.*', 'unit_tests',
-        '*.functional_tests', '*.functional_tests.*', 'functional_tests.*', 'functional_tests'
+        '*.unit_tests',
+        '*.unit_tests.*',
+        'unit_tests.*',
+        'unit_tests',
+        '*.functional_tests',
+        '*.functional_tests.*',
+        'functional_tests.*',
+        'functional_tests',
     ]),
     zip_safe=False,
     package_data={
-        '': get_inspectors_additional_files()
+        '': get_inspectors_additional_files(),
     },
     entry_points={
         'console_scripts': [
-            'review=src.python.review.run_tool:main'
-        ]
-    }
+            'review=src.python.review.run_tool:main',
+        ],
+    },
 )

src/python/common/tool_arguments.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass
+from enum import Enum, unique
+from typing import List, Optional
+
+from src.python.review.application_config import LanguageVersion
+from src.python.review.inspectors.inspector_type import InspectorType
+
+
+@unique
+class VerbosityLevel(Enum):
+    """
+    Same meaning as the logging level. Should be used in command-line args.
+    """
+    DEBUG = 3
+    INFO = 2
+    ERROR = 1
+    DISABLE = 0
+
+    @classmethod
+    def values(cls) -> List[int]:
+        return [member.value for member in VerbosityLevel]
+
+
+@dataclass(frozen=True)
+class ArgumentsInfo:
+    short_name: Optional[str]
+    long_name: str
+    description: str
+
+
+@unique
+class RunToolArgument(Enum):
+    VERBOSITY = ArgumentsInfo('-v', '--verbosity',
+                              'Choose logging level: '
+                              f'{VerbosityLevel.ERROR.value} - ERROR; '
+                              f'{VerbosityLevel.INFO.value} - INFO; '
+                              f'{VerbosityLevel.DEBUG.value} - DEBUG; '
+                              f'{VerbosityLevel.DISABLE.value} - disable logging; '
+                              'default is 0')
+
+    inspectors = [inspector.lower() for inspector in InspectorType.available_values()]
+    disabled_inspectors_example = f'-d {inspectors[0].lower()},{inspectors[1].lower()}'
+
+    DISABLE = ArgumentsInfo('-d', '--disable',
+                            'Disable inspectors. '
+                            f'Available values: {", ".join(inspectors)}. '
+                            f'Example: {disabled_inspectors_example}')
+
+    DUPLICATES = ArgumentsInfo(None, '--allow-duplicates',
+                               'Allow duplicate issues found by different linters. '
+                               'By default, duplicates are skipped.')
+
+    LANG_VERSION = ArgumentsInfo(None, '--language-version',
+                                 'Specify the language version for JAVA inspectors.'
+                                 'Available values are: '
+                                 f'{LanguageVersion.PYTHON_3.value}, {LanguageVersion.JAVA_8.value}, '
+                                 f'{LanguageVersion.JAVA_11.value}, {LanguageVersion.KOTLIN.value}.')
+
+    CPU = ArgumentsInfo(None, '--n-cpu',
+                        'Specify number of cpu that can be used to run inspectors')
+
+    PATH = ArgumentsInfo(None, 'path', 'Path to file or directory to inspect.')
+
+    FORMAT = ArgumentsInfo('-f', '--format',
+                           'The output format. Default is JSON.')
+
+    START_LINE = ArgumentsInfo('-s', '--start-line',
+                               'The first line to be analyzed. It starts from 1.')
+
+    END_LINE = ArgumentsInfo('-e', '--end-line', 'The end line to be analyzed or None.')
+
+    NEW_FORMAT = ArgumentsInfo(None, '--new-format',
+                               'The argument determines whether the tool '
+                               'should use the new format')

src/python/review/application_config.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
 from enum import Enum, unique
-from typing import Optional, Set, List
+from typing import List, Optional, Set
 
+from src.python.review.common.file_system import Extension
 from src.python.review.inspectors.inspector_type import InspectorType
 
 
@@ -22,7 +23,30 @@ class LanguageVersion(Enum):
     JAVA_8 = 'java8'
     JAVA_9 = 'java9'
     JAVA_11 = 'java11'
+    PYTHON_3 = 'python3'
+    KOTLIN = 'kotlin'
 
     @classmethod
     def values(cls) -> List[str]:
         return [member.value for member in cls.__members__.values()]
+
+    @classmethod
+    def language_to_extension_dict(cls) -> dict:
+        return {cls.PYTHON_3.value: Extension.PY.value,
+                cls.JAVA_7.value: Extension.JAVA.value,
+                cls.JAVA_8.value: Extension.JAVA.value,
+                cls.JAVA_9.value: Extension.JAVA.value,
+                cls.JAVA_11.value: Extension.JAVA.value,
+                cls.KOTLIN.value: Extension.KT.value}
+
+    @classmethod
+    def language_by_extension(cls, lang: str) -> str:
+        return cls.language_to_extension_dict()[lang]
+
+    def is_java(self) -> bool:
+        return (
+            self == LanguageVersion.JAVA_7
+            or self == LanguageVersion.JAVA_8
+            or self == LanguageVersion.JAVA_9
+            or self == LanguageVersion.JAVA_11
+        )

src/python/review/common/file_system.py

@@ -4,7 +4,7 @@
 from contextlib import contextmanager
 from enum import Enum, unique
 from pathlib import Path
-from typing import List, Union, Callable
+from typing import Callable, List, Union
 
 
 @unique
@@ -65,19 +65,16 @@ def new_temp_dir() -> Path:
 def create_file(file_path: Union[str, Path], content: str):
     file_path = Path(file_path)
 
-    create_directory(os.path.dirname(file_path))
-    with open(file_path, 'w') as f:
-        f.write(content)
-
-
-def create_directory(directory: str) -> None:
-    os.makedirs(directory, exist_ok=True)
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    with open(file_path, 'w+') as f:
+        f.writelines(content)
+        yield Path(file_path)
 
 
 def get_file_line(path: Path, line_number: int):
     return linecache.getline(
         str(path),
-        line_number
+        line_number,
     ).strip()
 
 

src/python/review/common/java_compiler.py

@@ -18,7 +18,7 @@ def javac(javac_args: Union[str, Path]) -> bool:
         output_bytes: bytes = subprocess.check_output(
             f'javac {javac_args}',
             shell=True,
-            stderr=subprocess.STDOUT
+            stderr=subprocess.STDOUT,
         )
         output_str = str(output_bytes, Encoding.UTF_ENCODING.value)
 

src/python/review/common/parallel_runner.py

@@ -37,7 +37,7 @@ def inspect_in_parallel(path: Path,
     with multiprocessing.Pool(config.n_cpu) as pool:
         issues = pool.map(
             functools.partial(run_inspector, path, config),
-            inspectors
+            inspectors,
         )
 
     return list(itertools.chain(*issues))

src/python/review/common/subprocess_runner.py

@@ -9,7 +9,7 @@ def run_in_subprocess(command: List[str]) -> str:
     process = subprocess.run(
         command,
         stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE
+        stderr=subprocess.PIPE,
     )
 
     stdout = process.stdout.decode()

src/python/review/inspectors/checkstyle/checkstyle.py

@@ -35,15 +35,15 @@ class CheckstyleInspector(BaseInspector):
             r'Boolean expression complexity is (\d+)',
 
         'LineLengthCheck':
-            r'Line is longer than \d+ characters \(found (\d+)\)'
+            r'Line is longer than \d+ characters \(found (\d+)\)',
     }
 
     @classmethod
     def _create_command(cls, path: Path, output_path: Path) -> List[str]:
         return [
             'java', '-jar', PATH_TOOLS_CHECKSTYLE_JAR,
             '-c', PATH_TOOLS_CHECKSTYLE_CONFIG,
-            '-f', 'xml', '-o', output_path, str(path)
+            '-f', 'xml', '-o', output_path, str(path),
         ]
 
     def inspect(self, path: Path, config: dict) -> List[BaseIssue]:

src/python/review/inspectors/checkstyle/files/config.xml

@@ -44,8 +44,9 @@
         <module name="MissingSwitchDefault"/>
         <module name="ModifiedControlVariable"/>
         <module name="MultipleStringLiterals">
-            <property name="allowedDuplicates" value="2"/>
-            <property name="ignoreStringsRegexp" value="^.{1,5}$"/>
+            <!-- Consistent with AvoidDuplicateLiterals from PMD -->
+            <property name="allowedDuplicates" value="3"/>
+            <property name="ignoreStringsRegexp" value='^".{0,2}"$'/>
         </module>
         <module name="MultipleVariableDeclarations"/>
         <module name="NoFinalizer"/>
@@ -87,16 +88,42 @@
         </module>
         <module name="ConstantName">
             <property name="format" value="^log(ger)?$|^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use UPPER_CASE_SNAKE_CASE (checked pattern ''{1}'')."/>
+        </module>
+        <module name="LambdaParameterName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
+        </module>
+        <module name="LocalFinalVariableName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
+        </module>
+        <module name="LocalVariableName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
+        </module>
+        <module name="MemberName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
+        </module>
+        <module name="MethodName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
         </module>
-        <module name="LambdaParameterName"/>
-        <module name="LocalFinalVariableName"/>
-        <module name="LocalVariableName"/>
-        <module name="MemberName"/>
-        <module name="MethodName"/>
         <module name="PackageName"/>
-        <module name="ParameterName"/>
-        <module name="StaticVariableName"/>
-        <module name="TypeName"/>
+        <module name="ParameterName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
+        </module>
+        <module name="StaticVariableName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use camelCase (checked pattern ''{1}'')."/>
+        </module>
+        <module name="TypeName">
+            <message key="name.invalidPattern"
+                     value="Name ''{0}'' must use PascalCase (checked pattern ''{1}'')."/>
+        </module>
 
         <!-- Checks for Size Violations.                    -->
         <!-- See http://checkstyle.sf.net/config_sizes.html -->