diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e971e1aa..54dd6ade 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,56 +10,34 @@ jobs: container: stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3 steps: - - name: Install git - run: | - apt-get update - apt-get -y install git - - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v1 - name: Install requirements run: | pip install --no-cache-dir -r requirements-test.txt -r requirements.txt - - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules # TODO: change max-complexity into 10 after refactoring - # TODO: remove R504, A003, E800, E402, WPS1, WPS2, WPS3, WPS4, WPS5, WPS6, H601 - flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS1,WPS2,WPS3,WPS4,WPS5,WPS6,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules - - - name: Sort whitelists - run: | - for file in "whitelist.txt" "src/python/review/inspectors/flake8/whitelist.txt" - do - LC_ALL=C sort $file -o $file - done - - - name: Commit sorted whitelists - uses: EndBug/add-and-commit@v7.2.1 - with: - add: "['whitelist.txt', 'src/python/review/inspectors/flake8/whitelist.txt']" - message: 'Sort whitelists (Github Actions)' - + flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules - name: Set up Eslint run: | # Consistent with eslint version in Dockerfile npm install eslint@7.5.0 -g && eslint --init - - name: Test with pytest run: | pytest - + - name: Install review module + run: | + pip install . - name: Check installed module can run python linters run: | - python src/python/review/run_tool.py setup.py - + review setup.py - name: Check installed module can run java linters run: | - python src/python/review/run_tool.py test/resources/inspectors/java/test_algorithm_with_scanner.java - + review test/resources/inspectors/java/test_algorithm_with_scanner.java - name: Check installed module can run js linters run: | - python src/python/review/run_tool.py test/resources/inspectors/js/case0_no_issues.js + review test/resources/inspectors/js/case0_no_issues.js \ No newline at end of file diff --git a/README.md b/README.md index 2310f493..a20f8978 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,6 @@ Simply clone the repository and run the following commands: 1. `pip install -r requirements.txt` 2. `pip install -r requirements-test.txt` for tests -3. `pip install -r requirements-evaluation.txt` for evaluation ## Usage diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile deleted file mode 100644 index 90433662..00000000 --- a/docker/dev/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM python:3.8.2-alpine3.11 - -RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \ - && apk add --update nodejs npm - -RUN npm i -g eslint@7.5.0 - -RUN java -version -RUN ls /usr/lib/jvm - -# Install numpy and pandas for tests -RUN apk add --no-cache python3-dev libstdc++ && \ - apk add --no-cache g++ && \ - ln -s /usr/include/locale.h /usr/include/xlocale.h && \ - pip3 install numpy && \ - pip3 install pandas - -# Other dependencies -RUN apk add bash - -# Dependencies and package installation -WORKDIR / - -COPY requirements-test.txt review/requirements-test.txt -RUN pip3 install --no-cache-dir -r review/requirements-test.txt - -COPY requirements.txt review/requirements.txt -RUN pip3 install --no-cache-dir -r review/requirements.txt - -COPY requirements-evaluation.txt review/requirements-evaluation.txt -RUN pip3 install --no-cache-dir -r review/requirements-evaluation.txt - -COPY . review - -# Container's enviroment variables -ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk -ENV PATH="$JAVA_HOME/bin:${PATH}" - -CMD ["/bin/bash"] \ No newline at end of file diff --git a/requirements-evaluation.txt b/requirements-evaluation.txt deleted file mode 100644 index 046f8026..00000000 --- a/requirements-evaluation.txt +++ /dev/null @@ -1 +0,0 @@ -plotly==4.14.3 diff --git a/requirements-roberta.txt b/requirements-roberta.txt deleted file mode 100644 index 15421cf1..00000000 --- a/requirements-roberta.txt +++ /dev/null @@ -1,6 +0,0 @@ -tqdm==4.49.0 -scikit-learn==0.24.2 -transformers==4.6.1 -tokenizers==0.10.2 -torch==1.8.1 -wandb==0.10.31 \ No newline at end of file diff --git a/setup.py b/setup.py index 8b23d17a..7a119b2e 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,15 @@ def get_inspectors_additional_files() -> List[str]: return result +def get_requires() -> List[str]: + with open(current_dir / 'requirements.txt') as requirements_file: + return requirements_file.read().split('\n') + + setup( - name='review', + name='hyperstyle', version=get_version(), - description='review', + description='A tool for running a set of pre-configured linters and evaluating code quality.', long_description=get_long_description(), long_description_content_type='text/markdown', url='https://github.com/hyperskill/hyperstyle', @@ -47,7 +52,7 @@ def get_inspectors_additional_files() -> List[str]: ], keywords='code review', python_requires='>=3.8, <4', - install_requires=['upsourceapi'], + install_requires=get_requires(), packages=find_packages(exclude=[ '*.unit_tests', '*.unit_tests.*', diff --git a/src/python/common/tool_arguments.py b/src/python/common/tool_arguments.py index 0edb4689..025a1006 100644 --- a/src/python/common/tool_arguments.py +++ b/src/python/common/tool_arguments.py @@ -2,7 +2,6 @@ from enum import Enum, unique from typing import List, Optional -from src.python.evaluation.common.util import ColumnName from src.python.review.application_config import LanguageVersion from src.python.review.inspectors.inspector_type import InspectorType @@ -84,22 +83,3 @@ class RunToolArgument(Enum): GROUP_BY_DIFFICULTY = ArgumentsInfo(None, '--group-by-difficulty', 'With this flag, the final grade will be grouped by the issue difficulty.') - - SOLUTIONS_FILE_PATH = ArgumentsInfo(None, 'solutions_file_path', - 'Local XLSX-file or CSV-file path. ' - 'Your file must include column-names: ' - f'"{ColumnName.CODE.value}" and ' - f'"{ColumnName.LANG.value}". Acceptable values for ' - f'"{ColumnName.LANG.value}" column are: ' - f'{LanguageVersion.PYTHON_3.value}, {LanguageVersion.JAVA_8.value}, ' - f'{LanguageVersion.JAVA_11.value}, {LanguageVersion.KOTLIN.value}.') - - DIFFS_FILE_PATH = ArgumentsInfo(None, 'diffs_file_path', - 'Path to a file with serialized diffs that were founded by diffs_between_df.py') - - QODANA_SOLUTIONS_FILE_PATH = ArgumentsInfo(None, 'solutions_file_path', - 'Csv file with solutions. This file must be graded by Qodana.') - - QODANA_INSPECTIONS_PATH = ArgumentsInfo(None, 'inspections_path', 'Path to a CSV file with inspections list.') - - QODANA_DUPLICATES = ArgumentsInfo(None, '--remove-duplicates', 'Remove duplicates around inspections') diff --git a/src/python/evaluation/README.md b/src/python/evaluation/README.md deleted file mode 100644 index b98e2deb..00000000 --- a/src/python/evaluation/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Hyperstyle evaluation - -This tool allows running the `Hyperstyle` tool on a `xlsx` or `csv` table to get code quality for all code fragments. -Please, note that your input file should consist of at least 2 obligatory columns to run the tool on its code fragments: - -- `code` -- `lang` - -Possible values for column `lang` are: `python3`, `kotlin`, `java8`, `java11`. - -Output file is a new `xlsx` or `csv` file with the all columns from the input file and two additional ones: -- `grade` -- `traceback` (optional) - -Grade assessment is conducted by [`run_tool.py`](https://github.com/hyperskill/hyperstyle/blob/main/README.md) with default arguments. - Avaliable values for column `grade` are: BAD, MODERATE, GOOD, EXCELLENT. - `traceback` column stores full inspectors feedback on each code fragment. - More details on enabling traceback column in **Optional Arguments** table. - -## Usage - -Run the [evaluation_run_tool.py](evaluation_run_tool.py) with the arguments from command line. - -Required arguments: - -`solutions_file_path` — path to xlsx-file or csv-file with code samples to inspect. - -Optional arguments: -Argument | Description ---- | --- -|**‑f**, **‑‑format**| The output format. Available values: `json`, `text`. The default value is `json` . Use this argument when `traceback` is enabled, otherwise it will not be used.| -|**‑tp**, **‑‑tool‑path**| Path to run-tool. Default is `src/python/review/run_tool.py` .| -|**‑‑traceback**| To include a column with errors traceback into an output file. Default is `False`.| -|**‑ofp**, **‑‑output‑folder‑path**| An explicit folder path to store file with results. Default is a parent directory of a folder with xlsx-file or csv-file sent for inspection. | -|**‑ofn**, **‑‑output‑file‑name**| A name of an output file where evaluation results will be stored. Default is `results.xlsx` or `results.csv`.| -|**‑‑to‑drop‑nan**| If True, empty code fragments will be deleted from df. Default is `False`.| diff --git a/src/python/evaluation/__init__.py b/src/python/evaluation/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/common/__init__.py b/src/python/evaluation/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/common/csv_util.py b/src/python/evaluation/common/csv_util.py deleted file mode 100644 index c2956e57..00000000 --- a/src/python/evaluation/common/csv_util.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path -from typing import Union - -import pandas as pd -from src.python.review.common.file_system import Encoding - - -def write_dataframe_to_csv(csv_file_path: Union[str, Path], df: pd.DataFrame) -> None: - # Get error with this encoding=ENCODING on several fragments. So change it then to 'utf8' - try: - df.to_csv(csv_file_path, encoding=Encoding.ISO_ENCODING.value, index=False) - except UnicodeEncodeError: - df.to_csv(csv_file_path, encoding=Encoding.UTF_ENCODING.value, index=False) diff --git a/src/python/evaluation/common/pandas_util.py b/src/python/evaluation/common/pandas_util.py deleted file mode 100644 index 5aadf8ab..00000000 --- a/src/python/evaluation/common/pandas_util.py +++ /dev/null @@ -1,107 +0,0 @@ -import json -import logging -from pathlib import Path -from typing import Any, Iterable, List, Set, Union - -import numpy as np -import pandas as pd -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.common.xlsx_util import create_workbook, remove_sheet, write_dataframe_to_xlsx_sheet -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import Extension, get_restricted_extension -from src.python.review.reviewers.utils.print_review import convert_json_to_issues - -logger = logging.getLogger(__name__) - - -def filter_df_by_language(df: pd.DataFrame, languages: Set[LanguageVersion], - column: str = ColumnName.LANG.value) -> pd.DataFrame: - return filter_df_by_iterable_value(df, column, set(map(lambda l: l.value, languages))) - - -def filter_df_by_iterable_value(df: pd.DataFrame, column: str, value: Iterable) -> pd.DataFrame: - return df.loc[df[column].isin(value)] - - -def filter_df_by_single_value(df: pd.DataFrame, column: str, value: Any) -> pd.DataFrame: - return df.loc[df[column] == value] - - -def drop_duplicates(df: pd.DataFrame, column: str = ColumnName.CODE.value) -> pd.DataFrame: - return df.drop_duplicates(column, keep='last').reset_index(drop=True) - - -# Find all rows and columns where two dataframes are inconsistent. -# For example: -# row | column | -# ------------------------- -# 3 | column_1 | True -# | column_2 | True -# ------------------------- -# 4 | column_1 | True -# | column_2 | True -# means first and second dataframes have different values -# in column_1 and in column_2 in 3-th and 4-th rows -def get_inconsistent_positions(first: pd.DataFrame, second: pd.DataFrame) -> pd.DataFrame: - ne_stacked = (first != second).stack() - inconsistent_positions = ne_stacked[ne_stacked] - inconsistent_positions.index.names = [ColumnName.ROW.value, ColumnName.COLUMN.value] - return inconsistent_positions - - -# Create a new dataframe with all items that are different. -# For example: -# | old | new -# --------------------------------- -# row column | | -# 3 grade | EXCELLENT | MODERATE -# 4 grade | EXCELLENT | BAD -def get_diffs(first: pd.DataFrame, second: pd.DataFrame) -> pd.DataFrame: - changed = get_inconsistent_positions(first, second) - - difference_locations = np.where(first != second) - changed_from = first.values[difference_locations] - changed_to = second.values[difference_locations] - return pd.DataFrame({ - ColumnName.OLD.value: changed_from, - ColumnName.NEW.value: changed_to}, - index=changed.index) - - -def get_solutions_df(ext: Extension, file_path: Union[str, Path]) -> pd.DataFrame: - try: - if ext == Extension.XLSX: - lang_code_dataframe = pd.read_excel(file_path) - else: - lang_code_dataframe = pd.read_csv(file_path) - except FileNotFoundError as e: - logger.error('XLSX-file or CSV-file with the specified name does not exists.') - raise e - - return lang_code_dataframe - - -def get_solutions_df_by_file_path(path: Path) -> pd.DataFrame: - ext = get_restricted_extension(path, [Extension.XLSX, Extension.CSV]) - return get_solutions_df(ext, path) - - -def write_df_to_file(df: pd.DataFrame, output_file_path: Path, extension: Extension) -> None: - if extension == Extension.CSV: - write_dataframe_to_csv(output_file_path, df) - elif extension == Extension.XLSX: - create_workbook(output_file_path) - write_dataframe_to_xlsx_sheet(output_file_path, df, 'inspection_results') - # remove empty sheet that was initially created with the workbook - remove_sheet(output_file_path, 'Sheet') - - -def get_issues_from_json(str_json: str) -> List[PenaltyIssue]: - parsed_json = json.loads(str_json)['issues'] - return convert_json_to_issues(parsed_json) - - -def get_issues_by_row(df: pd.DataFrame, row: int) -> List[PenaltyIssue]: - return get_issues_from_json(df.iloc[row][ColumnName.TRACEBACK.value]) diff --git a/src/python/evaluation/common/util.py b/src/python/evaluation/common/util.py deleted file mode 100644 index cc7cb309..00000000 --- a/src/python/evaluation/common/util.py +++ /dev/null @@ -1,53 +0,0 @@ -from enum import Enum, unique -from typing import Set - -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import Extension - - -@unique -class ColumnName(Enum): - CODE = 'code' - LANG = 'lang' - LANGUAGE = 'language' - GRADE = 'grade' - ID = 'id' - COLUMN = 'column' - ROW = 'row' - OLD = 'old' - NEW = 'new' - IS_PUBLIC = 'is_public' - DECREASED_GRADE = 'decreased_grade' - PENALTY = 'penalty' - USER = 'user' - HISTORY = 'history' - TIME = 'time' - TRACEBACK = 'traceback' - - -@unique -class EvaluationArgument(Enum): - TRACEBACK = 'traceback' - RESULT_FILE_NAME = 'evaluation_results' - RESULT_FILE_NAME_XLSX = f'{RESULT_FILE_NAME}{Extension.XLSX.value}' - RESULT_FILE_NAME_CSV = f'{RESULT_FILE_NAME}{Extension.CSV.value}' - - -script_structure_rule = ('Please, make sure your XLSX-file matches following script standards: \n' - '1. Your XLSX-file or CSV-file should have 2 obligatory columns named:' - f'"{ColumnName.CODE.value}" & "{ColumnName.LANG.value}". \n' - f'"{ColumnName.CODE.value}" column -- relates to the code-sample. \n' - f'"{ColumnName.LANG.value}" column -- relates to the language of a ' - 'particular code-sample. \n' - '2. Your code samples should belong to the one of the supported languages. \n' - 'Supported languages are: Java, Kotlin, Python. \n' - f'3. Check that "{ColumnName.LANG.value}" column cells are filled with ' - 'acceptable language-names: \n' - f'Acceptable language-names are: {LanguageVersion.PYTHON_3.value}, ' - f'{LanguageVersion.JAVA_8.value} ,' - f'{LanguageVersion.JAVA_11.value} and {LanguageVersion.KOTLIN.value}.') - - -# Split string by separator -def parse_set_arg(str_arg: str, separator: str = ',') -> Set[str]: - return set(str_arg.split(separator)) diff --git a/src/python/evaluation/common/xlsx_util.py b/src/python/evaluation/common/xlsx_util.py deleted file mode 100644 index e4a3dcf4..00000000 --- a/src/python/evaluation/common/xlsx_util.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging.config -from pathlib import Path -from typing import Union - -import pandas as pd -from openpyxl import load_workbook, Workbook - -logger = logging.getLogger(__name__) - - -def remove_sheet(workbook_path: Union[str, Path], sheet_name: str, to_raise_error: bool = False) -> None: - try: - workbook = load_workbook(workbook_path) - workbook.remove(workbook[sheet_name]) - workbook.save(workbook_path) - - except KeyError as e: - message = f'Sheet with specified name: {sheet_name} does not exist.' - if to_raise_error: - logger.exception(message) - raise e - else: - logger.info(message) - - -def create_workbook(output_file_path: Path) -> Workbook: - workbook = Workbook() - workbook.save(output_file_path) - return workbook - - -def write_dataframe_to_xlsx_sheet(xlsx_file_path: Union[str, Path], df: pd.DataFrame, sheet_name: str, - mode: str = 'a', to_write_row_names: bool = False) -> None: - """ - mode: str Available values are {'w', 'a'}. File mode to use (write or append). - to_write_row_names: bool Write row names. - """ - - with pd.ExcelWriter(xlsx_file_path, mode=mode) as writer: - df.to_excel(writer, sheet_name=sheet_name, index=to_write_row_names) diff --git a/src/python/evaluation/evaluation_config.py b/src/python/evaluation/evaluation_config.py deleted file mode 100644 index 06672696..00000000 --- a/src/python/evaluation/evaluation_config.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging.config -import os -from argparse import Namespace -from pathlib import Path -from typing import List, Optional, Union - -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.util import EvaluationArgument -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import ( - Extension, - get_parent_folder, - get_restricted_extension, -) - -logger = logging.getLogger(__name__) - - -class EvaluationConfig: - def __init__(self, args: Namespace): - self.tool_path: Union[str, Path] = args.tool_path - self.format: str = args.format - self.solutions_file_path: Union[str, Path] = args.solutions_file_path - self.traceback: bool = args.traceback - self.with_history: bool = args.with_history - self.output_folder_path: Union[str, Path] = args.output_folder_path - self.extension: Extension = get_restricted_extension(self.solutions_file_path, [Extension.XLSX, Extension.CSV]) - self.__init_output_file_name(args.output_file_name) - self.to_drop_nan: bool = args.to_drop_nan - - def __init_output_file_name(self, output_file_name: Optional[str]): - if output_file_name is None: - self.output_file_name = f'{EvaluationArgument.RESULT_FILE_NAME.value}{self.extension.value}' - else: - self.output_file_name = output_file_name - - def build_command(self, inspected_file_path: Union[str, Path], lang: str, history: Optional[str]) -> List[str]: - command = [LanguageVersion.PYTHON_3.value, - self.tool_path, - inspected_file_path, - RunToolArgument.FORMAT.value.short_name, self.format] - - if self.with_history and history is not None: - command.extend([RunToolArgument.HISTORY.value.long_name, history]) - - if lang == LanguageVersion.JAVA_8.value or lang == LanguageVersion.JAVA_11.value: - command.extend([RunToolArgument.LANG_VERSION.value.long_name, lang]) - return command - - def get_output_file_path(self) -> Path: - if self.output_folder_path is None: - try: - self.output_folder_path = get_parent_folder(Path(self.solutions_file_path)) - os.makedirs(self.output_folder_path, exist_ok=True) - except FileNotFoundError as e: - logger.error('XLSX-file or CSV-file with the specified name does not exists.') - raise e - return Path(self.output_folder_path) / self.output_file_name diff --git a/src/python/evaluation/evaluation_run_tool.py b/src/python/evaluation/evaluation_run_tool.py deleted file mode 100644 index 6304bf4f..00000000 --- a/src/python/evaluation/evaluation_run_tool.py +++ /dev/null @@ -1,172 +0,0 @@ -import argparse -import logging.config -import os -import re -import sys -import time -import traceback -from pathlib import Path -from typing import Optional - -sys.path.append('') -sys.path.append('../../..') - -import pandas as pd -from pandarallel import pandarallel -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import get_solutions_df, write_df_to_file -from src.python.evaluation.common.util import ColumnName, EvaluationArgument, script_structure_rule -from src.python.evaluation.evaluation_config import EvaluationConfig -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import create_file -from src.python.review.common.subprocess_runner import run_in_subprocess -from src.python.review.reviewers.perform_review import OutputFormat - -logger = logging.getLogger(__name__) - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.SOLUTIONS_FILE_PATH.value.description) - - parser.add_argument('-tp', '--tool-path', - default=Path(f'{os.path.dirname(os.path.abspath(__file__))}/../review/run_tool.py'), - type=lambda value: Path(value).absolute(), - help='Path to script to run on files.') - - parser.add_argument('--traceback', - help='If True, column with the full inspector feedback will be added ' - 'to the output file with results.', - action='store_true') - - parser.add_argument('-ofp', '--output-folder-path', - help='An absolute path to the folder where file with evaluation results' - 'will be stored.' - 'Default is the path to a directory, where is the folder with xlsx or csv file.', - # if None default path will be specified based on solutions_file_path. - default=None, - type=str) - - parser.add_argument('-ofn', '--output-file-name', - help='Filename for that will be created to store inspection results.' - f'Default is "{EvaluationArgument.RESULT_FILE_NAME.value}" ' - f'with the same extension as the input file has', - default=None, - type=str) - - parser.add_argument(RunToolArgument.FORMAT.value.short_name, - RunToolArgument.FORMAT.value.long_name, - default=OutputFormat.JSON.value, - choices=OutputFormat.values(), - type=str, - help=f'{RunToolArgument.FORMAT.value.description}' - f'Use this argument when {EvaluationArgument.TRACEBACK.value} argument' - 'is enabled argument will not be used otherwise.') - - parser.add_argument('--with-history', - help=f'If True, then history will be taken into account when calculating the grade. ' - f'In that case, for each fragment, the "{ColumnName.HISTORY.value}" column ' - 'must contain the history of previous errors.', - action='store_true') - - parser.add_argument('--to-drop-nan', - help='If True, empty code fragments will be deleted from df', - action='store_true') - - -def get_language_version(lang_key: str) -> LanguageVersion: - try: - return LanguageVersion(lang_key) - except ValueError as e: - logger.error(script_structure_rule) - # We should raise KeyError since it is incorrect value for key in a column - raise KeyError(e) - - -def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str], - config: EvaluationConfig) -> Optional[str]: - print(f'current id: {fragment_id}') - # Tool does not work correctly with tmp files from module on macOS - # thus we create a real file in the file system - extension = get_language_version(lang).extension_by_language().value - tmp_file_path = config.solutions_file_path.parent.absolute() / f'inspected_code_{fragment_id}{extension}' - temp_file = next(create_file(tmp_file_path, code)) - command = config.build_command(temp_file, lang, history) - results = run_in_subprocess(command) - os.remove(temp_file) - return results - - -def __get_grade_from_traceback(traceback: str) -> str: - # this regular expression matches final tool grade: EXCELLENT, GOOD, MODERATE or BAD - return re.match(r'^.*{"code":\s"([A-Z]+)"', traceback).group(1) - - -# TODO: calculate grade after it -def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataFrame) -> pd.DataFrame: - report = pd.DataFrame(columns=lang_code_dataframe.columns) - report[ColumnName.TRACEBACK.value] = [] - - pandarallel.initialize() - if config.traceback: - report[ColumnName.TRACEBACK.value] = [] - try: - if config.to_drop_nan: - lang_code_dataframe = lang_code_dataframe.dropna() - lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply( - lambda row: __inspect_row(row[ColumnName.LANG.value], - row[ColumnName.CODE.value], - row[ColumnName.ID.value], - row.get(ColumnName.HISTORY.value), - config), axis=1) - - lang_code_dataframe[ColumnName.GRADE.value] = lang_code_dataframe.parallel_apply( - lambda row: __get_grade_from_traceback(row[ColumnName.TRACEBACK.value]), axis=1) - - if not config.traceback: - del lang_code_dataframe[ColumnName.TRACEBACK.value] - return lang_code_dataframe - - except ValueError as e: - logger.error(script_structure_rule) - # parallel_apply can raise ValueError but it connected to KeyError: not all columns exist in df - raise KeyError(e) - - except Exception as e: - traceback.print_exc() - logger.exception('An unexpected error.') - raise e - - -def main() -> int: - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - start = time.time() - args = parser.parse_args() - config = EvaluationConfig(args) - lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - results = inspect_solutions_df(config, lang_code_dataframe) - write_df_to_file(results, config.get_output_file_path(), config.extension) - end = time.time() - print(f'All time: {end - start}') - return 0 - - except FileNotFoundError: - logger.error('XLSX-file or CSV-file with the specified name does not exists.') - return 2 - - except KeyError: - logger.error(script_structure_rule) - return 2 - - except Exception: - traceback.print_exc() - logger.exception('An unexpected error.') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/inspectors/README.md b/src/python/evaluation/inspectors/README.md deleted file mode 100644 index 762602e9..00000000 --- a/src/python/evaluation/inspectors/README.md +++ /dev/null @@ -1,312 +0,0 @@ -# Hyperstyle evaluation: inspectors - -This module allows comparing two different versions of `Hyperstyle` tool. -This module contains _preprocessing_ stage and _analysing_ stage. -`Preprocessing` stage includes: -- [filter_solutions.py](filter_solutions.py) script, that allows keeping only necessary languages in - the `csv` or `xslx` file with student solutions and drop duplicates of code fragments (optional); -- [distribute_grades.py](distribute_grades.py) allows distributing calculated grades and traceback - for unique solutions into all solutions. -- [generate_history.py](generate_history.py) allows you to generate history based on issues from previous solutions. - -`Analysing` stage includes: -- [diffs_between_df.py](diffs_between_df.py) allows finding a difference between - old and new grades and collect issues that were found in new data -- [print_inspectors_statistics.py](print_inspectors_statistics.py) allows printing statistics - that were found by [diffs_between_df.py](diffs_between_df.py) -- [get_worse_public_examples.py](get_worse_public_examples.py) allows getting - top N worse public examples from a dataset. The measure is to count unique new inspections. - -___ - -## Preprocessing - -### Filter solutions - -[filter_solutions.py](filter_solutions.py) script allows keeping only necessary languages in - the `csv` or `xslx` file with student solutions and drop duplicates of code fragments (optional). - -Please, note that your input file must meet the requirements to [evaluation](./../evaluation_run_tool.py) tool. -You can find all requirements in the evaluation [README](./../README.md) file. - -Output file is a new `xlsx` or `csv` (the same format with the input file) file with the all columns -from the input file. - -#### Usage - -Run the [filter_solutions.py](filter_solutions.py) with the arguments from command line. - -Required arguments: - -`solutions_file_path` — path to xlsx-file or csv-file with code samples. - -Optional arguments: -Argument | Description ---- | --- -|**‑l**, **‑‑languages**| Set of languages to keep in the dataset. Available values: `java7`, `java8`, `java9` `java11`, `python3`, `kotlin`. The default value is set of all languages.| -|**‑‑duplicates**| If True, drop duplicates in the "code" column. By default is disabled.| - -The resulting file will be stored in the same folder as the input file. - -___ - -### Distribute grades - -[distribute_grades.py](distribute_grades.py) allows distributing calculated grades and traceback - for unique solutions into all solutions. - -Please, note that your input file with all code fragments should consist of at least 1 obligatory columns: - -- `code`. - -Please, note that your input file with unique code fragments should consist of at least 2 obligatory columns: - -- `code`, -- `grade`, -- `traceback` (optional), - -and must have all fragments from the input file with all code fragments. - -Output file is a new `xlsx` or `csv` (the same format with the input files) file with the all columns -from the input file with unique solutions. - -#### Usage - -Run the [distribute_grades.py](distribute_grades.py) with the arguments from command line. - -Required arguments: - -- `solutions_file_path_all` — path to xlsx-file or csv-file with all code samples, -- `solutions_file_path_uniq` — path to xlsx-file or csv-file with unique code samples, - -The resulting file will be stored in the same folder as the input file with all samples. - ----- - -### Generate history - -[generate_history.py](generate_history.py) allows you to generate history based on issues from previous solutions. - -Please, note that your solutions file should consist of at least 4 obligatory columns: - -- `user`, -- `lang`, -- `time`, -- `traceback`. - -You can get such a file with [evaluation_run_tool.py](../evaluation_run_tool.py). - -The output file is a new `xlsx` or `csv` (the same format with the input files) file with all columns from the input -except for `traceback` and `grade` (this behavior can be changed when you run the script). - -#### Usage - -Run the [generate_history.py](generate_history.py) with the arguments from command line. - -Required argument: - -- `solutions_file_path` — path to xlsx-file or csv-file with necessary columns, - -Optional arguments: -Argument | Description ---- | --- -|**‑o**, **‑‑output‑path**| The path where the dataset with history will be saved. If not specified, the dataset will be saved next to the original one. | -|**‑‑to‑drop‑traceback**| The `traceback` column will be removed from the final dataset. Default is false. | -|**‑‑to‑drop‑grades**| The `grade` column will be removed from the final dataset. Default is false.| - -___ - -## Analysing - -### Find diffs - -[diffs_between_df.py](diffs_between_df.py) allows finding a difference between - old and new grades and collect issues that were found in new data. - -Please, note that your input files should consist of at least 3 obligatory columns: - -- `id`, -- `grade`, -- `traceback`. - -Output file is a `pickle` file with serialized dictionary with the result. - - -#### Usage - -Run the [diffs_between_df.py](diffs_between_df.py) with the arguments from command line. - -Required arguments: - -- `solutions_file_path_old` — path to xlsx-file or csv-file with code samples that was graded by the old version of the tool, -- `solutions_file_path_new` — path to xlsx-file or csv-file with code samples that was graded by the new version of the tool. - -The resulting file will be stored in the same folder as the `solutions_file_path_old` input file. - -An example of the pickle` file is: - -```json -{ - grade: [2, 3], - decreased_grade: [1], - user: 2, - traceback: { - 1: { - PenaltyIssue( - origin_class='C0305', - description='Trailing newlines', - line_no=15, - column_no=1, - type=IssueType('CODE_STYLE'), - - file_path=Path(), - inspector_type=InspectorType.UNDEFINED, - influence_on_penalty=0, - ), PenaltyIssue( - origin_class='E211', - description='whitespace before \'(\'', - line_no=1, - column_no=6, - type=IssueType('CODE_STYLE'), - - file_path=Path(), - inspector_type=InspectorType.UNDEFINED, - influence_on_penalty=0.6, - ), - } - }, - penalty: { - 1: { - PenaltyIssue( - origin_class='E211', - description='whitespace before \'(\'', - line_no=1, - column_no=6, - type=IssueType('CODE_STYLE'), - - file_path=Path(), - inspector_type=InspectorType.UNDEFINED, - influence_on_penalty=0.6, - ), - } - } -} -``` -In the `grade` field are stored fragments ids for which grade was increased in the new data. -In the `decreased_grade` field are stored fragments ids for which grade was decreased in the new data. -In the `user` field are stored count unique users in the new dataset. -In the `traceback` field for fragments ids are stored set of issues. These issues were found in the new data and were not found in the old data. -In the `penalty` field for fragments ids are stored set of issues. These issues have not zero `influence_on_penalty` coefficient. - -___ - -### Print statistics - -[print_inspectors_statistics.py](print_inspectors_statistics.py) allows print statistics - that were calculated by [diffs_between_df.py](diffs_between_df.py) - -#### Usage - -Run the [print_inspectors_statistics.py](print_inspectors_statistics.py) with the arguments from command line. - -Required arguments: - -- `diffs_file_path` — path to a `pickle` file, that was calculated by [diffs_between_df.py](diffs_between_df.py). - -Optional arguments: -Argument | Description ---- | --- -|**‑‑categorize**| If True, statistics will be categorized by several categories. By default is disabled.| -|**‑n**, **‑‑top‑n**| The top N items will be printed. Default value is 10.| -|**‑‑full‑stat**| If True, full statistics (with all issues) will be printed. By default is disabled.| - -The statistics will be printed into console. - -The output contains: -- was found incorrect grades or not; -- how many grades have decreased value; -- how many unique users was found in the new dataset; -- for new issues and for penalty statistics: - - how many fragments has additional issues; - - how many unique issues was found; - - top N issues in the format: (issue_key, frequency); - - short categorized statistics: for each category how many issues were found and how many - fragments have these issues; - - \[Optional\] full categorized statistics: for each category for each issue how many - fragments have this issue -- for each category base influence on the penalty statistics: min, max and median values - -An example of the printed statistics (without full categorized statistics): - -```json -SUCCESS! Was not found incorrect grades. -All grades are equal. -______ -NEW INSPECTIONS STATISTICS: -39830 fragments has additional issues -139 unique issues was found -4671 unique users was found! -______ -Top 10 issues: -SC200: 64435 times -WPS432: 17477 times -WPS221: 10618 times -WPS336: 4965 times -H601: 3826 times -SC100: 2719 times -WPS319: 2655 times -WPS317: 2575 times -WPS515: 1783 times -WPS503: 1611 times -______ -CODE_STYLE: 28 issues, 26171 fragments -BEST_PRACTICES: 76 issues, 88040 fragments -ERROR_PRONE: 17 issues, 2363 fragments -COMPLEXITY: 17 issues, 13928 fragments -COHESION: 1 issues, 3826 fragments -______ -______ -PENALTY INSPECTIONS STATISTICS; -Statistics is empty! -______ -______ -INFLUENCE ON PENALTY STATISTICS; -CODE_STYLE issues: min=1, max=100, median=86 -BEST_PRACTICES issues: min=1, max=100, median=98.0 -COMPLEXITY issues: min=1, max=100, median=16.0 -MAINTAINABILITY issues: min=1, max=7, median=2.0 -CYCLOMATIC_COMPLEXITY issues: min=1, max=58, median=11.5 -COHESION issues: min=1, max=100, median=56 -BOOL_EXPR_LEN issues: min=6, max=6, median=6 -______ -``` - ---- - -### Get worse public examples - -[get_worse_public_examples.py](get_worse_public_examples.py) allows getting - top N worse public examples from a dataset. The measure is to count unique new inspections. - -#### Usage - -Run the [get_worse_public_examples.py](get_worse_public_examples.py) with the arguments from command line. - -Required arguments: - -- `solutions_file_path` — path to xlsx-file or csv-file with graded code samples; -- `diffs_file_path` — path to a `pickle` file, that was calculated by [diffs_between_df.py](diffs_between_df.py). - -Please, note that your `solutions_file_path` file with code fragments should consist of at least 2 obligatory columns: - -- `code`, -- `traceback`, -- `is_public`, -- `id`. - -Optional arguments: -Argument | Description ---- | --- -|**‑n**, **‑‑n**| The N worse fragments will be saved.| - -The resulting file will be stored in the same folder as the `solutions_file_path` input file. diff --git a/src/python/evaluation/inspectors/__init__.py b/src/python/evaluation/inspectors/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/inspectors/common/__init__.py b/src/python/evaluation/inspectors/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/inspectors/common/statistics.py b/src/python/evaluation/inspectors/common/statistics.py deleted file mode 100644 index 401a29a6..00000000 --- a/src/python/evaluation/inspectors/common/statistics.py +++ /dev/null @@ -1,108 +0,0 @@ -from collections import defaultdict -from dataclasses import dataclass -from statistics import median -from typing import Dict, List, Tuple - -from src.python.review.inspectors.issue import BaseIssue, IssueType, ShortIssue - - -@dataclass(frozen=True, eq=True) -class PenaltyIssue(BaseIssue): - influence_on_penalty: int - - -@dataclass(frozen=True) -class IssuesStatistics: - stat: Dict[ShortIssue, int] - fragments_in_stat: int - - def print_full_statistics(self, n: int, full_stat: bool, separator: str = '') -> None: - if self.fragments_in_stat == 0: - print('Statistics is empty!') - return - - print(f'{self.fragments_in_stat} fragments has additional issues') - print(f'{self.count_unique_issues()} unique issues was found') - - self.print_top_n(n, separator) - self.print_short_categorized_statistics() - print(separator) - - if full_stat: - self.print_full_inspectors_statistics() - - def print_top_n(self, n: int, separator: str) -> None: - top_n = self.get_top_n_issues(n) - print(separator) - print(f'Top {n} issues:') - for issue, freq in top_n: - IssuesStatistics.print_issue_with_freq(issue, freq) - print(separator) - - def print_full_inspectors_statistics(self, to_categorize: bool = True) -> None: - if to_categorize: - categorized_statistics: Dict[IssueType, Dict[ShortIssue, int]] = self.get_categorized_statistics() - for category, issues in categorized_statistics.items(): - print(f'{category.value} issues:') - self.__print_stat(issues) - else: - self.__print_stat(self.stat) - - @classmethod - def __print_stat(cls, stat: Dict[ShortIssue, int]) -> None: - for issue, freq in stat.items(): - cls.print_issue_with_freq(issue, freq, prefix='- ') - - @classmethod - def print_issue_with_freq(cls, issue: ShortIssue, freq: int, prefix: str = '', suffix: str = '') -> None: - print(f'{prefix}{issue.origin_class}: {freq} times{suffix}') - - def get_categorized_statistics(self) -> Dict[IssueType, Dict[ShortIssue, int]]: - categorized_stat: Dict[IssueType, Dict[ShortIssue, int]] = defaultdict(dict) - for issue, freq in self.stat.items(): - categorized_stat[issue.type][issue] = freq - return categorized_stat - - # Get statistics for each IssueType: count unique issues, count fragments with these issues - def get_short_categorized_statistics(self) -> Dict[IssueType, Tuple[int, int]]: - categorized_statistics: Dict[IssueType, Dict[ShortIssue, int]] = self.get_categorized_statistics() - short_categorized_statistics = defaultdict(tuple) - for issue_type, stat in categorized_statistics.items(): - unique_issues = len(stat) - fragments = sum(stat.values()) - short_categorized_statistics[issue_type] = (unique_issues, fragments) - return short_categorized_statistics - - def print_short_categorized_statistics(self) -> None: - short_categorized_statistics = self.get_short_categorized_statistics() - for category, stat in short_categorized_statistics.items(): - print(f'{category.value}: {stat[0]} issues, {stat[1]} fragments') - - def get_top_n_issues(self, n: int) -> List[ShortIssue]: - return sorted(self.stat.items(), key=lambda t: t[1], reverse=True)[:n] - - def count_unique_issues(self) -> int: - return len(self.stat) - - -# Store list of penalty influences for each category -@dataclass -class PenaltyInfluenceStatistics: - stat: Dict[IssueType, List[float]] - - def __init__(self, issues_stat_dict: Dict[int, List[PenaltyIssue]]): - self.stat = defaultdict(list) - for _, issues in issues_stat_dict.items(): - for issue in issues: - self.stat[issue.type].append(issue.influence_on_penalty) - - def print_stat(self): - for category, issues in self.stat.items(): - print(f'{category.value} issues: min={min(issues)}, max={max(issues)}, median={median(issues)}') - - -@dataclass(frozen=True) -class GeneralInspectorsStatistics: - new_issues_stat: IssuesStatistics - penalty_issues_stat: IssuesStatistics - penalty_influence_stat: PenaltyInfluenceStatistics diff --git a/src/python/evaluation/inspectors/diffs_between_df.py b/src/python/evaluation/inspectors/diffs_between_df.py deleted file mode 100644 index 5556484f..00000000 --- a/src/python/evaluation/inspectors/diffs_between_df.py +++ /dev/null @@ -1,109 +0,0 @@ -import argparse -from pathlib import Path - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import ( - get_inconsistent_positions, get_issues_by_row, get_solutions_df, get_solutions_df_by_file_path, -) -from src.python.evaluation.common.util import ColumnName -from src.python.review.common.file_system import ( - Extension, get_parent_folder, get_restricted_extension, serialize_data_and_write_to_file, -) -from src.python.review.quality.model import QualityType - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name}_old', - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be graded ' - f'(file contains grade and traceback (optional) columns)') - - parser.add_argument(f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name}_new', - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be graded ' - f'(file contains grade and traceback (optional) columns)') - - -# Find difference between two dataframes. Return dict: -# { -# grade: [list_of_fragment_ids], -# decreased_grade: [list_of_fragment_ids], -# user: count_unique_users, -# traceback: { -# fragment_id: [list of issues] -# }, -# penalty: { -# fragment_id: [list of issues] -# }, -# } -# The key contains only fragments that increase quality in new df -# The key contains only fragments that decrease quality in new df -# The key count number of unique users in the new dataset -# The key contains list of new issues for each fragment -# The key contains list of issues with not zero influence_on_penalty coefficient -def find_diffs(old_df: pd.DataFrame, new_df: pd.DataFrame) -> dict: - if ColumnName.HISTORY.value in new_df.columns: - del new_df[ColumnName.HISTORY.value] - new_df = new_df.reindex(columns=old_df.columns) - inconsistent_positions = get_inconsistent_positions(old_df, new_df) - diffs = { - ColumnName.GRADE.value: [], - ColumnName.DECREASED_GRADE.value: [], - ColumnName.TRACEBACK.value: {}, - ColumnName.PENALTY.value: {}, - } - if ColumnName.USER.value in new_df.columns: - diffs[ColumnName.USER.value] = len(new_df[ColumnName.USER.value].unique()) - else: - diffs[ColumnName.USER.value] = 0 - # Keep only diffs in the TRACEBACK column - for row, _ in filter(lambda t: t[1] == ColumnName.TRACEBACK.value, inconsistent_positions.index): - old_value = old_df.iloc[row][ColumnName.GRADE.value] - new_value = new_df.iloc[row][ColumnName.GRADE.value] - old_quality = QualityType(old_value).to_number() - new_quality = QualityType(new_value).to_number() - fragment_id = old_df.iloc[row][ColumnName.ID.value] - if new_quality > old_quality: - # It is an unexpected keys, we should check the algorithm - diffs[ColumnName.GRADE.value].append(fragment_id) - else: - if new_quality < old_quality: - diffs[ColumnName.DECREASED_GRADE.value].append(fragment_id) - old_issues = get_issues_by_row(old_df, row) - new_issues = get_issues_by_row(new_df, row) - # Find difference between issues - if len(old_issues) > len(new_issues): - raise ValueError(f'New dataframe contains less issues than old for fragment {id}') - difference = set(set(new_issues) - set(old_issues)) - if len(difference) > 0: - diffs[ColumnName.TRACEBACK.value][fragment_id] = difference - - # Find issues with influence_in_penalty > 0 - penalty = set(filter(lambda i: i.influence_on_penalty > 0, new_issues)) - if len(penalty) > 0: - diffs[ColumnName.PENALTY.value][fragment_id] = penalty - return diffs - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - old_solutions_file_path = args.solutions_file_path_old - output_ext = get_restricted_extension(old_solutions_file_path, [Extension.XLSX, Extension.CSV]) - old_solutions_df = get_solutions_df(output_ext, old_solutions_file_path) - - new_solutions_file_path = args.solutions_file_path_new - new_solutions_df = get_solutions_df_by_file_path(new_solutions_file_path) - - diffs = find_diffs(old_solutions_df, new_solutions_df) - output_path = get_parent_folder(Path(old_solutions_file_path)) / f'diffs{Extension.PICKLE.value}' - serialize_data_and_write_to_file(output_path, diffs) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/distribute_grades.py b/src/python/evaluation/inspectors/distribute_grades.py deleted file mode 100644 index 0518b9a1..00000000 --- a/src/python/evaluation/inspectors/distribute_grades.py +++ /dev/null @@ -1,66 +0,0 @@ -import argparse -from pathlib import Path -from typing import Dict, Optional, Tuple - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import get_solutions_df, get_solutions_df_by_file_path, write_df_to_file -from src.python.evaluation.common.util import ColumnName -from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension - -CodeToGradesDict = Dict[str, Tuple[str, Optional[str]]] - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name}_all', - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be in the uniq file') - - parser.add_argument(f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name}_uniq', - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be graded ' - f'(file contains grade and traceback (optional) columns)') - - -def __add_grade(code_to_grades_dict: CodeToGradesDict, code: str, grade: str, traceback: Optional[str]) -> None: - code_to_grades_dict[code] = (grade, traceback) - - -# Return a dictionary that contains code fragments -# with their grades and traceback (optional, can be None) -def get_code_to_grades_dict(df: pd.DataFrame) -> CodeToGradesDict: - code_to_grades_dict: CodeToGradesDict = {} - df.apply(lambda row: __add_grade(code_to_grades_dict, - row[ColumnName.CODE.value], - row[ColumnName.GRADE.value], - row[ColumnName.TRACEBACK.value]), axis=1) - return code_to_grades_dict - - -def fill_all_solutions_df(all_solutions_df: pd.DataFrame, code_to_grades_dict: CodeToGradesDict) -> pd.DataFrame: - all_solutions_df[ColumnName.GRADE.value], all_solutions_df[ColumnName.TRACEBACK.value] = zip( - *all_solutions_df[ColumnName.CODE.value].map(lambda code: code_to_grades_dict[code])) - return all_solutions_df - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - all_solutions_file_path = args.solutions_file_path_all - output_ext = get_restricted_extension(all_solutions_file_path, [Extension.XLSX, Extension.CSV]) - all_solutions_df = get_solutions_df(output_ext, all_solutions_file_path) - uniq_solutions_df = get_solutions_df_by_file_path(args.solutions_file_path_uniq) - - code_to_grades_dict = get_code_to_grades_dict(uniq_solutions_df) - all_solutions_df = fill_all_solutions_df(all_solutions_df, code_to_grades_dict) - - output_path = get_parent_folder(Path(all_solutions_file_path)) - write_df_to_file(all_solutions_df, output_path / f'evaluation_result_all{output_ext.value}', output_ext) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/filter_issues.py b/src/python/evaluation/inspectors/filter_issues.py deleted file mode 100644 index 60276f20..00000000 --- a/src/python/evaluation/inspectors/filter_issues.py +++ /dev/null @@ -1,67 +0,0 @@ -import argparse -from pathlib import Path -from typing import List, Set - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import get_issues_from_json, get_solutions_df_by_file_path -from src.python.evaluation.common.util import ColumnName, parse_set_arg -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.review.common.file_system import Extension, get_parent_folder, serialize_data_and_write_to_file -from src.python.review.inspectors.issue import BaseIssue - - -TRACEBACK = ColumnName.TRACEBACK.value -ID = ColumnName.ID.value -GRADE = ColumnName.GRADE.value - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be graded ') - - parser.add_argument('-i', '--issues', - help='Set of issues', - default='') - - -def __get_new_issues(traceback: str, new_issues_classes: Set[str]) -> List[PenaltyIssue]: - all_issues = get_issues_from_json(traceback) - return list(filter(lambda i: i.origin_class in new_issues_classes, all_issues)) - - -def __add_issues_for_fragment(fragment_id: int, new_issues: List[BaseIssue], diffs: dict) -> None: - if len(new_issues) > 0: - diffs[TRACEBACK][fragment_id] = new_issues - - -# Make a dict with the same structure as in the find_diffs function from diffs_between_df.py -def get_statistics_dict(solutions_df: pd.DataFrame, new_issues_classes: Set[str]) -> dict: - diffs = { - GRADE: [], - TRACEBACK: {}, - } - solutions_df.apply(lambda row: __add_issues_for_fragment(row[ID], - __get_new_issues(row[TRACEBACK], new_issues_classes), - diffs), axis=1) - return diffs - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - solutions_file_path = args.solutions_file_path - solutions_df = get_solutions_df_by_file_path(solutions_file_path) - issues = parse_set_arg(args.issues) - - diffs = get_statistics_dict(solutions_df, issues) - output_path = get_parent_folder(Path(solutions_file_path)) / f'diffs{Extension.PICKLE.value}' - serialize_data_and_write_to_file(output_path, diffs) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/filter_solutions.py b/src/python/evaluation/inspectors/filter_solutions.py deleted file mode 100644 index 99d3ac89..00000000 --- a/src/python/evaluation/inspectors/filter_solutions.py +++ /dev/null @@ -1,60 +0,0 @@ -import argparse -import logging -from pathlib import Path -from typing import Set - -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import ( - drop_duplicates, - filter_df_by_language, - get_solutions_df, - write_df_to_file, -) -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension - -logger = logging.getLogger(__name__) - - -def parse_languages(value: str) -> Set[LanguageVersion]: - passed_names = value.lower().split(',') - allowed_names = {lang.value for lang in LanguageVersion} - if not all(name in allowed_names for name in passed_names): - raise argparse.ArgumentError('--languages', 'Incorrect --languages\' names') - - return {LanguageVersion(name) for name in passed_names} - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.SOLUTIONS_FILE_PATH.value.description) - - parser.add_argument('-l', '--languages', - help='Set of languages to keep in the dataset', - type=parse_languages, - default=set(LanguageVersion)) - - parser.add_argument('--duplicates', - help='If True, drop duplicates in the "code" column.', - action='store_true') - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - solutions_file_path = args.solutions_file_path - ext = get_restricted_extension(solutions_file_path, [Extension.XLSX, Extension.CSV]) - solutions_df = get_solutions_df(ext, solutions_file_path) - - filtered_df = filter_df_by_language(solutions_df, args.languages) - if args.duplicates: - filtered_df = drop_duplicates(filtered_df) - output_path = get_parent_folder(Path(solutions_file_path)) - write_df_to_file(filtered_df, output_path / f'filtered_solutions{ext.value}', ext) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/generate_history.py b/src/python/evaluation/inspectors/generate_history.py deleted file mode 100644 index 2ff95c34..00000000 --- a/src/python/evaluation/inspectors/generate_history.py +++ /dev/null @@ -1,131 +0,0 @@ -import argparse -import json -from collections import Counter -from pathlib import Path - -import pandas as pd -from pandarallel import pandarallel -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import ( - get_issues_from_json, - get_solutions_df_by_file_path, - write_df_to_file, -) -from src.python.evaluation.common.util import ColumnName, EvaluationArgument -from src.python.evaluation.evaluation_run_tool import get_language_version -from src.python.review.common.file_system import ( - Extension, - get_name_from_path, - get_parent_folder, - get_restricted_extension, -) -from src.python.review.common.language import Language - -TRACEBACK = EvaluationArgument.TRACEBACK.value -GRADE = ColumnName.GRADE.value -HISTORY = ColumnName.HISTORY.value -USER = ColumnName.USER.value -LANG = ColumnName.LANG.value -TIME = ColumnName.TIME.value -EXTRACTED_ISSUES = 'extracted_issues' - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=f'Path to csv or xlsx file. Your dataset must include column-names: ' - f'"{USER}", "{LANG}", "{TIME}, "{TRACEBACK}".', - ) - - parser.add_argument( - '-o', '--output-path', - type=lambda value: Path(value).absolute(), - help='The path where the dataset with history will be saved. ' - 'If not specified, the dataset will be saved next to the original one.', - ) - - parser.add_argument( - '--to-drop-traceback', - help=f'The "{TRACEBACK}" column will be removed from the final dataset.', - action='store_true', - ) - - parser.add_argument( - '--to-drop-grade', - help=f'The "{GRADE}" column will be removed from the final dataset.', - action='store_true', - ) - - -def _update_counter(extracted_issues: str, counter: Counter) -> None: - issue_classes = [] - if extracted_issues: - issue_classes = extracted_issues.split(',') - - counter.update(issue_classes) - - -def _add_history(row, solutions_df: pd.DataFrame) -> str: - counter = Counter() - - filtered_df = solutions_df[ - (solutions_df[USER] == row[USER]) & (solutions_df[LANG] == row[LANG]) & (solutions_df[TIME] < row[TIME]) - ] - filtered_df.apply(lambda row: _update_counter(row[EXTRACTED_ISSUES], counter), axis=1) - - history = {} - - # If we were unable to identify the language version, we return an empty history - try: - lang_version = get_language_version(row[LANG]) - except KeyError: - return json.dumps(history) - - lang = Language.from_language_version(lang_version) - if len(counter) != 0: - history = {lang.value.lower(): [{'origin_class': key, 'number': value} for key, value in counter.items()]} - - return json.dumps(history) - - -def _extract_issues(traceback: str) -> str: - issues = get_issues_from_json(traceback) - issue_classes = [issue.origin_class for issue in issues] - return ','.join(issue_classes) - - -def main(): - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - pandarallel.initialize() - - solutions_file_path = args.solutions_file_path - solutions_df = get_solutions_df_by_file_path(solutions_file_path) - solutions_df[EXTRACTED_ISSUES] = solutions_df.parallel_apply(lambda row: _extract_issues(row[TRACEBACK]), axis=1) - solutions_df[HISTORY] = solutions_df.parallel_apply(_add_history, axis=1, args=(solutions_df,)) - - columns_to_drop = [EXTRACTED_ISSUES] - - if args.to_drop_grade: - columns_to_drop.append(GRADE) - - if args.to_drop_traceback: - columns_to_drop.append(TRACEBACK) - - solutions_df.drop(columns=columns_to_drop, inplace=True, errors='ignore') - - output_path = args.output_path - if output_path is None: - output_dir = get_parent_folder(solutions_file_path) - dataset_name = get_name_from_path(solutions_file_path, with_extension=False) - output_path = output_dir / f'{dataset_name}_with_history{Extension.CSV.value}' - - output_ext = get_restricted_extension(solutions_file_path, [Extension.XLSX, Extension.CSV]) - write_df_to_file(solutions_df, output_path, output_ext) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/get_worse_public_examples.py b/src/python/evaluation/inspectors/get_worse_public_examples.py deleted file mode 100644 index 980c8a9e..00000000 --- a/src/python/evaluation/inspectors/get_worse_public_examples.py +++ /dev/null @@ -1,68 +0,0 @@ -import argparse -from pathlib import Path -from typing import Dict, List - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import filter_df_by_single_value, get_solutions_df_by_file_path -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.review.common.file_system import deserialize_data_from_file, Extension, get_parent_folder - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.SOLUTIONS_FILE_PATH.value.description) - - parser.add_argument(RunToolArgument.DIFFS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.DIFFS_FILE_PATH.value.description) - - parser.add_argument('-n', '--n', - help='The N worse fragments will be saved', - type=int, - default=10) - - -def __get_new_inspections(fragment_id_to_issues: Dict[int, List[PenaltyIssue]], fragment_id: int) -> str: - return ','.join(set(map(lambda i: i.origin_class, fragment_id_to_issues.get(fragment_id, [])))) - - -def __get_public_fragments(solutions_df: pd.DataFrame, diffs_dict: dict) -> pd.DataFrame: - # Keep only public solutions - public_fragments = filter_df_by_single_value(solutions_df, ColumnName.IS_PUBLIC.value, 'YES') - count_inspections_column = 'count_inspections' - new_inspections_column = 'new_inspections' - - # Get only new inspections and count them - fragment_id_to_issues = diffs_dict[ColumnName.TRACEBACK.value] - public_fragments[new_inspections_column] = public_fragments.apply( - lambda row: __get_new_inspections(fragment_id_to_issues, row[ColumnName.ID.value]), axis=1) - public_fragments[count_inspections_column] = public_fragments.apply( - lambda row: len(row[new_inspections_column].split(',')), axis=1) - - public_fragments = public_fragments.sort_values(count_inspections_column, ascending=False) - # Keep only public columns - return public_fragments[[ColumnName.CODE.value, ColumnName.TRACEBACK.value, new_inspections_column]] - - -# TODO: add readme -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - solutions_file_path = args.solutions_file_path - solutions_df = get_solutions_df_by_file_path(solutions_file_path) - diffs = deserialize_data_from_file(args.diffs_file_path) - - public_fragments = __get_public_fragments(solutions_df, diffs) - - output_path = get_parent_folder(Path(solutions_file_path)) / f'worse_fragments{Extension.CSV.value}' - write_dataframe_to_csv(output_path, public_fragments.head(args.n)) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/inspectors_stat/README.md b/src/python/evaluation/inspectors/inspectors_stat/README.md deleted file mode 100644 index f8a4ad94..00000000 --- a/src/python/evaluation/inspectors/inspectors_stat/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Hyperstyle evaluation: inspectors statistics gathering - -This module allows gathering statistics about inspections that are used -during analysis for a specific language. We collect all available issues' keys, -removed ignored ones and gather statistics for fours main categories: - -- code style issues; -- best practice issues; -- error-prone issues; -- code complexity issues. - -More information about these categories can be found on [this](https://support.hyperskill.org/hc/en-us/articles/360049582712-Code-style-Code-quality) page. - -## Current statistics - -The current statistics is: - -| | Error prone | Code style | Code complexity | Best practice | -|------------|:-----------:|:----------:|:---------------:|:-------------:| -| Python | 162 | 146 | 35 | 254 | -| Java | 51 | 50 | 8 | 110 | -| JavaScript | 15 | 17 | 1 | 34 | -| Kotlin | 21 | 70 | 12 | 75 | - - -## Usage - -Run the [statistics_gathering.py](statistics_gathering.py) with the arguments from command line. - -Required arguments: - -`language` — the language for which statistics will be gathering. -Available values are: `python`, `java`, `kotlin`, `javascript`. - -An example of the output is: - -```text -Collected statistics for python language: -best practices: 254 times; -code style: 146 times; -complexity: 35 times; -error prone: 162 times; -undefined: 3 times; -Note: undefined means a category that is not categorized among the four main categories. Most likely it is info category -``` diff --git a/src/python/evaluation/inspectors/inspectors_stat/__init__.py b/src/python/evaluation/inspectors/inspectors_stat/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/inspectors/inspectors_stat/issues/__init__.py b/src/python/evaluation/inspectors/inspectors_stat/issues/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/inspectors/inspectors_stat/issues/flake8_all_issues.py b/src/python/evaluation/inspectors/inspectors_stat/issues/flake8_all_issues.py deleted file mode 100644 index eecc92dc..00000000 --- a/src/python/evaluation/inspectors/inspectors_stat/issues/flake8_all_issues.py +++ /dev/null @@ -1,601 +0,0 @@ -# According to https://gist.github.com/sharkykh/c76c80feadc8f33b129d846999210ba3 -ALL_STANDARD_ISSUES = { - # Indentation - 'E101': 'indentation contains mixed spaces and tabs', - 'E111': 'indentation is not a multiple of four', - 'E112': 'expected an indented block', - 'E113': 'unexpected indentation', - 'E114': 'indentation is not a multiple of four (comment)', - 'E115': 'expected an indented block (comment)', - 'E116': 'unexpected indentation (comment)', - 'E121': 'continuation line under-indented for hanging indent', - 'E122': 'continuation line missing indentation or outdented', - 'E123': 'closing bracket does not match indentation of opening bracket\'s line', - 'E124': 'closing bracket does not match visual indentation', - 'E125': 'continuation line with same indent as next logical line', - 'E126': 'continuation line over-indented for hanging indent', - 'E127': 'continuation line over-indented for visual indent', - 'E128': 'continuation line under-indented for visual indent', - 'E129': 'visually indented line with same indent as next logical line', - 'E131': 'continuation line unaligned for hanging indent', - 'E133': 'closing bracket is missing indentation', - - # Whitespace - 'E201': 'whitespace after \'(\'', - 'E202': 'whitespace before \')\'', - 'E203': 'whitespace before \':\'', - 'E211': 'whitespace before \'(\'', - 'E221': 'multiple spaces before operator', - 'E222': 'multiple spaces after operator', - 'E223': 'tab before operator', - 'E224': 'tab after operator', - 'E225': 'missing whitespace around operator', - 'E226': 'missing whitespace around arithmetic operator', - 'E227': 'missing whitespace around bitwise or shift operator', - 'E228': 'missing whitespace around modulo operator', - 'E231': 'missing whitespace after \',\', \';\', or \':\'', - 'E241': 'multiple spaces after \',\'', - 'E242': 'tab after \',\'', - 'E251': 'unexpected spaces around keyword / parameter equals', - 'E261': 'at least two spaces before inline comment', - 'E262': 'inline comment should start with \'# \'', - 'E265': 'block comment should start with \'# \'', - 'E266': 'too many leading \'#\' for block comment', - 'E271': 'multiple spaces after keyword', - 'E272': 'multiple spaces before keyword', - 'E273': 'tab after keyword', - 'E274': 'tab before keyword', - 'E275': 'missing whitespace after keyword', - - # Blank line - 'E301': 'expected 1 blank line, found 0', - 'E302': 'expected 2 blank lines, found 0', - 'E303': 'too many blank lines (3)', - 'E304': 'blank lines found after function decorator', - 'E305': 'expected 2 blank lines after end of function or class', - 'E306': 'expected 1 blank line before a nested definition', - - # Import - 'E401': 'multiple imports on one line', - 'E402': 'module level import not at top of file', - - # Line length - 'E501': 'line too long (82 > 79 characters)', - 'E502': 'the backslash is redundant between brackets', - - # Statement - 'E701': 'multiple statements on one line (colon)', - 'E702': 'multiple statements on one line (semicolon)', - 'E703': 'statement ends with a semicolon', - 'E704': 'multiple statements on one line (def)', - 'E711': 'comparison to None should be \'if cond is None:\'', - 'E712': 'comparison to True should be \'if cond is True:\' or \'if cond:\'', - 'E713': 'test for membership should be \'not in\'', - 'E714': 'test for object identity should be \'is not\'', - 'E721': 'do not compare types, use \'isinstance()\'', - 'E722': 'do not use bare except, specify exception instead', - 'E731': 'do not use variables named \'l\', \'O\', or \'I\'', - 'E741': 'do not use variables named \'l\', \'O\', or \'I\'', - 'E742': 'do not define classes named \'l\', \'O\', or \'I\'', - 'E743': 'do not define functions named \'l\', \'O\', or \'I\'', - - # Runtime - 'E901': 'SyntaxError or IndentationError', - 'E902': 'IOError', - - # Indentation warning - 'W191': 'indentation contains tabs', - - # Whitespace warning - 'W291': 'trailing whitespace', - 'W292': 'no newline at end of file', - 'W293': 'blank line contains whitespace', - - # Blank line warning - 'W391': 'blank line at end of file', - - # Line break warning - 'W503': 'line break before binary operator', - 'W504': 'line break after binary operator', - 'W505': 'doc line too long (82 > 79 characters)', - - # Deprecation warning - 'W601': '.has_key() is deprecated, use \'in\'', - 'W602': 'deprecated form of raising exception', - 'W603': '\'<>\' is deprecated, use \'!=\'', - 'W604': 'backticks are deprecated, use \'repr()\'', - 'W605': 'invalid escape sequence \'x\'', - 'W606': '\'async\' and \'await\' are reserved keywords starting with Python 3.7', - - 'F401': 'module imported but unused', - 'F402': 'import module from line N shadowed by loop variable', - 'F403': '\'from module import *\' used; unable to detect undefined names', - 'F404': 'future import(s) name after other statements', - 'F405': 'name may be undefined, or defined from star imports: module', - 'F406': '\'from module import *\' only allowed at module level', - 'F407': 'an undefined __future__ feature name was imported', - - 'F601': 'dictionary key name repeated with different values', - 'F602': 'dictionary key variable name repeated with different values', - 'F621': 'too many expressions in an assignment with star-unpacking', - 'F622': 'two or more starred expressions in an assignment (a, *b, *c = d)', - 'F631': 'assertion test is a tuple, which are always True', - - 'F701': 'a break statement outside of a while or for loop', - 'F702': 'a continue statement outside of a while or for loop', - 'F703': 'a continue statement in a finally block in a loop', - 'F704': 'a yield or yield from statement outside of a function', - 'F705': 'a return statement with arguments inside a generator', - 'F706': 'a return statement outside of a function/method', - 'F707': 'an except: block as not the last exception handler', - 'F721': 'doctest syntax error', - 'F722': 'syntax error in forward type annotation', - - 'F811': 'redefinition of unused name from line N', - 'F812': 'list comprehension redefines name from line N', - 'F821': 'undefined name name', - 'F822': 'undefined name name in __all__', - 'F823': 'local variable name ... referenced before assignment', - 'F831': 'duplicate argument name in function definition', - 'F841': 'local variable name is assigned to but never used', - - 'F901': 'raise NotImplemented should be raise NotImplementedError', - - 'N801': 'class names should use CapWords convention', - 'N802': 'function name should be lowercase', - 'N803': 'argument name should be lowercase', - 'N804': 'first argument of a classmethod should be named \'cls\'', - 'N805': 'first argument of a method should be named \'self\'', - 'N806': 'variable in function should be lowercase', - 'N807': 'function name should not start or end with \'__\'', - 'N811': 'constant imported as non constant', - 'N812': 'lowercase imported as non lowercase', - 'N813': 'camelcase imported as lowercase', - 'N814': 'camelcase imported as constant', - 'N815': 'mixedCase variable in class scope', - 'N816': 'mixedCase variable in global scope', -} - -# According to https://pypi.org/project/flake8-bugbear/ -ALL_BUGBEAR_ISSUES = { - 'B001': 'Do not use bare except:, it also catches unexpected events like memory errors, interrupts, system exit, ' - 'and so on. Prefer except Exception:. If you’re sure what you’re doing, be explicit and write except ' - 'BaseException:. Disable E722 to avoid duplicate warnings.', - 'B002': 'Python does not support the unary prefix increment. Writing ++n is equivalent to +(+(n)), which equals ' - 'n. You meant n += 1.', - 'B003': 'Assigning to os.environ doesn’t clear the environment. Subprocesses are going to see outdated variables, ' - 'in disagreement with the current process. Use os.environ.clear() or the env= argument to Popen.', - 'B004': 'Using hasattr(x, \'__call__\') to test if x is callable is unreliable. If x implements custom ' - '__getattr__ or its __call__ is itself not callable, you might get misleading results. Use callable(x) ' - 'for consistent results.', - 'B005': 'Using .strip() with multi-character strings is misleading the reader. It looks like stripping a ' - 'substring. Move your character set to a constant if this is deliberate. Use .replace() or regular ' - 'expressions to remove string fragments.', - 'B006': 'Do not use mutable data structures for argument defaults. They are created during function definition ' - 'time. All calls to the function reuse this one instance of that data structure, persisting changes ' - 'between them.', - 'B007': 'Loop control variable not used within the loop body. If this is intended, start the name with an ' - 'underscore.', - 'B008': 'Do not perform function calls in argument defaults. The call is performed only once at function ' - 'definition time. All calls to your function will reuse the result of that definition-time function call. ' - 'If this is intended, assign the function call to a module-level variable and use that variable as a ' - 'default value.', - 'B009': 'Do not call getattr(x, \'attr\'), instead use normal property access: x.attr. Missing a default to ' - 'getattr will cause an AttributeError to be raised for non-existent properties. There is no additional ' - 'safety in using getattr if you know the attribute name ahead of time.', - 'B010': 'Do not call setattr(x, \'attr\', val), instead use normal property access: x.attr = val. There is no ' - 'additional safety in using setattr if you know the attribute name ahead of time.', - 'B011': 'Do not call assert False since python -O removes these calls. Instead callers should raise ' - 'AssertionError().', - 'B012': 'Use of break, continue or return inside finally blocks will silence exceptions or override return values ' - 'from the try or except blocks. To silence an exception, do it explicitly in the except block. To ' - 'properly use a break, continue or return refactor your code so these statements are not in the finally ' - 'block.', - 'B013': 'A length-one tuple literal is redundant. Write except SomeError: instead of except (SomeError,):.', - 'B014': 'Redundant exception types in except (Exception, TypeError):. Write except Exception:, which catches ' - 'exactly the same exceptions.', - 'B015': 'Pointless comparison. This comparison does nothing but waste CPU instructions. Either prepend assert or ' - 'remove it.', - 'B016': 'Cannot raise a literal. Did you intend to return it or raise an Exception?', - 'B017': 'self.assertRaises(Exception): should be considered evil. It can lead to your test passing even if the ' - 'code being tested is never executed due to a typo. Either assert for a more specific exception (builtin ' - 'or custom), use assertRaisesRegex, or use the context manager form of assertRaises (with ' - 'self.assertRaises(Exception) as ex:) with an assertion against the data available in ex.', - - # Python 3 compatibility warnings - 'B301': 'Python 3 does not include .iter* methods on dictionaries. The default behavior is to return iterables. ' - 'Simply remove the iter prefix from the method. For Python 2 compatibility, also prefer the Python 3 ' - 'equivalent if you expect that the size of the dict to be small and bounded. The performance regression ' - 'on Python 2 will be negligible and the code is going to be the clearest. Alternatively, use six.iter* or ' - 'future.utils.iter*.', - 'B302': 'Python 3 does not include .view* methods on dictionaries. The default behavior is to return viewables. ' - 'Simply remove the view prefix from the method. For Python 2 compatibility, also prefer the Python 3 ' - 'equivalent if you expect that the size of the dict to be small and bounded. The performance regression ' - 'on Python 2 will be negligible and the code is going to be the clearest. Alternatively, use six.view* or ' - 'future.utils.view*.', - 'B303': 'The __metaclass__ attribute on a class definition does nothing on Python 3. Use class MyClass(BaseClass, ' - 'metaclass=...). For Python 2 compatibility, use six.add_metaclass.', - 'B304': 'sys.maxint is not a thing on Python 3. Use sys.maxsize.', - 'B305': '.next() is not a thing on Python 3. Use the next() builtin. For Python 2 compatibility, use six.next().', - 'B306': 'BaseException.message has been deprecated as of Python 2.6 and is removed in Python 3. Use str(e) to ' - 'access the user-readable message. Use e.args to access arguments passed to the exception.', -} - -# According to https://github.com/gforcada/flake8-builtins/blob/master/flake8_builtins.py#L49 -ALL_BUILTINS_ISSUES = { - 'A001': 'variable is shadowing a python builtin', - 'A002': 'argument is shadowing a python builtin', - 'A003': 'class attribute is shadowing a python builtin', -} - -# According to https://github.com/afonasev/flake8-return -ALL_RETURN_ISSUES = { - 'R501': 'do not explicitly return None in function if it is the only possible return value.', - 'R502': 'do not implicitly return None in function able to return non-None value.', - 'R503': 'missing explicit return at the end of function able to return non-None value.', - 'R504': 'unecessary variable assignement before return statement.', -} - -# According to https://pypi.org/project/flake8-string-format/ -ALL_FORMAT_STRING_ISSUES = { - # Presence of implicit parameters - 'P101': 'format string does contain unindexed parameters', - 'P102': 'docstring does contain unindexed parameters', - 'P103': 'other string does contain unindexed parameters', - - # Missing values in the parameters - 'P201': 'format call uses too large index (INDEX)', - 'P202': 'format call uses missing keyword (KEYWORD)', - 'P203': 'format call uses keyword arguments but no named entries', - 'P204': 'format call uses variable arguments but no numbered entries', - 'P205': 'format call uses implicit and explicit indexes together', - - # Unused values in the parameters - 'P301': 'format call provides unused index (INDEX)', - 'P302': 'format call provides unused keyword (KEYWORD)', -} - -# According to https://pypi.org/project/flake8-import-order/ -ALL_IMPORT_ORDER_ISSUES = { - 'I100': 'Your import statements are in the wrong order.', - 'I101': 'The names in your from import are in the wrong order.', - 'I201': 'Missing newline between import groups.', - 'I202': 'Additional newline in a group of imports.', -} - -# According to https://pypi.org/project/flake8-comprehensions/ -ALL_COMPREHENSIONS_ISSUES = { - 'C400': 'Unnecessary generator - rewrite as a comprehension.', - 'C401': 'Unnecessary generator - rewrite as a comprehension.', - 'C402': 'Unnecessary generator - rewrite as a comprehension.', - - 'C403': 'Unnecessary list comprehension - rewrite as a comprehension.', - 'C404': 'Unnecessary list comprehension - rewrite as a comprehension.', - - 'C405': 'Unnecessary literal - rewrite as a literal.', - 'C406': 'Unnecessary literal - rewrite as a literal.', - - 'C408': 'Unnecessary call - rewrite as a literal.', - - 'C409': ' Unnecessary passed to () - (remove the outer call to ``' - '()/rewrite as a `` literal).', - 'C410': ' Unnecessary passed to () - (remove the outer call to ``' - '()/rewrite as a `` literal).', - - 'C411': 'Unnecessary list call - remove the outer call to list().', - - 'C413': 'Unnecessary call around sorted().', - - 'C414': 'Unnecessary call within ().', - - 'C415': 'Unnecessary subscript reversal of iterable within ().', - - 'C416': 'Unnecessary comprehension - rewrite using ().', -} - -# According to https://pypi.org/project/flake8-spellcheck/ -ALL_SPELLCHECK_ISSUES = { - 'SC100': 'Spelling error in comments', - 'SC200': 'Spelling error in name (e.g. variable, function, class)', -} - -# According to https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html -ALL_WPS_ISSUES = { - # Naming - 'WPS100': 'Found wrong module name', - 'WPS101': 'Found wrong module magic name', - 'WPS102': 'Found incorrect module name pattern', - 'WPS110': 'Found wrong variable name', - 'WPS111': 'Found too short name', - 'WPS112': 'Found private name pattern', - 'WPS113': 'Found same alias import', - 'WPS114': 'Found underscored number name pattern', - 'WPS115': 'Found upper-case constant in a class', - 'WPS116': 'Found consecutive underscores name', - 'WPS117': 'Found name reserved for first argument', - 'WPS118': 'Found too long name', - 'WPS119': 'Found unicode name', - 'WPS120': 'Found regular name with trailing underscore', - 'WPS121': 'Found usage of a variable marked as unused', - 'WPS122': 'Found all unused variables definition', - 'WPS123': 'Found wrong unused variable name', - 'WPS124': 'Found unreadable characters combination', - 'WPS125': 'Found builtin shadowing', - - # Complexity - 'WPS200': 'Found module with high Jones Complexity score', - 'WPS201': 'Found module with too many imports', - 'WPS202': 'Found too many module members', - 'WPS203': 'Found module with too many imported names', - 'WPS204': 'Found overused expression', - 'WPS210': 'Found too many local variables', - 'WPS211': 'Found too many arguments', - 'WPS212': 'Found too many return statements', - 'WPS213': 'Found too many expressions', - 'WPS214': 'Found too many methods', - 'WPS215': 'Too many base classes', - 'WPS216': 'Too many decorators', - 'WPS217': 'Found too many await expressions', - 'WPS218': 'Found too many `assert` statements', - 'WPS219': 'Found too deep access level', - 'WPS220': 'Found too deep nesting', - 'WPS221': 'Found line with high Jones Complexity', - 'WPS222': 'Found a condition with too much logic', - 'WPS223': 'Found too many `elif` branches', - 'WPS224': 'Found a comprehension with too many `for` statements', - 'WPS225': 'Found too many `except` cases', - 'WPS226': 'Found string constant over-use', - 'WPS227': 'Found too long yield tuple', - 'WPS228': 'Found too long compare', - 'WPS229': 'Found too long ``try`` body length', - 'WPS230': 'Found too many public instance attributes', - 'WPS231': 'Found function with too much cognitive complexity', - 'WPS232': 'Found module cognitive complexity that is too high', - 'WPS233': 'Found call chain that is too long', - 'WPS234': 'Found overly complex annotation', - 'WPS235': 'Found too many imported names from a module', - 'WPS236': 'Found too many variables used to unpack a tuple', - 'WPS237': 'Found a too complex `f` string', - 'WPS238': 'Found too many raises in a function', - - # Consistency - 'WPS300': 'Found local folder import', - 'WPS301': 'Found dotted raw import', - 'WPS302': 'Found unicode string prefix', - 'WPS303': 'Found underscored number', - 'WPS304': 'Found partial float', - 'WPS305': 'Found `f` string', - 'WPS306': 'Found class without a base class', - 'WPS307': 'Found list comprehension with multiple `if`s', - 'WPS308': 'Found constant comparison', - 'WPS309': 'Found reversed compare order', - 'WPS310': 'Found bad number suffix', - 'WPS311': 'Found multiple `in` compares', - 'WPS312': 'Found comparison of a variable to itself', - 'WPS313': 'Found parenthesis immediately after a keyword', - 'WPS314': 'Found conditional that always evaluates the same', - 'WPS315': 'Found extra `object` in parent classes list', - 'WPS316': 'Found context manager with too many assignments', - 'WPS317': 'Found incorrect multi-line parameters', - 'WPS318': 'Found extra indentation', - 'WPS319': 'Found bracket in wrong position', - 'WPS320': 'Found multi-line function type annotation', - 'WPS321': 'Found uppercase string modifier', - 'WPS322': 'Found incorrect multi-line string', - 'WPS323': 'Found `%` string formatting', - 'WPS324': 'Found inconsistent `return` statement', - 'WPS325': 'Found inconsistent `yield` statement', - 'WPS326': 'Found implicit string concatenation', - 'WPS327': 'Found useless `continue` at the end of the loop', - 'WPS328': 'Found useless node', - 'WPS329': 'Found useless `except` case', - 'WPS330': 'Found unnecessary operator', - 'WPS332': 'Found walrus operator', - 'WPS333': 'Found implicit complex compare', - 'WPS334': 'Found reversed complex comparison', - 'WPS335': 'Found incorrect `for` loop iter type', - 'WPS336': 'Found explicit string concatenation', - 'WPS337': 'Found multiline conditions', - 'WPS338': 'Found incorrect order of methods in a class', - 'WPS339': 'Found number with meaningless zeros', - 'WPS340': 'Found exponent number with positive exponent', - 'WPS341': 'Found wrong hex number case', - 'WPS342': 'Found implicit raw string', - 'WPS343': 'Found wrong complex number suffix', - 'WPS344': 'Found explicit zero division', - 'WPS345': 'Found meaningless number operation', - 'WPS346': 'Found wrong operation sign', - 'WPS347': 'Found vague import that may cause confusion', - 'WPS348': 'Found a line that starts with a dot', - 'WPS349': 'Found redundant subscript slice', - 'WPS350': 'Found usable augmented assign pattern', - 'WPS351': 'Found unnecessary literals', - 'WPS352': 'Found multiline loop', - 'WPS353': 'Found incorrect `yield from` target', - 'WPS354': 'Found consecutive `yield` expressions', - 'WPS355': 'Found an unnecessary blank line before a bracket', - 'WPS356': 'Found an unnecessary iterable unpacking', - 'WPS357': 'Found a ``\\r`` (carriage return) line break', - 'WPS358': 'Found a float zero (0.0)', - 'WPS359': 'Found an iterable unpacking to list', - 'WPS360': 'Found an unnecessary use of a raw string', - 'WPS361': 'Found an inconsistently structured comprehension', - 'WPS362': 'Found assignment to a subscript slice', - - # Best practices - 'WPS400': 'Found wrong magic comment', - 'WPS401': 'Found wrong doc comment', - 'WPS402': 'Found `noqa` comments overuse', - 'WPS403': 'Found `noqa` comments overuse', - 'WPS404': 'Found complex default value', - 'WPS405': 'Found wrong `for` loop variable definition', - 'WPS406': 'Found wrong context manager variable definition', - 'WPS407': 'Found mutable module constant', - 'WPS408': 'Found duplicate logical condition', - 'WPS409': 'Found heterogeneous compare', - 'WPS410': 'Found wrong metadata variable', - 'WPS411': 'Found empty module', - 'WPS412': 'Found `__init__.py` module with logic', - 'WPS413': 'Found bad magic module function', - 'WPS414': 'Found incorrect unpacking target', - 'WPS415': 'Found duplicate exception', - 'WPS416': 'Found `yield` inside comprehension', - 'WPS417': 'Found non-unique item in hash', - 'WPS418': 'Found exception inherited from `BaseException`', - 'WPS419': 'Found `try`/`else`/`finally` with multiple return paths', - 'WPS420': 'Found wrong keyword', - 'WPS421': 'Found wrong function call', - 'WPS422': 'Found future import', - 'WPS423': 'Found raise NotImplemented', - 'WPS424': 'Found except `BaseException`', - 'WPS425': 'Found boolean non-keyword argument', - 'WPS426': 'Found `lambda` in loop\'s body', - 'WPS427': 'Found unreachable code', - 'WPS428': 'Found statement that has no effect', - 'WPS429': 'Found multiple assign targets', - 'WPS430': 'Found nested function', - 'WPS431': 'Found nested class', - 'WPS432': 'Found magic number', - 'WPS433': 'Found nested import', - 'WPS434': 'Found reassigning variable to itself', - 'WPS435': 'Found list multiply', - 'WPS436': 'Found protected module import', - 'WPS437': 'Found protected attribute usage', - 'WPS438': 'Found `StopIteration` raising inside generator', - 'WPS439': 'Found unicode escape in a binary string', - 'WPS440': 'Found block variables overlap', - 'WPS441': 'Found control variable used after block', - 'WPS442': 'Found outer scope names shadowing', - 'WPS443': 'Found unhashable item', - 'WPS444': 'Found incorrect keyword condition', - 'WPS445': 'Found incorrectly named keyword in the starred dict', - 'WPS446': 'Found approximate constant', - 'WPS447': 'Found alphabet as strings', - 'WPS448': 'Found incorrect exception order', - 'WPS449': 'Found float used as a key', - 'WPS450': 'Found protected object import', - 'WPS451': 'Found positional-only argument', - 'WPS452': 'Found `break` or `continue` in `finally` block', - 'WPS453': 'Found executable mismatch', - 'WPS454': 'Found wrong `raise` exception type', - 'WPS455': 'Found non-trivial expression as an argument for "except"', - 'WPS456': 'Found "NaN" as argument to float()', - 'WPS457': 'Found an infinite while loop', - 'WPS458': 'Found imports collision', - 'WPS459': 'Found comparison with float or complex number', - 'WPS460': 'Found single element destructuring', - 'WPS461': 'Forbidden inline ignore', - 'WPS462': 'Wrong multiline string usage', - 'WPS463': 'Found a getter without a return value', - 'WPS464': 'Found empty comment', - 'WPS465': 'Found likely bitwise and boolean operation mixup', - 'WPS466': 'Found new-styled decorator', - - # Refactoring - 'WPS500': 'Found `else` in a loop without `break`', - 'WPS501': 'Found `finally` in `try` block without `except`', - 'WPS502': 'Found simplifiable `if` condition', - 'WPS503': 'Found useless returning `else` statement', - 'WPS504': 'Found negated condition', - 'WPS505': 'Found nested `try` block', - 'WPS506': 'Found useless lambda declaration', - 'WPS507': 'Found useless `len()` compare', - 'WPS508': 'Found incorrect `not` with compare usage', - 'WPS509': 'Found incorrectly nested ternary', - 'WPS510': 'Found `in` used with a non-set container', - 'WPS511': 'Found separate `isinstance` calls that can be merged for', - 'WPS512': 'Found `isinstance` call with a single element tuple', - 'WPS513': 'Found implicit `elif` condition', - 'WPS514': 'Found implicit `in` condition', - 'WPS515': 'Found `open()` used without a context manager', - 'WPS516': 'Found `type()` used to compare types', - 'WPS517': 'Found pointless starred expression', - 'WPS518': 'Found implicit `enumerate()` call', - 'WPS519': 'Found implicit `sum()` call', - 'WPS520': 'Found compare with falsy constant', - 'WPS521': 'Found wrong `is` compare', - 'WPS522': 'Found implicit primitive in a form of `lambda`', - 'WPS523': 'Found incorrectly swapped variables', - 'WPS524': 'Found self assignment with refactored assignment', - 'WPS525': 'Found wrong `in` compare with single item container', - 'WPS526': 'Found implicit `yield from` usage', - 'WPS527': 'Found not a tuple used as an argument', - 'WPS528': 'Found implicit `.items()` usage', - 'WPS529': 'Found implicit `.get()` dict usage', - 'WPS530': 'Found implicit negative index', - 'WPS531': 'Found simplifiable returning `if` condition in a function', - - # OOP - 'WPS600': 'Found subclassing a builtin', - 'WPS601': 'Found shadowed class attribute', - 'WPS602': 'Found using `@staticmethod`', - 'WPS603': 'Found using restricted magic method', - 'WPS604': 'Found incorrect node inside `class` body', - 'WPS605': 'Found method without arguments', - 'WPS606': 'Found incorrect base class', - 'WPS607': 'Found incorrect `__slots__` syntax', - 'WPS608': 'Found incorrect `super()` call', - 'WPS609': 'Found direct magic attribute usage', - 'WPS610': 'Found forbidden `async` magic method usage', - 'WPS611': 'Found forbidden `yield` magic method usage', - 'WPS612': 'Found useless overwritten method', - 'WPS613': 'Found incorrect `super()` call context: incorrect name access', - 'WPS614': 'Found descriptor applied on a function', - 'WPS615': 'Found unpythonic getter or setter', -} - -# According to the flake8 inspector config -FLAKE8_DISABLED_ISSUES = { - 'W291', - 'W292', # no newline at end of file - 'W293', - 'W503', # line break before binary operator - 'C408', # unnecessary (dict/list/tuple) call - rewrite as a literal - 'E501', # line too long - 'E800', # commented out code - 'I101', # order of imports within a line - 'I202', # additional new line - 'Q000', - 'E301', 'E302', 'E303', 'E304', 'E305', - 'E402', # module level import not at top of file - 'I100', # Import statements are in the wrong order - # WPS: Naming - 'WPS110', # Forbid blacklisted variable names. - 'WPS111', # Forbid short variable or module names. - 'WPS112', # Forbid private name pattern. - 'WPS114', # Forbid names with underscored numbers pattern. - 'WPS125', # Forbid variable or module names which shadow builtin names. - # WPS: Consistency - 'WPS303', # Forbid underscores in numbers. - 'WPS305', # Forbid f strings. - 'WPS306', # Forbid writing classes without base classes. - 'WPS318', # Forbid extra indentation. - 'WPS323', # Forbid % formatting on strings. - 'WPS324', # Enforce consistent return statements. - 'WPS335', # Forbid wrong for loop iter targets. - 'WPS358', # Forbid using float zeros: 0.0. - 'WPS362', # Forbid assignment to a subscript slice. - # WPS: Best practices - 'WPS404', # Forbid complex defaults. - 'WPS420', # Forbid some python keywords. - 'WPS421', # Forbid calling some built-in functions.(e.g., print) - 'WPS429', # Forbid multiple assignments on the same line. - 'WPS430', # Forbid nested functions. - 'WPS431', # Forbid nested classes. - 'WPS435', # Forbid multiplying lists. - # WPS: Refactoring - 'WPS518', # Forbid implicit enumerate() calls. - 'WPS527', # Require tuples as arguments for frozenset. - # WPS: OOP - 'WPS602', # Forbid @staticmethod decorator. - # flake8-string-format - 'P101', - 'P102', - 'P103', - 'F522', # unused named arguments. - 'F523', # unused positional arguments. - 'F524', # missing argument. - 'F525', # mixing automatic and manual numbering. - # flake8-commas - 'C814', # missing trailing comma in Python 2 -} diff --git a/src/python/evaluation/inspectors/inspectors_stat/issues/other_issues.py b/src/python/evaluation/inspectors/inspectors_stat/issues/other_issues.py deleted file mode 100644 index 365e6d74..00000000 --- a/src/python/evaluation/inspectors/inspectors_stat/issues/other_issues.py +++ /dev/null @@ -1,8 +0,0 @@ -PYTHON_RADON_ISSUES = { - 'RAD100': 'MAINTAINABILITY index', -} - -PYTHON_AST_ISSUES = { - 'C001': 'Boolean expressions length', - 'C002': 'Functions length', -} diff --git a/src/python/evaluation/inspectors/inspectors_stat/issues/pylint_all_issues.py b/src/python/evaluation/inspectors/inspectors_stat/issues/pylint_all_issues.py deleted file mode 100644 index b73e9efd..00000000 --- a/src/python/evaluation/inspectors/inspectors_stat/issues/pylint_all_issues.py +++ /dev/null @@ -1,496 +0,0 @@ -# According to https://seanwasere.com/pylint--list-msgs/ -ALL_ISSUES = { - 'C0102': 'Used when the name is listed in the black list (unauthorized names).', - 'C0103': 'Used when the name doesn\'t conform to naming rules associated to its type (constant, variable, ' - 'class...).', - 'C0111': 'Used when a module, function, class or method has no docstring.Some special methods like __init__ ' - 'doesn\'t necessary require a docstring.', - 'C0112': 'Used when a module, function, class or method has an empty docstring (it would be too easy ;).', - 'C0113': 'Used when a boolean expression contains an unneeded negation.', - 'C0121': 'Used when an expression is compared to singleton values like True, False or None.', - 'C0122': 'Used when the constant is placed on the left side of a comparison. It is usually clearer in intent to ' - 'place it in the right hand side of the comparison.', - 'C0123': 'The idiomatic way to perform an explicit typecheck in Python is to use isinstance(x, Y) rather than ' - 'type(x) == Y, type(x) is Y. Though there are unusual situations where these give different results.', - 'C0200': 'Emitted when code that iterates with range and len is encountered. Such code can be simplified by using ' - 'the enumerate builtin.', - 'C0201': 'Emitted when the keys of a dictionary are iterated through the .keys() method. It is enough to just ' - 'iterate through the dictionary itself, as in "for key in dictionary".', - 'C0202': 'Used when a class method has a first argument named differently than the value specified in ' - 'valid-classmethod-first-arg option (default to "cls"), recommended to easily differentiate them from ' - 'regular instance methods.', - 'C0203': 'Used when a metaclass method has a first argument named differently than the value specified in ' - 'valid-classmethod-first-arg option (default to "cls"), recommended to easily differentiate them from ' - 'regular instance methods.', - 'C0204': 'Used when a metaclass class method has a first argument named differently than the value specified in ' - 'valid-metaclass-classmethod-first-arg option (default to "mcs"), recommended to easily differentiate ' - 'them from regular instance methods.', - 'C0205': 'Used when a class __slots__ is a simple string, rather than an iterable.', - 'C0301': 'Used when a line is longer than a given number of characters.', - 'C0302': 'Used when a module has too many lines, reducing its readability.', - 'C0303': 'Used when there is whitespace between the end of a line and the newline.', - 'C0304': 'Used when the last line in a file is missing a newline.', - 'C0305': 'Used when there are trailing blank lines in a file.', - 'C0321': 'Used when more than on statement are found on the same line.', - 'C0325': 'Used when a single item in parentheses follows an if, for, or other keyword.', - 'C0326': 'Used when a wrong number of spaces is used around an operator, bracket or block opener.', - 'C0327': 'Used when there are mixed (LF and CRLF) newline signs in a file.', - 'C0328': 'Used when there is different newline than expected.', - 'C0330': 'bad-continuation', - 'C0401': 'Used when a word in comment is not spelled correctly.', - 'C0402': 'Used when a word in docstring is not spelled correctly.', - 'C0403': 'Used when a word in docstring cannot be checked by enchant.', - 'C0410': 'Used when import statement importing multiple modules is detected.', - 'C0411': 'Used when PEP8 import order is not respected (standard imports first, then third-party libraries, ' - 'then local imports)', - 'C0412': 'Used when imports are not grouped by packages', - 'C0413': 'Used when code and imports are mixed', - 'C0414': 'Used when an import alias is same as original package.e.g using import numpy as numpy instead of import ' - 'numpy as np', - 'C1801': 'Used when Pylint detects that len(sequence) is being used inside a condition to determine if a sequence ' - 'is empty. Instead of comparing the length to 0, rely on the fact that empty sequences are false.', - - 'E0001': 'Used when a syntax error is raised for a module.', - 'E0011': 'Used when an unknown inline option is encountered.', - 'E0012': 'Used when a bad value for an inline option is encountered.', - 'E0100': 'Used when the special class method __init__ is turned into a generator by a yield in its body.', - 'E0101': 'Used when the special class method __init__ has an explicit return value.', - 'E0102': 'Used when a function / class / method is redefined.', - 'E0103': 'Used when break or continue keywords are used outside a loop.', - 'E0104': 'Used when a "return" statement is found outside a function or method.', - 'E0105': 'Used when a "yield" statement is found outside a function or method.', - 'E0107': 'Used when you attempt to use the C-style pre-increment or pre-decrement operator -- and ++, ' - 'which doesn\'t exist in Python.', - 'E0108': 'Duplicate argument names in function definitions are syntax errors.', - 'E0110': 'Used when an abstract class with `abc.ABCMeta` as metaclass has abstract methods and is instantiated.', - 'E0111': 'Used when the first argument to reversed() builtin isn\'t a sequence (does not implement __reversed__, ' - 'nor __getitem__ and __len__', - 'E0112': 'Emitted when there are more than one starred expressions (`*x`) in an assignment. This is a SyntaxError.', - 'E0113': 'Emitted when a star expression is used as a starred assignment target.', - 'E0114': 'Emitted when a star expression is not used in an assignment target.', - 'E0115': 'Emitted when a name is both nonlocal and global.', - 'E0116': 'Emitted when the `continue` keyword is found inside a finally clause, which is a SyntaxError.', - 'E0117': 'Emitted when a nonlocal variable does not have an attached name somewhere in the parent scopes', - 'E0118': 'Emitted when a name is used prior a global declaration, which results in an error since Python 3.6. ' - 'This message can\'t be emitted when using Python < 3.6.', - 'E0119': 'Emitted when format function is not called on str object. ' - 'This might not be what the user intended to do.', - 'E0202': 'Used when a class defines a method which is hidden by an instance attribute from an ancestor class or ' - 'set by some client code.', - 'E0203': 'Used when an instance member is accessed before it\'s actually assigned.', - 'E0211': 'Used when a method which should have the bound instance as first argument has no argument defined.', - 'E0213': 'Used when a method has an attribute different the "self" as first argument. This is considered as an ' - 'error since this is a so common convention that you shouldn\'t break it!', - 'E0236': 'Used when an invalid (non-string) object occurs in __slots__.', - 'E0237': 'Used when assigning to an attribute not defined in the class slots.', - 'E0238': 'Used when an invalid __slots__ is found in class. Only a string, an iterable or a sequence is permitted.', - 'E0239': 'Used when a class inherits from something which is not a class.', - 'E0240': 'Used when a class has an inconsistent method resolution order.', - 'E0241': 'Used when a class has duplicate bases.', - 'E0301': 'Used when an __iter__ method returns something which is not an iterable (i.e. has no `__next__` method)', - 'E0302': 'Emitted when a special method was defined with an invalid number of parameters. If it has too few or ' - 'too many, it might not work at all.', - 'E0303': 'Used when a __len__ method returns something which is not a non-negative integer', - 'E0401': 'Used when pylint has been unable to import a module.', - 'E0402': 'Used when a relative import tries to access too many levels in the current package.', - 'E0601': 'Used when a local variable is accessed before its assignment.', - 'E0602': 'Used when an undefined variable is accessed.', - 'E0603': 'Used when an undefined variable name is referenced in __all__.', - 'E0604': 'Used when an invalid (non-string) object occurs in __all__.', - 'E0611': 'Used when a name cannot be found in a module.', - 'E0633': 'Used when something which is not a sequence is used in an unpack assignment', - 'E0701': 'Used when except clauses are not in the correct order (from the more specific to the more generic). If ' - 'you don\'t fix the order, some exceptions may not be caught by the most specific handler.', - 'E0702': 'Used when something which is neither a class, an instance or a string is raised (i.e. a `TypeError` ' - 'will be raised).', - 'E0703': 'Used when using the syntax "raise ... from ...", where the exception context is not an exception, ' - 'nor None.', - 'E0704': 'Used when a bare raise is not used inside an except clause. This generates an error, since there are no ' - 'active exceptions to be reraised. An exception to this rule is represented by a bare raise inside a ' - 'finally clause, which might work, as long as an exception is raised inside the try block, ' - 'but it is nevertheless a code smell that must not be relied upon.', - 'E0710': 'Used when a new style class which doesn\'t inherit from BaseException is raised.', - 'E0711': 'Used when NotImplemented is raised instead of NotImplementedError', - 'E0712': 'Used when a class which doesn\'t inherit from Exception is used as an exception in an except clause.', - 'E1003': 'Used when another argument than the current class is given as first argument of the super builtin.', - 'E1101': 'Used when a variable is accessed for an unexistent member.', - 'E1102': 'Used when an object being called has been inferred to a non callable object.', - 'E1111': 'Used when an assignment is done on a function call but the inferred function doesn\'t return anything.', - 'E1120': 'Used when a function call passes too few arguments.', - 'E1121': 'Used when a function call passes too many positional arguments.', - 'E1123': 'Used when a function call passes a keyword argument that doesn\'t correspond to one of the function\'s ' - 'parameter names.', - 'E1124': 'Used when a function call would result in assigning multiple values to a function parameter, one value ' - 'from a positional argument and one from a keyword argument.', - 'E1125': 'Used when a function call does not pass a mandatory keyword-only argument.', - 'E1126': 'Used when a sequence type is indexed with an invalid type. Valid types are ints, slices, and objects ' - 'with an __index__ method.', - 'E1127': 'Used when a slice index is not an integer, None, or an object with an __index__ method.', - 'E1128': 'Used when an assignment is done on a function call but the inferred function returns nothing but None.', - 'E1129': 'Used when an instance in a with statement doesn\'t implement the context manager protocol(' - '__enter__/__exit__).', - 'E1130': 'Emitted when a unary operand is used on an object which does not support this type of operation.', - 'E1131': 'Emitted when a binary arithmetic operation between two operands is not supported.', - 'E1132': 'Emitted when a function call got multiple values for a keyword.', - 'E1133': 'Used when a non-iterable value is used in place where iterable is expected', - 'E1134': 'Used when a non-mapping value is used in place where mapping is expected', - 'E1135': 'Emitted when an instance in membership test expression doesn\'t implement membership protocol (' - '__contains__/__iter__/__getitem__).', - 'E1136': 'Emitted when a subscripted value doesn\'t support subscription (i.e. doesn\'t define __getitem__ method ' - 'or __class_getitem__ for a class).', - 'E1137': 'Emitted when an object does not support item assignment (i.e. doesn\'t define __setitem__ method).', - 'E1138': 'Emitted when an object does not support item deletion (i.e. doesn\'t define __delitem__ method).', - 'E1139': 'Emitted whenever we can detect that a class is using, as a metaclass, something which might be invalid ' - 'for using as a metaclass.', - 'E1140': 'Emitted when a dict key is not hashable (i.e. doesn\'t define __hash__ method).', - 'E1200': 'Used when an unsupported format character is used in a logging statement format string.', - 'E1201': 'Used when a logging statement format string terminates before the end of a conversion specifier.', - 'E1205': 'Used when a logging format string is given too many arguments.', - 'E1206': 'Used when a logging format string is given too few arguments.', - 'E1300': 'Used when an unsupported format character is used in a format string.', - 'E1301': 'Used when a format string terminates before the end of a conversion specifier.', - 'E1302': 'Used when a format string contains both named (e.g. \'%(foo)d\') and unnamed (e.g. \'%d\') conversion ' - 'specifiers. This is also used when a named conversion specifier contains * for the minimum field width ' - 'and/or precision.', - 'E1303': 'Used when a format string that uses named conversion specifiers is used with an argument that is not a ' - 'mapping.', - 'E1304': 'Used when a format string that uses named conversion specifiers is used with a dictionary that doesn\'t ' - 'contain all the keys required by the format string.', - 'E1305': 'Used when a format string that uses unnamed conversion specifiers is given too many arguments.', - 'E1306': 'Used when a format string that uses unnamed conversion specifiers is given too few arguments', - 'E1307': 'Used when a type required by format string is not suitable for actual argument type', - 'E1310': 'The argument to a str.{l,r,}strip call contains a duplicate character,', - 'E1507': 'Env manipulation functions support only string type arguments. See ' - 'https://docs.python.org/3/library/os.html#os.getenv.', - 'E1601': 'Used when a print statement is used (`print` is a function in Python 3)', - 'E1602': 'Used when parameter unpacking is specified for a function(Python 3 doesn\'t allow it)', - 'E1603': 'Python3 will not allow implicit unpacking of exceptions in except clauses. See ' - 'http://www.python.org/dev/peps/pep-3110/', - 'E1604': 'Used when the alternate raise syntax \'raise foo, bar\' is used instead of \'raise foo(bar)\'.', - 'E1605': 'Used when the deprecated "``" (backtick) operator is used instead of the str() function.', - 'E1700': 'Used when an `yield` or `yield from` statement is found inside an async function. This message can\'t ' - 'be emitted when using Python < 3.5.', - 'E1701': 'Used when an async context manager is used with an object that does not implement the async context ' - 'management protocol. This message can\'t be emitted when using Python < 3.5.', - - # refactoring related checks - 'R0123': 'Used when comparing an object to a literal, which is usually what you do not want to do, since you can ' - 'compare to a different literal than what was expected altogether.', - 'R0124': 'Used when something is compared against itself.', - 'R0201': 'Used when a method doesn\'t use its bound instance, and so could be written as a function.', - 'R0202': 'Used when a class method is defined without using the decorator syntax.', - 'R0203': 'Used when a static method is defined without using the decorator syntax.', - 'R0205': 'Used when a class inherit from object, which under python3 is implicit, hence can be safely removed ' - 'from bases.', - 'R0401': 'Used when a cyclic import between two or more modules is detected.', - 'R0801': 'Indicates that a set of similar lines has been detected among multiple file. This usually means that ' - 'the code should be refactored to avoid this duplication.', - 'R0901': 'Used when class has too many parent classes, try to reduce this to get a simpler (and so easier to use) ' - 'class.', - 'R0902': 'Used when class has too many instance attributes, try to reduce this to get a simpler (and so easier to ' - 'use) class.', - 'R0903': 'Used when class has too few public methods, so be sure it\'s really worth it.', - 'R0904': 'Used when class has too many public methods, try to reduce this to get a simpler (and so easier to use) ' - 'class.', - 'R0911': 'Used when a function or method has too many return statement, making it hard to follow.', - 'R0912': 'Used when a function or method has too many branches, making it hard to follow.', - 'R0913': 'Used when a function or method takes too many arguments.', - 'R0914': 'Used when a function or method has too many local variables.', - 'R0915': 'Used when a function or method has too many statements. You should then split it in smaller functions / ' - 'methods.', - 'R0916': 'Used when an if statement contains too many boolean expressions.', - 'R1701': 'Used when multiple consecutive isinstance calls can be merged into one.', - 'R1702': 'Used when a function or a method has too many nested blocks. This makes the code less understandable ' - 'and maintainable.', - 'R1703': 'Used when an if statement can be replaced with \'bool(test)\'.', - 'R1704': 'Used when a local name is redefining an argument, which might suggest a potential error. This is taken ' - 'in account only for a handful of name binding operations, such as for iteration, with statement ' - 'assignment and exception handler assignment.', - 'R1705': 'Used in order to highlight an unnecessary block of code following an if containing a return statement. ' - 'As such, it will warn when it encounters an else following a chain of ifs, all of them containing a ' - 'return statement.', - 'R1706': 'Used when one of known pre-python 2.5 ternary syntax is used.', - 'R1707': 'In Python, a tuple is actually created by the comma symbol, not by the parentheses. Unfortunately, ' - 'one can actually create a tuple by misplacing a trailing comma, which can lead to potential weird bugs ' - 'in your code. You should always use parentheses explicitly for creating a tuple.', - 'R1708': 'According to PEP479, the raise of StopIteration to end the loop of a generator may lead to hard to find ' - 'bugs. This PEP specify that raise StopIteration has to be replaced by a simple return statement', - 'R1709': 'Emitted when redundant pre-python 2.5 ternary syntax is used.', - 'R1710': 'According to PEP8, if any return statement returns an expression, any return statements where no value ' - 'is returned should explicitly state this as return None, and an explicit return statement should be ' - 'present at the end of the function (if reachable)', - 'R1711': 'Emitted when a single "return" or "return None" statement is found at the end of function or method ' - 'definition. This statement can safely be removed because Python will implicitly return None', - 'R1712': 'You do not have to use a temporary variable in order to swap variables. Using "tuple unpacking" to ' - 'directly swap variables makes the intention more clear.', - 'R1713': 'Using str.join(sequence) is faster, uses less memory and increases readability compared to for-loop ' - 'iteration.', - 'R1714': 'To check if a variable is equal to one of many values,combine the values into a tuple and check if the ' - 'variable is contained "in" it instead of checking for equality against each of the values.This is ' - 'faster and less verbose.', - 'R1715': 'Using the builtin dict.get for getting a value from a dictionary if a key is present or a default if ' - 'not, is simpler and considered more idiomatic, although sometimes a bit slower', - 'R1716': 'This message is emitted when pylint encounters boolean operation like"a < b and b < c", suggesting ' - 'instead to refactor it to "a < b < c"', - 'R1717': 'Although there is nothing syntactically wrong with this code, it is hard to read and can be simplified ' - 'to a dict comprehension.Also it is faster since you don\'t need to create another transient list', - 'R1718': 'Although there is nothing syntactically wrong with this code, it is hard to read and can be simplified ' - 'to a set comprehension.Also it is faster since you don\'t need to create another transient list', - 'R1719': 'Used when an if expression can be replaced with \'bool(test)\'.', - 'R1720': 'Used in order to highlight an unnecessary block of code following an if containing a raise statement. ' - 'As such, it will warn when it encounters an else following a chain of ifs, all of them containing a ' - 'raise statement.', - - # warnings for stylistic issues, or minor programming issues - 'W0101': 'Used when there is some code behind a "return" or "raise" statement, which will never be accessed.', - 'W0102': 'Used when a mutable value as list or dictionary is detected in a default value for an argument.', - 'W0104': 'Used when a statement doesn\'t have (or at least seems to) any effect.', - 'W0105': 'Used when a string is used as a statement (which of course has no effect). This is a particular case of ' - 'W0104 with its own message so you can easily disable it if you\'re using those strings as ' - 'documentation, instead of comments.', - 'W0106': 'Used when an expression that is not a function call is assigned to nothing. Probably something else was ' - 'intended.', - 'W0107': 'Used when a "pass" statement that can be avoided is encountered.', - 'W0108': 'Used when the body of a lambda expression is a function call on the same argument list as the lambda ' - 'itself; such lambda expressions are in all but a few cases replaceable with the function being called ' - 'in the body of the lambda.', - 'W0109': 'Used when a dictionary expression binds the same key multiple times.', - 'W0111': 'Used when assignment will become invalid in future Python release due to introducing new keyword.', - 'W0120': 'Loops should only have an else clause if they can exit early with a break statement, otherwise the ' - 'statements under else should be on the same scope as the loop itself.', - 'W0122': 'Used when you use the "exec" statement (function for Python 3), to discourage its usage. That doesn\'t ' - 'mean you cannot use it !', - 'W0123': 'Used when you use the "eval" function, to discourage its usage. Consider using `ast.literal_eval` for ' - 'safely evaluating strings containing Python expressions from untrusted sources.', - 'W0124': 'Emitted when a `with` statement component returns multiple values and uses name binding with `as` only ' - 'for a part of those values, as in with ctx() as a, b. This can be misleading, since it\'s not clear if ' - 'the context manager returns a tuple or if the node without a name binding is another context manager.', - 'W0125': 'Emitted when a conditional statement (If or ternary if) uses a constant value for its test. This might ' - 'not be what the user intended to do.', - 'W0143': 'This message is emitted when pylint detects that a comparison with a callable was made, which might ' - 'suggest that some parenthesis were omitted, resulting in potential unwanted behaviour.', - 'W0150': 'Used when a break or a return statement is found inside the finally clause of a try...finally block: ' - 'the exceptions raised in the try clause will be silently swallowed instead of being re-raised.', - 'W0199': 'A call of assert on a tuple will always evaluate to true if the tuple is not empty, and will always ' - 'evaluate to false if it is.', - 'W0201': 'Used when an instance attribute is defined outside the __init__ method.', - 'W0211': 'Used when a static method has "self" or a value specified in valid- classmethod-first-arg option or ' - 'valid-metaclass-classmethod-first-arg option as first argument.', - 'W0212': 'Used when a protected member (i.e. class member with a name beginning with an underscore) is access ' - 'outside the class or a descendant of the class where it\'s defined.', - 'W0221': 'Used when a method has a different number of arguments than in the implemented interface or in an ' - 'overridden method.', - 'W0222': 'Used when a method signature is different than in the implemented interface or in an overridden method.', - 'W0223': 'Used when an abstract method (i.e. raise NotImplementedError) is not overridden in concrete class.', - 'W0231': 'Used when an ancestor class method has an __init__ method which is not called by a derived class.', - 'W0232': 'Used when a class has no __init__ method, neither its parent classes.', - 'W0233': 'Used when an __init__ method is called on a class which is not in the direct ancestors for the analysed ' - 'class.', - 'W0235': 'Used whenever we can detect that an overridden method is useless, relying on super() delegation to do ' - 'the same thing as another method from the MRO.', - 'W0301': 'Used when a statement is ended by a semi-colon (";"), which isn\'t necessary (that\'s python, not C ;).', - 'W0311': 'Used when an unexpected number of indentation\'s tabulations or spaces has been found.', - 'W0312': 'Used when there are some mixed tabs and spaces in a module.', - 'W0401': 'Used when `from module import *` is detected.', - 'W0402': 'Used a module marked as deprecated is imported.', - 'W0404': 'Used when a module is reimported multiple times.', - 'W0406': 'Used when a module is importing itself.', - 'W0410': 'Python 2.5 and greater require __future__ import to be the first non docstring statement in the module.', - 'W0511': 'Used when a warning note as FIXME or XXX is detected.', - 'W0601': 'Used when a variable is defined through the "global" statement but the variable is not defined in the ' - 'module scope.', - 'W0602': 'Used when a variable is defined through the "global" statement but no assignment to this variable is ' - 'done.', - 'W0603': 'Used when you use the "global" statement to update a global variable. Pylint just try to discourage ' - 'this usage. That doesn\'t mean you cannot use it !', - 'W0604': 'Used when you use the "global" statement at the module level since it has no effect', - 'W0611': 'Used when an imported module or variable is not used.', - 'W0612': 'Used when a variable is defined but not used.', - 'W0613': 'Used when a function or method argument is not used.', - 'W0614': 'Used when an imported module or variable is not used from a `\'from X import *\'` style import.', - 'W0621': 'Used when a variable\'s name hides a name defined in the outer scope.', - 'W0622': 'Used when a variable or function override a built-in.', - 'W0623': 'Used when an exception handler assigns the exception to an existing name', - 'W0631': 'Used when a loop variable (i.e. defined by a for loop or a list comprehension or a generator ' - 'expression) is used outside the loop.', - 'W0632': 'Used when there is an unbalanced tuple unpacking in assignment', - 'W0640': 'A variable used in a closure is defined in a loop. This will result in all closures using the same ' - 'value for the closed-over variable.', - 'W0641': 'Used when a variable is defined but might not be used. The possibility comes from the fact that locals(' - ') might be used, which could consume or not the said variable', - 'W0642': 'Invalid assignment to self or cls in instance or class method respectively.', - 'W0702': 'Used when an except clause doesn\'t specify exceptions type to catch.', - 'W0703': 'Used when an except catches a too general exception, possibly burying unrelated errors.', - 'W0705': 'Used when an except catches a type that was already caught by a previous handler.', - 'W0706': 'Used when an except handler uses raise as its first or only operator. This is useless because it raises ' - 'back the exception immediately. Remove the raise operator or the entire try-except-raise block!', - 'W0711': 'Used when the exception to catch is of the form "except A or B:". If intending to catch multiple, ' - 'rewrite as "except (A, B):"', - 'W0715': 'Used when passing multiple arguments to an exception constructor, the first of them a string literal ' - 'containing what appears to be placeholders intended for formatting', - 'W0716': 'Used when an operation is done against an exception, but the operation is not valid for the exception ' - 'in question. Usually emitted when having binary operations between exceptions in except handlers.', - 'W1113': 'When defining a keyword argument before variable positional arguments, one can end up in having ' - 'multiple values passed for the aforementioned parameter in case the method is called with keyword ' - 'arguments.', - 'W1201': 'Used when a logging statement has a call form of "logging.(format_string % (' - 'format_args...))". Such calls should leave string interpolation to the logging method itself and be ' - 'written "logging.(format_string, format_args...)" so that the program may avoid ' - 'incurring the cost of the interpolation in those cases in which no message will be logged. For more, ' - 'see http://www.python.org/dev/peps/pep-0282/.', - 'W1202': 'Used when a logging statement has a call form of "logging.(format_string.format(' - 'format_args...))". Such calls should use % formatting instead, but leave interpolation to the logging ' - 'function by passing the parameters as arguments.', - 'W1203': 'Used when a logging statement has a call form of "logging.method(f"..."))". Such calls should use % ' - 'formatting instead, but leave interpolation to the logging function by passing the parameters as ' - 'arguments.', - 'W1300': 'Used when a format string that uses named conversion specifiers is used with a dictionary whose keys ' - 'are not all strings.', - 'W1301': 'Used when a format string that uses named conversion specifiers is used with a dictionary that contains ' - 'keys not required by the format string.', - 'W1302': 'Used when a PEP 3101 format string is invalid.', - 'W1303': 'Used when a PEP 3101 format string that uses named fields doesn\'t ' - 'receive one or more required keywords.', - 'W1304': 'Used when a PEP 3101 format string that uses named fields is used with an argument that is not required ' - 'by the format string.', - 'W1305': 'Used when a PEP 3101 format string contains both automatic field numbering and manual ' - 'field specification.', - 'W1306': 'Used when a PEP 3101 format string uses an attribute specifier ({0.length}), but the argument passed ' - 'for formatting doesn\'t have that attribute.', - 'W1307': 'Used when a PEP 3101 format string uses a lookup specifier ({a[1]}), but the argument passed for ' - 'formatting doesn\'t contain or doesn\'t have that key as an attribute.', - 'W1308': 'Used when we detect that a string formatting is repeating an argument instead of using named string ' - 'arguments', - 'W1401': 'Used when a backslash is in a literal string but not as an escape.', - 'W1402': 'Used when an escape like \\u is encountered in a byte string where it has no effect.', - 'W1403': 'String literals are implicitly concatenated in a ' - 'literal iterable definition : maybe a comma is missing ?', - 'W1501': 'Python supports: r, w, a[, x] modes with b, +, and U (only with r) options. ' - 'See http://docs.python.org/2/library/functions.html#open', - 'W1503': 'The first argument of assertTrue and assertFalse is a condition. If a constant is passed as parameter, ' - 'that condition will be always true. In this case a warning should be emitted.', - 'W1505': 'The method is marked as deprecated and will be removed in a future version of Python. Consider looking ' - 'for an alternative in the documentation.', - 'W1506': 'The warning is emitted when a threading.Thread class is instantiated without the target function being ' - 'passed. By default, the first parameter is the group param, not the target param.', - 'W1507': 'os.environ is not a dict object but proxy object, so shallow copy has still effects on original object. ' - 'See https://bugs.python.org/issue15373 for reference.', - 'W1508': 'Env manipulation functions return None or str values. Supplying anything different as a default may ' - 'cause bugs. See https://docs.python.org/3/library/os.html#os.getenv.', - 'W1509': 'The preexec_fn parameter is not safe to use in the presence of threads in your application. The child ' - 'process could deadlock before exec is called. If you must use it, keep it trivial! Minimize the number ' - 'of libraries you call into.https://docs.python.org/3/library/subprocess.html#popen-constructor', - # miss some inspections that were missed from Python 3 - - 'I0001': 'Used to inform that a built-in module has not been checked using the raw checkers.', - 'I0010': 'Used when an inline option is either badly formatted or can\'t be used inside modules.', - 'I0011': 'Used when an inline option disables a message or a messages category.', - 'I0013': 'Used to inform that the file will not be checked', - 'I0020': 'A message was triggered on a line, but suppressed explicitly by a disable= comment in the file. ' - 'This message is not generated for messages that are ignored due to configuration settings.', - 'I0021': 'Reported when a message is explicitly disabled for a line or a block of code, but never triggered.', - 'I0022': 'Some inline pylint options have been renamed or reworked, only the most recent form should be used. ' - 'NOTE:skip-all is only available with pylint >= 0.26', - 'I0023': 'Used when a message is enabled or disabled by id.', - 'I1101': 'Used when a variable is accessed for non-existent member of C extension. Due to unavailability of source ' - 'static analysis is impossible, but it may be performed by introspecting living objects in run-time.', -} - -# According to the pylint inspector config -PYLINT_DISABLED_ISSUES = { - 'C0103', # invalid-name - 'C0111', # missing-docstring - 'C0301', # line-too-long - 'C0304', # missing-final-newline - 'E1601', # print-statement - 'E1602', # parameter-unpacking - 'E1603', # unpacking-in-except - 'E1604', # old-raise-syntax - 'E1605', - 'I0001', # raw-checker-failed - 'I0010', # bad-inline-option - 'I0011', # locally-disabled - 'I0013', # file-ignored - 'I0020', # suppressed-message - 'I0021', # useless-suppression - 'I0022', # deprecated-pragma - 'I0023', # use-symbolic-message-instead - 'R0901', # too-many-ancestors - 'R0902', # too-many-instance-attributes - 'R0903', # too-few-public-methods - 'R0904', # too-many-public-methods, - 'R0911', # too-many-return-statements - 'R0912', # too-many-branches - 'R0913', # too-many-arguments - 'R0914', # too-many-locals - 'R0916', # too-many-boolean-expressions - 'W1601', # apply-builtin - 'W1602', # basestring-builtin - 'W1603', # buffer-builtin - 'W1604', # cmp-builtin - 'W1605', # coerce-builtin - 'W1606', - 'W1607', # file-builtin - 'W1608', # long-builtin - 'W1609', # raw_input-builtin - 'W1610', # reduce-builtin - 'W1611', - 'W1612', # unicode-builtin - 'W1613', - 'W1614', # coerce-method - 'W1615', - 'W1616', - 'W1617', - 'W1618', # no-absolute-import - 'W1619', # old-division - 'W1620', # dict-iter-method - 'W1621', # dict-view-method - 'W1622', # next-method-called - 'W1623', - 'W1624', # indexing-exception - 'W1625', # raising-string - 'W1626', # reload-builtin - 'W1627', # oct-method - 'W1628', # hex-method - 'W1629', # nonzero-method - 'W1630', # cmp-method - 'W1632', # input-builtin - 'W1633', # round-builtin - 'W1634', # intern-builtin - 'W1635', - 'W1636', # map-builtin-not-iterating - 'W1637', # zip-builtin-not-iterating - 'W1638', # range-builtin-not-iterating - 'W1639', # filter-builtin-not-iterating - 'W1640', # using-cmp-argument - 'W1641', # eq-without-hash - 'W1642', # div-method - 'W1643', - 'W1644', - 'W1645', # exception-message-attribute - 'W1646', - 'W1647', # sys-max-int - 'W1648', # bad-python3-import - 'W1649', # deprecated-string-function - 'W1650', # deprecated-str-translate-call - 'W1651', # deprecated-itertools-function - 'W1652', # deprecated-types-field - 'W1653', # next-method-defined - 'W1654', # dict-items-not-iterating - 'W1655', # dict-keys-not-iterating - 'W1656', # dict-values-not-iterating - 'W1657', # deprecated-operator-function - 'W1658', # deprecated-urllib-function - 'W1659', - 'W1660', # deprecated-sys-function - 'W1661', # exception-escape - 'W1662', # comprehension-escape, - 'W0603', # global-statement - 'C0413', # wrong-import-position - 'R0915', # too-many-statements - 'C0327', # mixed-line-endings - 'E0401', # import-error - 'C0303', # trailing-whitespace - 'R1705', # no-else-return - 'R1720', # no-else-raise -} diff --git a/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py b/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py deleted file mode 100644 index 0de99fc6..00000000 --- a/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py +++ /dev/null @@ -1,129 +0,0 @@ -import argparse -from typing import Callable, Dict, List, Set, Tuple - -from src.python.evaluation.inspectors.inspectors_stat.issues.flake8_all_issues import ( - ALL_BUGBEAR_ISSUES, ALL_BUILTINS_ISSUES, ALL_COMPREHENSIONS_ISSUES, ALL_FORMAT_STRING_ISSUES, - ALL_IMPORT_ORDER_ISSUES, ALL_RETURN_ISSUES, ALL_SPELLCHECK_ISSUES, ALL_STANDARD_ISSUES, ALL_WPS_ISSUES, - FLAKE8_DISABLED_ISSUES, -) -from src.python.evaluation.inspectors.inspectors_stat.issues.other_issues import PYTHON_AST_ISSUES, PYTHON_RADON_ISSUES -from src.python.evaluation.inspectors.inspectors_stat.issues.pylint_all_issues import ALL_ISSUES, PYLINT_DISABLED_ISSUES -from src.python.review.common.language import Language -from src.python.review.inspectors.checkstyle.checkstyle import CheckstyleInspector -from src.python.review.inspectors.checkstyle.issue_types import CHECK_CLASS_NAME_TO_ISSUE_TYPE -from src.python.review.inspectors.detekt.detekt import DetektInspector -from src.python.review.inspectors.detekt.issue_types import DETECT_CLASS_NAME_TO_ISSUE_TYPE -from src.python.review.inspectors.eslint.eslint import ESLintInspector -from src.python.review.inspectors.eslint.issue_types import ESLINT_CLASS_NAME_TO_ISSUE_TYPE -from src.python.review.inspectors.flake8.flake8 import Flake8Inspector -from src.python.review.inspectors.issue import ( - get_default_issue_stat, get_main_category_by_issue_type, IssuesStat, IssueType, -) -from src.python.review.inspectors.pmd.issue_types import PMD_RULE_TO_ISSUE_TYPE -from src.python.review.inspectors.pmd.pmd import PMDInspector -from src.python.review.inspectors.pyast.python_ast import PythonAstInspector -from src.python.review.inspectors.pylint.pylint import PylintInspector -from src.python.review.inspectors.radon.radon import RadonInspector - - -def __get_flake8_issue_keys() -> Set[str]: - issues_dicts = [ALL_STANDARD_ISSUES, ALL_BUGBEAR_ISSUES, ALL_BUILTINS_ISSUES, ALL_RETURN_ISSUES, - ALL_FORMAT_STRING_ISSUES, ALL_IMPORT_ORDER_ISSUES, ALL_COMPREHENSIONS_ISSUES, - ALL_SPELLCHECK_ISSUES, ALL_WPS_ISSUES] - all_issues = set().union(*map(lambda d: d.keys(), issues_dicts)) - return set(all_issues - set(FLAKE8_DISABLED_ISSUES)) - - -def __match_issue_keys_to_issue_type(issue_keys: Set[str], matcher: Callable) -> Dict[str, IssueType]: - matched_issues = {} - for key in issue_keys: - matched_issues[key] = matcher(key) - return matched_issues - - -# Count for each main category the frequency of issues for this category -def __gather_issues_stat(issue_types: List[IssueType]) -> IssuesStat: - main_category_to_issue_type = get_default_issue_stat() - for issue_type in issue_types: - main_category_to_issue_type[get_main_category_by_issue_type(issue_type)] += 1 - return main_category_to_issue_type - - -def __merge_issues_stats(*args: IssuesStat) -> IssuesStat: - assert len(args) >= 1, 'Please, use at least one argument' - final_stat = {} - for key in args[0].keys(): - final_stat[key] = sum(d[key] for d in args) - return final_stat - - -def __collect_language_stat(*args: Set[Tuple[Set[str], Callable]]) -> IssuesStat: - all_issue_types = [] - for issues, matcher in args: - all_issue_types.append(__match_issue_keys_to_issue_type(issues, matcher).values()) - return __merge_issues_stats(*map(lambda stat: __gather_issues_stat(stat), all_issue_types)) - - -def collect_stat_by_language(language: Language) -> IssuesStat: - if language == Language.PYTHON: - python_inspection_to_matcher = [ - (set(ALL_ISSUES.keys()) - set(PYLINT_DISABLED_ISSUES), PylintInspector.choose_issue_type), - (__get_flake8_issue_keys(), Flake8Inspector.choose_issue_type), - (set(PYTHON_RADON_ISSUES.keys()), RadonInspector.choose_issue_type), - (set(PYTHON_AST_ISSUES.keys()), PythonAstInspector.choose_issue_type), - ] - return __collect_language_stat(*python_inspection_to_matcher) - elif language == Language.JAVA: - java_inspection_to_matcher = [ - (set(PMD_RULE_TO_ISSUE_TYPE.keys()), PMDInspector.choose_issue_type), - (set(CHECK_CLASS_NAME_TO_ISSUE_TYPE.keys()), CheckstyleInspector.choose_issue_type), - ] - return __collect_language_stat(*java_inspection_to_matcher) - elif language == Language.KOTLIN: - kotlin_inspection_to_matcher = [ - (set(DETECT_CLASS_NAME_TO_ISSUE_TYPE.keys()), DetektInspector.choose_issue_type), - ] - return __collect_language_stat(*kotlin_inspection_to_matcher) - elif language == Language.JS: - js_inspection_to_matcher = [ - (set(ESLINT_CLASS_NAME_TO_ISSUE_TYPE.keys()), ESLintInspector.choose_issue_type), - ] - return __collect_language_stat(*js_inspection_to_matcher) - - raise NotImplementedError(f'Language {language} is not supported yet!') - - -def print_stat(language: Language, stat: IssuesStat) -> None: - print(f'Collected statistics for {language.value.lower()} language:') - for issue_type, freq in stat.items(): - print(f'{issue_type}: {freq} times;') - print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories.') - - -def __parse_language(language: str) -> Language: - try: - return Language(language.upper()) - except KeyError: - raise KeyError(f'Incorrect language key: {language}. Please, try again!') - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - languages = ', '.join(map(lambda l: l.lower(), Language.values())) - - parser.add_argument('language', - type=__parse_language, - help=f'The language for which statistics will be printed. Available values are: {languages}') - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - language = args.language - stat = collect_stat_by_language(language) - print_stat(language, stat) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/inspectors/print_inspectors_statistics.py b/src/python/evaluation/inspectors/print_inspectors_statistics.py deleted file mode 100644 index e3146cd6..00000000 --- a/src/python/evaluation/inspectors/print_inspectors_statistics.py +++ /dev/null @@ -1,95 +0,0 @@ -import argparse -from collections import defaultdict -from pathlib import Path -from typing import Dict, List - -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.inspectors.common.statistics import ( - GeneralInspectorsStatistics, IssuesStatistics, PenaltyInfluenceStatistics, PenaltyIssue, -) -from src.python.review.common.file_system import deserialize_data_from_file -from src.python.review.inspectors.issue import ShortIssue - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.DIFFS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.DIFFS_FILE_PATH.value.description) - - parser.add_argument('--categorize', - help='If True, statistics will be categorized by several categories.', - action='store_true') - - parser.add_argument('-n', '--top-n', - help='The top N items will be printed', - type=int, - default=10) - - parser.add_argument('--full-stat', - help='If True, full statistics will be printed.', - action='store_true') - - -def has_incorrect_grades(diffs_dict: dict) -> bool: - return len(diffs_dict.get(ColumnName.GRADE.value, [])) > 0 - - -def has_decreased_grades(diffs_dict: dict) -> bool: - return len(diffs_dict.get(ColumnName.DECREASED_GRADE.value, [])) > 0 - - -def __gather_issues_stat(issues_stat_dict: Dict[int, List[PenaltyIssue]]) -> IssuesStatistics: - fragments_in_stat = len(issues_stat_dict) - issues_dict: Dict[ShortIssue, int] = defaultdict(int) - for _, issues in issues_stat_dict.items(): - for issue in issues: - short_issue = ShortIssue(origin_class=issue.origin_class, type=issue.type) - issues_dict[short_issue] += 1 - return IssuesStatistics(issues_dict, fragments_in_stat) - - -def gather_statistics(diffs_dict: dict) -> GeneralInspectorsStatistics: - new_issues_stat = __gather_issues_stat(diffs_dict.get(ColumnName.TRACEBACK.value, {})) - penalty_issues_stat = __gather_issues_stat(diffs_dict.get(ColumnName.PENALTY.value, {})) - return GeneralInspectorsStatistics(new_issues_stat, penalty_issues_stat, - PenaltyInfluenceStatistics(diffs_dict.get(ColumnName.PENALTY.value, {}))) - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - separator = '______' - - diffs = deserialize_data_from_file(args.diffs_file_path) - if has_incorrect_grades(diffs): - print(f'WARNING! Was found incorrect grades in the following fragments: {diffs[ColumnName.GRADE.value]}.') - else: - print('SUCCESS! Was not found incorrect grades.') - - if not has_decreased_grades(diffs): - print('All grades are equal.') - else: - print(f'Decreased grades was found in {len(diffs[ColumnName.DECREASED_GRADE.value])} fragments') - print(f'{diffs.get(ColumnName.USER.value, 0)} unique users was found!') - print(separator) - - statistics = gather_statistics(diffs) - n = args.top_n - print('NEW INSPECTIONS STATISTICS:') - statistics.new_issues_stat.print_full_statistics(n, args.full_stat, separator) - print(separator) - - print('PENALTY INSPECTIONS STATISTICS;') - statistics.penalty_issues_stat.print_full_statistics(n, args.full_stat, separator) - print(separator) - - print('INFLUENCE ON PENALTY STATISTICS;') - statistics.penalty_influence_stat.print_stat() - print(separator) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/issues_statistics/README.md b/src/python/evaluation/issues_statistics/README.md deleted file mode 100644 index 0b7f83d7..00000000 --- a/src/python/evaluation/issues_statistics/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Hyperstyle evaluation: statistics - -This module allows you to collect and visualize the statistics of the tool. - -## Get raw issues -This script allows you to get raw issues (issues that have not yet been processed by the main algorithm) for each fragment from a dataset (`xlsx` or `csv` file). The dataset must have 3 obligatory columns: -- `id` -- `code` -- `lang` - -Possible values for column `lang` are: `python3`, `kotlin`, `javascript`, `java7`, `java8`, `java9`, `java11`, `java15`. - -The output file is a new `xlsx` or `csv` file with all columns from the input file and an additional column: `raw_issues`. - -### Usage -Run the [get_raw_issues.py](get_raw_issues.py) with the arguments from command line. - -**Required arguments:** -- `solutions_file_path` — path to xlsx-file or csv-file with code samples to inspect. - -**Optional arguments:** -| Argument | Description | -|----------|-------------| -| **‑‑allow‑duplicates** | Allow duplicate issues found by different linters. By default, duplicates are skipped. | -| **‑‑allow‑zero‑measure‑issues** | Allow issues with zero measure. By default, such issues are skipped. | -| **‑‑allow‑info‑issues** | Allow issues from the INFO category. By default, such issues are skipped. | -| **‑‑to‑save‑path** | Allows to save the path to the file where the issue was found. By default, the path is not saved. | -| **‑o**, **‑‑output** | Path where the dataset with raw issues will be saved. If not specified, the dataset will be saved next to the original one. | -| **‑l**, **‑‑log-output** | Path where logs will be stored. If not specified, then logs will be output to stderr. | - -## Get raw issues statistics -The script takes the dataframe obtained after executing [get_raw_issues.py](get_raw_issues.py) and outputs dataframes with statistics grouped by language. - -The input dataset must have 3 obligatory columns: -- `id` -- `code` -- `lang` -- `raw_issues` - -Possible values for column `lang` are: `python3`, `kotlin`, `javascript`, `java7`, `java8`, `java9`, `java11`, `java15`. - -The output files is a new `xlsx` or `csv` files which contains the `value` column and the columns responsible for its category statistics. - -The `value` column shows the metric value (for measurable issue categories), quantity (for quantitative issue categories) or `ratio * 100` (for `CODE_STYLE` and `LINE_LEN`), where `ratio` is calculated as in the corresponding rules (`CodeStyleRule` and `LineLengthRule`). - -The table cells indicate how often value occurs in one fragment (for quantitative categories) or in all fragments (for measurable categories). - -All output datasets are arranged in folders according to language. - -### Usage -Run the [get_raw_issues_statistics.py](get_raw_issues_statistics.py) with the arguments from command line. - -**Required arguments:** -- `solutions_with_raw_issues` — path to an xlsx- or csv-file with code samples and raw issues, which were received with [get_raw_issues.py](get_raw_issues.py). - -**Optional arguments:** -| Argument | Description | -|----------|-------------| -| **‑o**, **‑‑output** | Path to the folder where datasets with statistics will be saved. If not specified, the datasets will be saved in the folder next to the original dataset. | diff --git a/src/python/evaluation/issues_statistics/__init__.py b/src/python/evaluation/issues_statistics/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/issues_statistics/common/__init__.py b/src/python/evaluation/issues_statistics/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/issues_statistics/common/raw_issue_encoder_decoder.py b/src/python/evaluation/issues_statistics/common/raw_issue_encoder_decoder.py deleted file mode 100644 index 42206eb3..00000000 --- a/src/python/evaluation/issues_statistics/common/raw_issue_encoder_decoder.py +++ /dev/null @@ -1,67 +0,0 @@ -import json -from pathlib import Path - -from src.python.review.inspectors.inspector_type import InspectorType -from src.python.review.inspectors.issue import ( - BaseIssue, - CodeIssue, - get_issue_class_by_issue_type, - IssueData, - IssueDifficulty, - IssueType, - Measurable, - MEASURABLE_ISSUE_TYPE_TO_MEASURE_NAME, -) - -MEASURE = 'measure' - - -class RawIssueEncoder(json.JSONEncoder): - to_safe_path: bool - - def __init__(self, to_safe_path: bool = True, **kwargs): - super().__init__(**kwargs) - self.to_safe_path = to_safe_path - - def default(self, obj): - if isinstance(obj, BaseIssue): - issue_data = { - IssueData.ORIGIN_ClASS.value: obj.origin_class, - IssueData.ISSUE_TYPE.value: obj.type.value, - IssueData.DESCRIPTION.value: obj.description, - IssueData.FILE_PATH.value: str(obj.file_path) if self.to_safe_path else "", - IssueData.LINE_NUMBER.value: obj.line_no, - IssueData.COLUMN_NUMBER.value: obj.column_no, - IssueData.INSPECTOR_TYPE.value: obj.inspector_type.value, - IssueData.DIFFICULTY.value: obj.difficulty.value, - } - - if isinstance(obj, Measurable): - issue_data[MEASURE] = obj.measure() - - return issue_data - - return json.JSONEncoder.default(self, obj) - - -class RawIssueDecoder(json.JSONDecoder): - def __init__(self, *args, **kwargs): - super().__init__(object_hook=self.object_hook, *args, **kwargs) - - def object_hook(self, json_dict): - json_dict[IssueData.ISSUE_TYPE.value] = IssueType(json_dict[IssueData.ISSUE_TYPE.value]) - json_dict[IssueData.INSPECTOR_TYPE.value] = InspectorType(json_dict[IssueData.INSPECTOR_TYPE.value]) - json_dict[IssueData.FILE_PATH.value] = Path(json_dict[IssueData.FILE_PATH.value]) - # TODO: remove get after analyzing raw issue statistics - json_dict[IssueData.DIFFICULTY.value] = IssueDifficulty( - json_dict.get(IssueData.DIFFICULTY.value, IssueDifficulty.HARD.value), - ) - - issue_type = json_dict[IssueData.ISSUE_TYPE.value] - if issue_type in MEASURABLE_ISSUE_TYPE_TO_MEASURE_NAME.keys(): - measure_name = MEASURABLE_ISSUE_TYPE_TO_MEASURE_NAME[issue_type] - json_dict[measure_name] = json_dict.pop(MEASURE) - measurable_issue_class = get_issue_class_by_issue_type(issue_type) - return measurable_issue_class(**json_dict) - - return CodeIssue(**json_dict) diff --git a/src/python/evaluation/issues_statistics/get_raw_issues.py b/src/python/evaluation/issues_statistics/get_raw_issues.py deleted file mode 100644 index 9aaaac7d..00000000 --- a/src/python/evaluation/issues_statistics/get_raw_issues.py +++ /dev/null @@ -1,275 +0,0 @@ -import argparse -import json -import logging -import os -import sys -from pathlib import Path -from typing import List, Optional - -sys.path.append('') -sys.path.append('../../..') - -import numpy as np -import pandas as pd -from pandarallel import pandarallel -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path, write_df_to_file -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.issues_statistics.common.raw_issue_encoder_decoder import RawIssueEncoder -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import ( - create_file, - Extension, - get_name_from_path, - get_parent_folder, -) -from src.python.review.common.language import Language -from src.python.review.inspectors.issue import ( - BaseIssue, - IssueType, - Measurable, -) -from src.python.review.reviewers.common import LANGUAGE_TO_INSPECTORS -from src.python.review.reviewers.utils.issues_filter import filter_duplicate_issues - -LANG = ColumnName.LANG.value -CODE = ColumnName.CODE.value -ID = ColumnName.ID.value -RAW_ISSUES = 'raw_issues' - -ALLOWED_EXTENSION = {Extension.XLSX, Extension.CSV} - -ERROR_CODES = [ - 'E999', # flake8 - 'WPS000', # flake8 (wps) - 'E0001', # pylint -] - -logger = logging.getLogger(__name__) - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.SOLUTIONS_FILE_PATH.value.description, - ) - - parser.add_argument( - RunToolArgument.DUPLICATES.value.long_name, - action='store_true', - help=RunToolArgument.DUPLICATES.value.description, - ) - - parser.add_argument( - '--allow-zero-measure-issues', - action='store_true', - help='Allow issues with zero measure. By default, such issues are skipped.', - ) - - parser.add_argument( - '--allow-info-issues', - action='store_true', - help='Allow issues from the INFO category. By default, such issues are skipped.', - ) - - parser.add_argument( - '--to-save-path', - action='store_true', - help='Allows to save the path to the file where the issue was found. By default, the path is not saved.', - ) - - parser.add_argument( - '-o', '--output', - type=lambda value: Path(value).absolute(), - help='Path where the dataset with raw issues will be saved. ' - 'If not specified, the dataset will be saved next to the original one.', - ) - - parser.add_argument( - '-l', '--log-output', - type=lambda value: Path(value).absolute(), - help='Path where logs will be stored. If not specified, then logs will be output to stderr.', - ) - - -def _filter_issues( - issues: List[BaseIssue], - allow_duplicates: bool, - allow_zero_measure_issues: bool, - allow_info_issues: bool, -) -> List[BaseIssue]: - - filtered_issues = issues - - if not allow_duplicates: - filtered_issues = filter_duplicate_issues(filtered_issues) - - if not allow_zero_measure_issues: - filtered_issues = list( - filter(lambda issue: not isinstance(issue, Measurable) or issue.measure() != 0, filtered_issues), - ) - - if not allow_info_issues: - filtered_issues = list(filter(lambda issue: issue.type != IssueType.INFO, filtered_issues)) - - return filtered_issues - - -def _check_issues_for_errors(issues: List[BaseIssue]) -> bool: - origin_classes = {issue.origin_class for issue in issues} - return any(error_code in origin_classes for error_code in ERROR_CODES) - - -def _inspect_row( - row: pd.Series, - solutions_file_path: Path, - allow_duplicates: bool, - allow_zero_measure_issues: bool, - allow_info_issues: bool, - to_safe_path: bool, -) -> Optional[str]: - - print(f'{row[ID]}: processing started') - - if pd.isnull(row[LANG]): - logger.warning(f'{row[ID]}: no lang.') - return np.nan - - if pd.isnull(row[CODE]): - logger.warning(f'{row[ID]}: no code.') - return np.nan - - # If we were unable to identify the language version, we return None - language_version = LanguageVersion.from_value(row[LANG]) - if language_version is None: - logger.warning(f'{row[ID]}: it was not possible to determine the language version from "{row[LANG]}"') - return np.nan - - # If we were unable to identify the language, we return None - language = Language.from_language_version(language_version) - if language == Language.UNKNOWN: - logger.warning(f'{row[ID]}: it was not possible to determine the language from "{language_version}"') - return np.nan - - # If there are no inspectors for the language, then return None - inspectors = LANGUAGE_TO_INSPECTORS.get(language, []) - if not inspectors: - logger.warning(f'{row[ID]}: no inspectors were found for the {language}.') - return np.nan - - tmp_file_extension = language_version.extension_by_language().value - tmp_file_path = solutions_file_path.parent.absolute() / f'fragment_{row[ID]}{tmp_file_extension}' - temp_file = next(create_file(tmp_file_path, row[CODE])) - - inspectors_config = { - 'language_version': language_version, - 'n_cpu': 1, - } - - raw_issues = [] - - for inspector in inspectors: - try: - issues = inspector.inspect(temp_file, inspectors_config) - - if _check_issues_for_errors(issues): - logger.warning(f'{row[ID]}: inspector {inspector.inspector_type.value} failed.') - continue - - raw_issues.extend(issues) - - except Exception: - logger.warning(f'{row[ID]}: inspector {inspector.inspector_type.value} failed.') - - os.remove(temp_file) - - raw_issues = _filter_issues(raw_issues, allow_duplicates, allow_zero_measure_issues, allow_info_issues) - - json_issues = json.dumps(raw_issues, cls=RawIssueEncoder, to_safe_path=to_safe_path) - - print(f'{row[ID]}: processing finished.') - - return json_issues - - -def _is_correct_output_path(output_path: Path) -> bool: - try: - output_extension = Extension.get_extension_from_file(str(output_path)) - except ValueError: - return False - - return output_extension in ALLOWED_EXTENSION - - -def _get_output_path(solutions_file_path: Path, output_path: Optional[Path]) -> Path: - if output_path is not None: - if _is_correct_output_path(output_path): - return output_path - logger.warning('The output path is not correct. The resulting dataset will be saved next to the original one.') - - extension = Extension.get_extension_from_file(str(solutions_file_path)) - output_dir = get_parent_folder(solutions_file_path) - dataset_name = get_name_from_path(solutions_file_path, with_extension=False) - return output_dir / f'{dataset_name}_with_raw_issues{extension.value}' - - -def inspect_solutions( - solutions_df: pd.DataFrame, - solutions_file_path: Path, - allow_duplicates: bool, - allow_zero_measure_issues: bool, - allow_info_issues: bool, - to_save_path: bool, -) -> pd.DataFrame: - - pandarallel.initialize() - - solutions_df[RAW_ISSUES] = solutions_df.parallel_apply( - _inspect_row, - args=(solutions_file_path, allow_duplicates, allow_zero_measure_issues, allow_info_issues, to_save_path), - axis=1, - ) - - return solutions_df - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - if args.log_output is not None: - args.log_output.parent.mkdir(parents=True, exist_ok=True) - - logging.basicConfig( - filename=args.log_output, filemode='w', level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s', - ) - - solutions = get_solutions_df_by_file_path(args.solutions_file_path) - - logger.info('Dataset inspection started.') - - solutions_with_raw_issues = inspect_solutions( - solutions, - args.solutions_file_path, - args.allow_duplicates, - args.allow_zero_measure_issues, - args.allow_info_issues, - args.to_save_path, - ) - - logger.info('Dataset inspection finished.') - - output_path = _get_output_path(args.solutions_file_path, args.output) - output_extension = Extension.get_extension_from_file(str(output_path)) - - logger.info(f'Saving the dataframe to a file: {output_path}.') - - write_df_to_file(solutions_with_raw_issues, output_path, output_extension) - - logger.info('Saving complete.') - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/issues_statistics/get_raw_issues_statistics.py b/src/python/evaluation/issues_statistics/get_raw_issues_statistics.py deleted file mode 100644 index b32fe4eb..00000000 --- a/src/python/evaluation/issues_statistics/get_raw_issues_statistics.py +++ /dev/null @@ -1,227 +0,0 @@ -import argparse -import json -import logging -import sys -from collections import Counter -from json import JSONDecodeError -from pathlib import Path -from typing import Dict, List, Optional - -sys.path.append('') -sys.path.append('../../..') - -import pandas as pd -from pandarallel import pandarallel -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path, write_df_to_file -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.issues_statistics.common.raw_issue_encoder_decoder import RawIssueDecoder -from src.python.evaluation.issues_statistics.get_raw_issues import RAW_ISSUES -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import Extension, get_parent_folder, get_total_code_lines_from_code -from src.python.review.common.language import Language -from src.python.review.inspectors.issue import BaseIssue, ISSUE_TYPE_TO_CLASS, IssueType, Measurable -from src.python.review.quality.rules.code_style_scoring import CodeStyleRule -from src.python.review.quality.rules.line_len_scoring import LineLengthRule -from src.python.review.reviewers.utils.code_statistics import get_code_style_lines - -ID = ColumnName.ID.value -LANG = ColumnName.LANG.value -CODE = ColumnName.CODE.value - -CODE_STYLE_LINES = f'{IssueType.CODE_STYLE.value}_lines' -CODE_STYLE_RATIO = f'{IssueType.CODE_STYLE.value}_ratio' -LINE_LEN_NUMBER = f'{IssueType.LINE_LEN.value}_number' -LINE_LEN_RATIO = f'{IssueType.LINE_LEN.value}_ratio' -TOTAL_LINES = 'total_lines' -VALUE = 'value' - -OUTPUT_DF_NAME = 'stats' -DEFAULT_OUTPUT_FOLDER_NAME = 'raw_issues_statistics' - -logger = logging.getLogger(__name__) - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - 'solutions_with_raw_issues', - type=lambda value: Path(value).absolute(), - help=f'Local XLSX-file or CSV-file path. Your file must include column-names: ' - f'"{ID}", "{CODE}", "{LANG}", and "{RAW_ISSUES}".', - ) - - parser.add_argument( - '-o', '--output', - type=lambda value: Path(value).absolute(), - help='Path to the folder where datasets with statistics will be saved. ' - 'If not specified, the datasets will be saved in the folder next to the original one.', - ) - - parser.add_argument( - '-l', '--log-output', - type=lambda value: Path(value).absolute(), - help='Path where logs will be stored. If not specified, then logs will be output to stderr.', - ) - - -def _convert_language_code_to_language(fragment_id: str, language_code: str) -> str: - language_version = LanguageVersion.from_value(language_code) - - if language_version is None: - logger.warning(f'{fragment_id}: it was not possible to determine the language version from "{language_code}".') - return language_code - - language = Language.from_language_version(language_version) - - if language == Language.UNKNOWN: - logger.warning(f'{fragment_id}: it was not possible to determine the language from "{language_version}".') - return language_code - - return language.value - - -def _extract_stats_from_issues(row: pd.Series) -> pd.Series: - print(f'{row[ID]}: extracting stats.') - - if pd.isnull(row[CODE]): - logger.warning(f'{row[ID]}: no code.') - row[CODE] = "" - - if pd.isnull(row[LANG]): - logger.warning(f'{row[ID]}: no lang.') - row[LANG] = "" - - try: - issues: List[BaseIssue] = json.loads(row[RAW_ISSUES], cls=RawIssueDecoder) - except (JSONDecodeError, TypeError): - logger.warning(f'{row[ID]}: failed to decode issues.') - issues: List[BaseIssue] = [] - - counter = Counter([issue.type for issue in issues]) - - for issue_type, issue_class in ISSUE_TYPE_TO_CLASS.items(): - if issubclass(issue_class, Measurable): - row[issue_type.value] = [issue.measure() for issue in issues if isinstance(issue, issue_class)] - else: - row[issue_type.value] = counter[issue_type] - - row[CODE_STYLE_LINES] = get_code_style_lines(issues) - row[LINE_LEN_NUMBER] = counter[IssueType.LINE_LEN] - row[TOTAL_LINES] = get_total_code_lines_from_code(row[CODE]) - - row[LANG] = _convert_language_code_to_language(row[ID], row[LANG]) - - print(f'{row[ID]}: extraction of statistics is complete.') - - return row - - -def _convert_ratio_to_int(ratio: float): - """ - Round the ratio to 2 decimal places, multiply by 100, and take the integer part. - """ - return int((round(ratio, 2) * 100)) - - -def _group_stats_by_lang(df_with_stats: pd.DataFrame) -> Dict[str, pd.DataFrame]: - logger.info('The grouping of statistics by language has started.') - - result = {} - - df_grouped_by_lang = df_with_stats.groupby(LANG) - for lang in df_grouped_by_lang.groups: - logger.info(f'"{lang}" statistics grouping started.') - - lang_group = df_grouped_by_lang.get_group(lang) - - columns_with_stats = [] - - for issue_type, issue_class in ISSUE_TYPE_TO_CLASS.items(): - column = lang_group[issue_type.value] - if issubclass(issue_class, Measurable): - column = column.explode() - columns_with_stats.append(column.value_counts()) - - columns_with_stats.append(lang_group[TOTAL_LINES].value_counts()) - - line_len_ratio_column = lang_group.apply( - lambda row: LineLengthRule.get_ratio(row[LINE_LEN_NUMBER], row[TOTAL_LINES]), - axis=1, - ) - line_len_ratio_column = line_len_ratio_column.apply(_convert_ratio_to_int) - line_len_ratio_column.name = LINE_LEN_RATIO - columns_with_stats.append(line_len_ratio_column.value_counts()) - - code_style_ratio_column = lang_group.apply( - lambda row: CodeStyleRule.get_ratio( - row[CODE_STYLE_LINES], row[TOTAL_LINES], Language.from_value(str(lang), default=Language.UNKNOWN), - ), - axis=1, - ) - code_style_ratio_column = code_style_ratio_column.apply(_convert_ratio_to_int) - code_style_ratio_column.name = CODE_STYLE_RATIO - columns_with_stats.append(code_style_ratio_column.value_counts()) - - stats = pd.concat(columns_with_stats, axis=1).fillna(0).astype(int) - - # Put values in a separate column - stats.index.name = VALUE - stats.reset_index(inplace=True) - - result[str(lang)] = stats - logger.info(f'"{lang}" statistics grouping finished.') - - logger.info('The grouping of statistics by language has finished.') - - return result - - -def inspect_raw_issues(solutions_with_raw_issues: pd.DataFrame) -> Dict[str, pd.DataFrame]: - pandarallel.initialize() - - solutions_with_raw_issues = solutions_with_raw_issues.parallel_apply(_extract_stats_from_issues, axis=1) - - return _group_stats_by_lang(solutions_with_raw_issues) - - -def _get_output_folder(solutions_file_path: Path, output_folder: Optional[Path]): - if output_folder is not None: - return output_folder - - return get_parent_folder(solutions_file_path) / DEFAULT_OUTPUT_FOLDER_NAME - - -def _save_stats(stats_by_lang: Dict[str, pd.DataFrame], solutions_file_path: Path, output_path: Optional[Path]) -> None: - output_folder = _get_output_folder(solutions_file_path, output_path) - output_extension = Extension.get_extension_from_file(str(solutions_file_path)) - - logger.info(f'Saving statistics to a folder: {output_folder}.') - - for lang, stats in stats_by_lang.items(): - lang_folder = output_folder / lang - lang_folder.mkdir(parents=True, exist_ok=True) - write_df_to_file(stats, lang_folder / f'{OUTPUT_DF_NAME}{output_extension.value}', output_extension) - - logger.info('Saving statistics is complete.') - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - if args.log_output is not None: - args.log_output.parent.mkdir(parents=True, exist_ok=True) - - logging.basicConfig( - filename=args.log_output, filemode="w", level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s', - ) - - solutions_with_raw_issues = get_solutions_df_by_file_path(args.solutions_with_raw_issues) - - logger.info("Dataset inspection started.") - - stats_by_lang = inspect_raw_issues(solutions_with_raw_issues) - - logger.info("Dataset inspection finished.") - - _save_stats(stats_by_lang, args.solutions_with_raw_issues, args.output) diff --git a/src/python/evaluation/paper_evaluation/README.md b/src/python/evaluation/paper_evaluation/README.md deleted file mode 100644 index 5dac490b..00000000 --- a/src/python/evaluation/paper_evaluation/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Paper evaluation - -This module contains scripts for SIGCSE-2022 paper evaluation: - -- [Comparison with other tools](./comparison_with_other_tools/README.md) -- Formatting issues importance -- [Dynamics of student usage](./user_dynamics/README.md) \ No newline at end of file diff --git a/src/python/evaluation/paper_evaluation/__init__.py b/src/python/evaluation/paper_evaluation/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/README.md b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/README.md deleted file mode 100644 index 618165f7..00000000 --- a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Comparison with other tools evaluation - -This module allows getting statistic about using of several code quality tools. -In our work we compare the Hyperstyle tool with the [Tutor](https://www.hkeuning.nl/rpt/) tool. -Other tools (FrenchPress, WebTA, and AutoStyle) does not have open sources. - -To get statistics we use students solutions for six programming tasks, -but the main script can gather this statistics for any tasks. - -The tasks from the out dataset: -- **countEven**. The `countEven` method returns the number of even integers in the values-array. -- **sumValues**. The `sumValues` method adds up all numbers from the values-array, - or only the positive numbers if the `positivesOnly` boolean parameter is set - to `true`. -- **oddSum**. The method `oddSum` returns the sum of all numbers at an odd index - in the array parameter, until the number -1 is seen at an odd index. -- **calculateScore**. The `calculateScore` method calculates the score for a train trip. - The highest score is 10. The score is based on the number of changes and the day of - the week (Monday is 1, Sunday is 7). -- **hasDoubled**. Write a program that calculates in how many years your savings - have doubled with the given interest. -- **haveThree**. Given an array of ints, return true if the value 3 appears in the - array exactly 3 times, and no 3's are next to each other. - -The dataset has several columns: -- Student id (student_id); -- Task key (task_key); -- Code fragment (solution); -- Tutor error, if it is existed (tutor_error); -- Tutor issues keys (tutor_issues); -- Hyperstyle issues keys (hyperstyle_issues); -- Hyperstyle INFO issues keys (hyperstyle_info_issues); -- Code style issues count (code_style_issues_count). - -The dataset stores in the `csv` format. - -## Usage - -Run the [statistics_gathering.py](statistics_gathering.py) with the arguments from command line. - -Required arguments: - -`solutions_file_path` — path to csv-file with code samples. - -The statistics will be printed in the terminal. The statistics includes: -- Unique users count; -- Code snippets count; -- Tasks statistics: for each task count code snippets and count snippets with the Tutor errors; -- Count code fragments has Tutor errors; -- Count of unique errors was found in Tutor; -- Error statistics: for each error get the error text and frequency; -- Issues statistics: - - Count of unique issues in total; - - Common issues statistics: for all common issues for Hyperstyle and Tutor count frequency of this issue; - - Tutor unique issues statistics: for all Tutor issues (that were not found by Hyperstyle) count frequency of this issue; - - Hyperstyle unique issues statistics: for all Hyperstyle issues (that were not found by Tutor) count frequency of this issue; - - Count code style issues and count fragments with these issues. - diff --git a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/__init__.py b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/statistics_gathering.py b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/statistics_gathering.py deleted file mode 100644 index 130092eb..00000000 --- a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/statistics_gathering.py +++ /dev/null @@ -1,58 +0,0 @@ -import argparse -import logging -import sys -from pathlib import Path - -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import get_solutions_df -from src.python.evaluation.paper_evaluation.comparison_with_other_tools.tutor_statistics import ( - IssuesStatistics, TutorStatistics, -) -from src.python.evaluation.paper_evaluation.comparison_with_other_tools.util import ComparisonColumnName -from src.python.review.common.file_system import Extension, get_restricted_extension - -sys.path.append('') -sys.path.append('../../..') - -logger = logging.getLogger(__name__) - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help='Local CSV-file path with feedback from different tools. ' - 'Your file must include column-names:' - f'"{ComparisonColumnName.STUDENT_ID.name}" and ' - f'"{ComparisonColumnName.TASK_KEY.name}" and ' - f'"{ComparisonColumnName.SOLUTION.name}" and ' - f'"{ComparisonColumnName.TUTOR_ERROR.name}" and ') - - -def main() -> int: - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - solutions_file_path = args.solutions_file_path - extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) - solutions_df = get_solutions_df(extension, solutions_file_path) - tutor_stat = TutorStatistics(solutions_df, to_drop_duplicates=True) - tutor_stat.print_tasks_stat() - tutor_stat.print_error_stat() - print('ISSUES STAT:') - issue_stat = IssuesStatistics(solutions_df) - issue_stat.print_issues_stat() - return 0 - - except FileNotFoundError: - logger.error('CSV-file with the specified name does not exists.') - return 2 - - except Exception: - logger.exception('An unexpected error.') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py deleted file mode 100644 index 083d6311..00000000 --- a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py +++ /dev/null @@ -1,127 +0,0 @@ -from collections import Counter -from collections import defaultdict -from dataclasses import dataclass -from typing import Any, Dict, List - -import pandas as pd -from src.python.evaluation.common.pandas_util import filter_df_by_single_value -from src.python.evaluation.paper_evaluation.comparison_with_other_tools.util import ( - ComparisonColumnName, ERROR_CONST, TutorTask, -) - - -def sort_freq_dict(freq_dict: Dict[Any, int]) -> Dict[Any, int]: - return dict(sorted(freq_dict.items(), key=lambda item: item[1], reverse=True)) - - -@dataclass -class TutorStatistics: - unique_users: int - task_to_freq: Dict[TutorTask, int] - task_to_error_freq: Dict[TutorTask, int] - error_to_freq: Dict[str, int] - fragments_with_error: int = 0 - - __separator: str = '----------' - - def __init__(self, solutions_df: pd.DataFrame, to_drop_duplicates: bool = False): - if to_drop_duplicates: - solutions_df = solutions_df.drop_duplicates(ComparisonColumnName.SOLUTION.value) - self.unique_users = len(solutions_df[ComparisonColumnName.STUDENT_ID.value].unique()) - self.task_to_freq = defaultdict(int) - self.task_to_error_freq = defaultdict(int) - self.error_to_freq = defaultdict(int) - for task in TutorTask: - task_df = filter_df_by_single_value(solutions_df, ComparisonColumnName.TASK_KEY.value, task.value) - self.task_to_freq[task] = task_df.shape[0] - errors_list = list(map(lambda e_l: e_l.split(';'), - task_df[ComparisonColumnName.TUTOR_ERROR.value].dropna().values)) - for cell_errors in errors_list: - for error in cell_errors: - self.error_to_freq[error.strip()] += 1 - self.task_to_error_freq[task] += 1 - self.fragments_with_error += 1 - self.task_to_freq = sort_freq_dict(self.task_to_freq) - self.error_to_freq = sort_freq_dict(self.error_to_freq) - - def print_tasks_stat(self) -> None: - print(f'Unique users count: {self.unique_users}') - print(f'Code snippets count: {sum(self.task_to_freq.values())}') - print('Tasks statistics:') - for task, freq in self.task_to_freq.items(): - print(f'Task {task.value}: {freq} items; {self.task_to_error_freq[task]} with tutor errors') - print(self.__separator) - - def print_error_stat(self) -> None: - print(f'{self.fragments_with_error} code fragments has errors during running by Tutor') - print(f'{len(self.error_to_freq.keys())} unique errors was found in Tutor') - print('Error statistics:') - for error, freq in self.error_to_freq.items(): - print(f'{error}: {freq} items') - print(self.__separator) - - -@dataclass -class IssuesStatistics: - common_issue_to_freq: Dict[str, int] - tutor_uniq_issue_to_freq: Dict[str, int] - hyperstyle_uniq_issue_to_freq: Dict[str, int] - - code_style_issues_count: int - fragments_count_with_code_style_issues: int - - __separator: str = '----------' - - # TODO: info and code style issues - def __init__(self, solutions_df: pd.DataFrame, to_drop_duplicates: bool = False): - if to_drop_duplicates: - solutions_df = solutions_df.drop_duplicates(ComparisonColumnName.SOLUTION.value) - self.common_issue_to_freq = defaultdict(int) - self.tutor_uniq_issue_to_freq = defaultdict(int) - self.hyperstyle_uniq_issue_to_freq = defaultdict(int) - solutions_df.apply(lambda row: self.__init_solution_df_row(row), axis=1) - self.common_issue_to_freq = sort_freq_dict(self.common_issue_to_freq) - self.tutor_uniq_issue_to_freq = sort_freq_dict(self.tutor_uniq_issue_to_freq) - self.hyperstyle_uniq_issue_to_freq = sort_freq_dict(self.hyperstyle_uniq_issue_to_freq) - self.code_style_issues_count = sum(solutions_df[ComparisonColumnName.CODE_STYLE_ISSUES_COUNT.value]) - self.fragments_count_with_code_style_issues = len(list( - filter(lambda x: x != 0, solutions_df[ComparisonColumnName.CODE_STYLE_ISSUES_COUNT.value]))) - - @staticmethod - def __parse_issues(issues_str: str) -> List[str]: - if pd.isna(issues_str) or issues_str == ERROR_CONST: - return [] - return list(map(lambda i: i.strip(), issues_str.split(';'))) - - @staticmethod - def __add_issues(issues_dict: Dict[str, int], issues: List[str]) -> None: - for issue in issues: - issues_dict[issue] += 1 - - def __init_solution_df_row(self, row: pd.DataFrame) -> None: - tutor_issues = self.__parse_issues(row[ComparisonColumnName.TUTOR_ISSUES.value]) - hyperstyle_issues = self.__parse_issues(row[ComparisonColumnName.HYPERSTYLE_ISSUES.value]) - common_issues = list((Counter(tutor_issues) & Counter(hyperstyle_issues)).elements()) - self.__add_issues(self.common_issue_to_freq, common_issues) - self.__add_issues(self.tutor_uniq_issue_to_freq, list(set(tutor_issues) - set(common_issues))) - self.__add_issues(self.hyperstyle_uniq_issue_to_freq, list(set(hyperstyle_issues) - set(common_issues))) - - def __print_freq_issues_stat(self, freq_stat: Dict[str, int], prefix: str) -> None: - print(f'{prefix} issues statistics:') - for issue, freq in freq_stat.items(): - print(f'{issue} was found {freq} times') - print(self.__separator) - - def print_issues_stat(self) -> None: - uniq_issues = (len(self.common_issue_to_freq) - + len(self.tutor_uniq_issue_to_freq) - + len(self.hyperstyle_uniq_issue_to_freq) - ) - print(f'{uniq_issues} unique issues in total was found') - print(self.__separator) - self.__print_freq_issues_stat(self.common_issue_to_freq, 'Common') - self.__print_freq_issues_stat(self.tutor_uniq_issue_to_freq, 'Tutor unique') - self.__print_freq_issues_stat(self.hyperstyle_uniq_issue_to_freq, 'Hyperstyle unique') - print(f'{self.code_style_issues_count} code style issues (spaces, different brackets, indentations)' - f' was found in total by hyperstyle in {self.fragments_count_with_code_style_issues} fragments') - print(self.__separator) diff --git a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/util.py b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/util.py deleted file mode 100644 index eff1bc15..00000000 --- a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/util.py +++ /dev/null @@ -1,27 +0,0 @@ -from enum import Enum, unique - - -@unique -class ComparisonColumnName(Enum): - STUDENT_ID = 'student_id' - TASK_KEY = 'task_key' - SOLUTION = 'solution' - TUTOR_ERROR = 'tutor_error' - - TUTOR_ISSUES = 'tutor_issues' - HYPERSTYLE_ISSUES = 'hyperstyle_issues' - HYPERSTYLE_INFO_ISSUES = 'hyperstyle_info_issues' - CODE_STYLE_ISSUES_COUNT = 'code_style_issues_count' - - -ERROR_CONST = 'ERROR' - - -@unique -class TutorTask(Enum): - EVEN = 'countEven' - SUM_VALUES = 'sumValues' - ODD_SUM = 'oddSum' - SCORE = 'calculateScore' - HAS_DOUBLED = 'hasDoubled' - HAVE_THREE = 'haveThree' diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/README.md b/src/python/evaluation/paper_evaluation/issues_statistics/README.md deleted file mode 100644 index 0c185a79..00000000 --- a/src/python/evaluation/paper_evaluation/issues_statistics/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Raw issue statistics visualization - -This script allows you to visualize raw issue statistics for a paper. - -## Usage -Run the [raw_issues_statistics_visualization.py](./raw_issues_statistics_visualization.py) with the arguments from command line. - -**Required arguments**: - --`stats_path` — path to a file with stats that were founded by [get_raw_issues_statistics.py](../../issues_statistics/get_raw_issues_statistics.py). Must be an xlsx or csv file. --`config_path` — path to the yaml file containing information about the charts to be plotted. A description of the config and its example is provided in [this section](#config). --`save_dir` — directory where the plotted charts will be saved. - -**Optional arguments**: -Argument | Description ---- | --- -**‑‑file‑extension** | Allows you to select the extension of output files. Available extensions: `.png`, `.jpg`, `.jpeg`, `.webp`, `.svg`, `.pdf`, `.eps`, `.json`. Default is `.svg`. - -## Config -The configuration file is a yaml file where each group name has its config. The group config contains `plot_config` and configs for each column of statistics. - -The `plot_config` consists of the following parameters: -- `rows` — number of rows. Default: `1`. -- `cols` — number of cols. Default: `1`. -- `height` — graph height. Default: `800`. -- `width` — graph width. Default: `1600`. -- `x_axis_name` — name of the x-axis. Default: `Value`. -- `y_axis_name` — name of the y-axis. Default: `Quantity`. -- `specs` — сonfiguration of traces on the graph. See [documentation](https://plotly.com/python-api-reference/generated/plotly.subplots.make_subplots.html) for details. Default: `None`. - -The column config consists of the following arguments: -- `range_of_values` — allows you to filter the values. It is an array of two values: a and b. Only values that belong to the range [a, b) are taken into account when plotting. By default, all values are taken into account when plotting. -- `trace_name` — trace name. The default is the name of the column. - -## Examples -### config.yaml -```yaml -measurable: - plot_config: - rows: 2 - cols: 2 - specs: [[{}, {}], [{colspan: 2}, null]] - x_axis_name: Measure - y_axis_name: Number of issues - BOOL_EXPR_LEN: - range_of_values: [1, 11] - trace_name: Boolean Expresion Length - CYCLOMATIC_COMPLEXITY: - range_of_values: [1, 11] - trace_name: Cyclomatic Complexity - FUNC_LEN: - range_of_values: [0, 60] - trace_name: Function Length - -maintainability_and_cohesion: - plot_config: - rows: 2 - width: 1000 - x_axis_name: Lack of measure (%) - y_axis_name: Number of issues - MAINTAINABILITY: - trace_name: Maintainability - COHESION: - trace_name: Cohesion - -ratio: - plot_config: - rows: 2 - width: 1000 - x_axis_name: Ratio (%) - y_axis_name: Number of fragments - CODE_STYLE_ratio: - range_of_values: [ 1, 101 ] - trace_name: Code Style - LINE_LEN_ratio: - range_of_values: [ 1, 101 ] - trace_name: Line Length - -countable: - plot_config: - rows: 2 - cols: 2 - specs: [[{"rowspan": 2}, {}], [null, {}]] - x_axis_name: Number of issues in one fragment - y_axis_name: Number of fragments - ERROR_PRONE: - range_of_values: [ 0, 10 ] - trace_name: Error Prone - BEST_PRACTICES: - range_of_values: [ 0, 10 ] - trace_name: Best Practices - COMPLEXITY: - range_of_values: [ 0, 10 ] - trace_name: Complexity -``` - -### measurable.png - - -### maintainability_and_cohesion.png - - -### ratio.png - - -### countable.png - diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/__init__.py b/src/python/evaluation/paper_evaluation/issues_statistics/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/examples/countable.png b/src/python/evaluation/paper_evaluation/issues_statistics/examples/countable.png deleted file mode 100644 index 28f86363..00000000 Binary files a/src/python/evaluation/paper_evaluation/issues_statistics/examples/countable.png and /dev/null differ diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/examples/maintainability_and_cohesion.png b/src/python/evaluation/paper_evaluation/issues_statistics/examples/maintainability_and_cohesion.png deleted file mode 100644 index a037a49a..00000000 Binary files a/src/python/evaluation/paper_evaluation/issues_statistics/examples/maintainability_and_cohesion.png and /dev/null differ diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/examples/measurable.png b/src/python/evaluation/paper_evaluation/issues_statistics/examples/measurable.png deleted file mode 100644 index ae892783..00000000 Binary files a/src/python/evaluation/paper_evaluation/issues_statistics/examples/measurable.png and /dev/null differ diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/examples/ratio.png b/src/python/evaluation/paper_evaluation/issues_statistics/examples/ratio.png deleted file mode 100644 index 52d9953c..00000000 Binary files a/src/python/evaluation/paper_evaluation/issues_statistics/examples/ratio.png and /dev/null differ diff --git a/src/python/evaluation/paper_evaluation/issues_statistics/raw_issues_statistics_visualization.py b/src/python/evaluation/paper_evaluation/issues_statistics/raw_issues_statistics_visualization.py deleted file mode 100644 index 10450119..00000000 --- a/src/python/evaluation/paper_evaluation/issues_statistics/raw_issues_statistics_visualization.py +++ /dev/null @@ -1,207 +0,0 @@ -import argparse -import logging -import sys -from dataclasses import dataclass -from enum import Enum -from pathlib import Path -from typing import Dict, List, Optional - -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.plots.common.utils import get_supported_extensions, save_plot -from src.python.evaluation.plots.plotters.raw_issues_statistics_plotters import prepare_stats -from src.python.review.common.file_system import Extension, parse_yaml - -logger = logging.getLogger(__name__) -COLORWAY = ['rgb(47,22,84)', 'rgb(99,47,177)', 'rgb(153,110,216)'] - - -class _ConfigFields(Enum): - PLOT_CONFIG = 'plot_config' - ROWS = 'rows' - COLS = 'cols' - SPECS = 'specs' - HEIGHT = 'height' - WIDTH = 'width' - X_AXIS_NAME = 'x_axis_name' - Y_AXIS_NAME = 'y_axis_name' - - RANGE_OF_VALUES = 'range_of_values' - TRACE_NAME = 'trace_name' - - -_PLOT_CONFIG = _ConfigFields.PLOT_CONFIG.value -_ROWS = _ConfigFields.ROWS.value -_COLS = _ConfigFields.COLS.value -_SPECS = _ConfigFields.SPECS.value -_HEIGHT = _ConfigFields.HEIGHT.value -_WIDTH = _ConfigFields.WIDTH.value -_X_AXIS_NAME = _ConfigFields.X_AXIS_NAME.value -_Y_AXIS_NAME = _ConfigFields.Y_AXIS_NAME.value -_RANGE_OF_VALUES = _ConfigFields.RANGE_OF_VALUES.value -_TRACE_NAME = _ConfigFields.TRACE_NAME.value - - -@dataclass -class PlotConfig: - name: str - rows: int = 1 - cols: int = 1 - height: int = 800 - width: int = 1600 - x_axis_name: str = 'Value' - y_axis_name: str = 'Quantity' - specs: Optional[List] = None - - @staticmethod - def get_from_dict(plot_name: str, config: Dict) -> 'PlotConfig': - params = {'name': plot_name} - params.update(config) - return PlotConfig(**params) - - -@dataclass -class TraceConfig: - column: str - range_of_values: Optional[range] = None - trace_name: Optional[str] = None - - @staticmethod - def get_from_dict(column_name: str, config: Dict) -> 'TraceConfig': - params = {'column': column_name} - params.update(config) - - if _RANGE_OF_VALUES in params: - params[_RANGE_OF_VALUES] = range(*params[_RANGE_OF_VALUES]) - - return TraceConfig(**params) - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - 'stats_path', - type=lambda value: Path(value).absolute(), - help='Path to the statistics file. Must be an xlsx or csv file.', - ) - - parser.add_argument( - 'config_path', - type=lambda value: Path(value).absolute(), - help='Path to the yaml file containing information about the graphs to be plotted.', - ) - - parser.add_argument( - 'save_dir', - type=lambda value: Path(value).absolute(), - help='The directory where the plotted charts will be saved.', - ) - - parser.add_argument( - '--file-extension', - type=str, - default=Extension.SVG.value, - choices=get_supported_extensions(), - help='Allows you to select the extension of output files.', - ) - - -def _update_fig(fig: go.Figure, plot_config: PlotConfig) -> None: - fig.update_layout( - width=plot_config.width, - height=plot_config.height, - font_size=22, - paper_bgcolor='rgba(0,0,0,0)', - plot_bgcolor='rgba(0,0,0,0)', - colorway=COLORWAY, - ) - - axes_common_params = { - 'showline': True, - 'linewidth': 1, - 'linecolor': 'black', - 'mirror': True, - } - - fig.update_xaxes(title=plot_config.x_axis_name, **axes_common_params) - fig.update_yaxes(title=plot_config.y_axis_name, **axes_common_params) - - -def build_subplots(df: pd.DataFrame, plot_config: PlotConfig, trace_configs: List[TraceConfig]) -> go.Figure: - fig = make_subplots( - rows=plot_config.rows, - cols=plot_config.cols, - specs=plot_config.specs, - ) - - if plot_config.specs is None: - plot_config.specs = [[{} for _ in range(plot_config.cols)] for _ in range(plot_config.rows)] - - for row_index, row in enumerate(plot_config.specs, start=1): - for column_index, cell in enumerate(row, start=1): - if cell is None: - continue - - trace_config = trace_configs.pop(0) - - stats = prepare_stats( - df, - trace_config.column, - trace_config.range_of_values, - plot_config.x_axis_name, - plot_config.y_axis_name, - ) - - fig.add_scatter( - x=stats[plot_config.x_axis_name], - y=stats[plot_config.y_axis_name], - col=column_index, - row=row_index, - line={'width': 5}, - marker={'size': 10}, - name=trace_config.trace_name if trace_config.trace_name is not None else trace_config.column, - ) - - _update_fig(fig, plot_config) - - return fig - - -def plot_and_save(stats: pd.DataFrame, config: Dict, output_dir: Path, extension: Extension) -> None: - for group_name, group_config in config.items(): - plot_config = PlotConfig.get_from_dict(group_name, group_config.pop(_PLOT_CONFIG)) - trace_configs = [] - for column_name, column_config in group_config.items(): - trace_configs.append(TraceConfig.get_from_dict(column_name, column_config)) - subplots = build_subplots(stats, plot_config, trace_configs) - save_plot(subplots, output_dir, group_name, extension) - - -def main(): - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - - config = parse_yaml(args.config_path) - stats = get_solutions_df_by_file_path(args.stats_path) - - plot_and_save(stats, config, args.save_dir, Extension(args.file_extension)) - - return 0 - - except IndexError: - logger.error( - 'The number of traces must be consistent with the number of rows and columns, as well as the specs.', - ) - return 2 - - except Exception: - logger.exception('An unexpected error.') - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/survey_handler/README.md b/src/python/evaluation/paper_evaluation/survey_handler/README.md deleted file mode 100644 index 4afe932e..00000000 --- a/src/python/evaluation/paper_evaluation/survey_handler/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Surveys handlers - -These scripts allow handling surveys results for the SIGCSE paper. -We have two surveys (for Python and for Java) where participants should choose a fragments -that has better formatting. -Each question in the surveys have randomly orders for fragments. -The left fragment can have good formatting, but at the same time, it can have bad formatting. -To handle these cases we created JSON configs with this information and another one with the results. -These scripts allow processing these config files. - -## Usage - -Run the [survey_statistics_gathering.py](survey_statistics_gathering.py) with the arguments from command line. - -Required arguments: - -`questions_json_path` — path to the JSON with labelled questions; -`results_json_path` — path to the JSON with survey results. - -An example of `questions_json` file: -```json -{ - "questions": [ - { - "number": 1, - "left_fragment": "before_formatting", - "right_fragment": "after_formatting" - }, - { - "number": 2, - "left_fragment": "after_formatting", - "right_fragment": "before_formatting" - } - ] -} -``` - -An example of `results_json` file: - -```json -{ - "questions": [ - { - "number": 1, - "left_fragment": 0, - "right_fragment": 11, - "both": 0 - }, - { - "number": 2, - "left_fragment": 10, - "right_fragment": 0, - "both": 1 - } - ] -} -``` - -An example of the statistics: -```text -total participants=11 -------before----after----any---- -1. 0 11 0 -2. 1 10 0 -3. 0 11 0 -4. 0 11 0 -5. 0 11 0 -6. 1 10 0 -7. 0 11 0 -8. 1 8 2 -9. 0 11 0 -10. 0 8 3 -``` diff --git a/src/python/evaluation/paper_evaluation/survey_handler/__init__.py b/src/python/evaluation/paper_evaluation/survey_handler/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py deleted file mode 100644 index 8cfc898b..00000000 --- a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py +++ /dev/null @@ -1,61 +0,0 @@ -from dataclasses import dataclass -from enum import Enum, unique -from typing import Any, Dict, List - - -@dataclass -class Question: - with_formatting_count: int = 0 - without_formatting_count: int = 0 - any_formatting_count: int = 0 - - def get_total(self): - return self.with_formatting_count + self.without_formatting_count + self.any_formatting_count - - -@unique -class SurveyJsonField(Enum): - NUMBER = 'number' - LEFT_FRAGMENT = 'left_fragment' - RIGHT_FRAGMENT = 'right_fragment' - - BEFORE_FORMATTING = 'before_formatting' - BOTH = 'both' - - QUESTIONS = 'questions' - - -@dataclass -class SurveyStatistics: - questions: List[Question] - - def __init__(self, questions_json: List[Dict[str, Any]], results_json: List[Dict[str, int]]): - self.questions = [] - for result_json in results_json: - question_number = result_json[SurveyJsonField.NUMBER.value] - question = self.__find_json_question(questions_json, question_number) - if question[SurveyJsonField.LEFT_FRAGMENT.value] == SurveyJsonField.BEFORE_FORMATTING.value: - without_formatting_count = result_json[SurveyJsonField.LEFT_FRAGMENT.value] - with_formatting_count = result_json[SurveyJsonField.RIGHT_FRAGMENT.value] - else: - without_formatting_count = result_json[SurveyJsonField.RIGHT_FRAGMENT.value] - with_formatting_count = result_json[SurveyJsonField.LEFT_FRAGMENT.value] - any_formatting_count = result_json[SurveyJsonField.BOTH.value] - self.questions.append(Question(with_formatting_count, without_formatting_count, any_formatting_count)) - - @staticmethod - def __find_json_question(questions_json: List[Dict[str, Any]], question_number: int) -> Dict[str, Any]: - for question in questions_json: - if question[SurveyJsonField.NUMBER.value] == question_number: - return question - raise ValueError(f'Did not find question {question_number}') - - def print_stat(self): - if len(self.questions) == 0: - print('No questions found') - return - print(f'total participants={self.questions[0].get_total()}') - print('------before----after----any----') - for index, question in enumerate(self.questions): - print(f'{index + 1}.\t\t{question.without_formatting_count}\t\t{question.with_formatting_count}\t\t ' - f'{question.any_formatting_count}') diff --git a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py deleted file mode 100644 index 82b8b7d3..00000000 --- a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import json -import sys -from pathlib import Path - -from src.python.evaluation.evaluation_run_tool import logger -from src.python.evaluation.paper_evaluation.survey_handler.survey_statistics import SurveyJsonField, SurveyStatistics -from src.python.review.common.file_system import get_content_from_file - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('questions_json_path', - type=lambda value: Path(value).absolute(), - help='Path to the JSON with labelled questions') - - parser.add_argument('results_json_path', - type=lambda value: Path(value).absolute(), - help='Path to the JSON with survey results') - - -def main() -> int: - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - questions_json = json.loads(get_content_from_file(args.questions_json_path)) - results_json = json.loads(get_content_from_file(args.results_json_path)) - stat = SurveyStatistics( - questions_json[SurveyJsonField.QUESTIONS.value], - results_json[SurveyJsonField.QUESTIONS.value], - ) - stat.print_stat() - return 0 - - except FileNotFoundError: - logger.error('JSON file did not found') - return 2 - - except Exception: - logger.exception('An unexpected error.') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/README.md b/src/python/evaluation/paper_evaluation/user_dynamics/README.md deleted file mode 100644 index 7d004bb1..00000000 --- a/src/python/evaluation/paper_evaluation/user_dynamics/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Dynamics of student usage - -This module allows getting statistics about students dynamics in code quality issues improvements. - -## Usage - -Run the [dynamics_gathering.py](dynamics_gathering.py) with the arguments from command line. - -Required arguments: - -`solutions_file_path` — path to csv-file with code samples. - -In the result a file with students issues dynamics will be created. -We have three categories of dynamics: -- all (count of all code quality issues expect INFO issues) -- formatting (count of formatting code quality issues from CODE_STYLE category) -- other (all issues minus formatting issues) - -Each type of dynamics will be saved into a separated folder with csv files for each student. -Each csv file has only two columns: fragment id and issues count. - -An example of the csv file: -```text -issue_count,time -2,0 -20,1 -16,2 -15,3 -5,4 -5,5 -``` \ No newline at end of file diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/__init__.py b/src/python/evaluation/paper_evaluation/user_dynamics/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py deleted file mode 100644 index 6c4ab4ad..00000000 --- a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py +++ /dev/null @@ -1,91 +0,0 @@ -import argparse -import sys -from pathlib import Path -from typing import List - -import numpy as np -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import ( - filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger, -) -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn -from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension -from src.python.review.inspectors.issue import IssueType - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.SOLUTIONS_FILE_PATH.value) - - -ALL_ISSUES_COUNT = DynamicsColumn.ALL_ISSUES_COUNT.value -FORMATTING_ISSUES_COUNT = DynamicsColumn.FORMATTING_ISSUES_COUNT.value -OTHER_ISSUES_COUNT = DynamicsColumn.OTHER_ISSUES_COUNT.value - - -def __get_all_issues(traceback: str) -> List[PenaltyIssue]: - return list(filter(lambda i: i.type != IssueType.INFO, get_issues_from_json(traceback))) - - -def __get_formatting_issues(traceback: str) -> List[PenaltyIssue]: - return list(filter(lambda i: i.type == IssueType.CODE_STYLE, __get_all_issues(traceback))) - - -def __write_dynamics(output_path: Path, user_fragments: pd.DataFrame, index: int) -> None: - output_path.mkdir(parents=True, exist_ok=True) - user_fragments.columns = [DynamicsColumn.ISSUE_COUNT.value] - user_fragments[ColumnName.TIME.value] = np.arange(len(user_fragments)) - write_dataframe_to_csv(output_path / f'user_{index}{Extension.CSV.value}', user_fragments) - - -def __get_users_statistics(solutions_df: pd.DataFrame, output_path: Path) -> None: - users = solutions_df[ColumnName.USER.value].unique() - for index, user in enumerate(users): - user_df = filter_df_by_single_value(solutions_df, - ColumnName.USER.value, user).sort_values(ColumnName.TIME.value) - user_df[ALL_ISSUES_COUNT] = user_df.apply(lambda row: - len(__get_all_issues( - row[ColumnName.TRACEBACK.value])), - axis=1) - user_df[FORMATTING_ISSUES_COUNT] = user_df.apply(lambda row: - len(__get_formatting_issues( - row[ColumnName.TRACEBACK.value])), - axis=1) - user_df[OTHER_ISSUES_COUNT] = user_df[ALL_ISSUES_COUNT] - user_df[FORMATTING_ISSUES_COUNT] - - __write_dynamics(output_path / 'all', user_df[[ALL_ISSUES_COUNT]], index) - __write_dynamics(output_path / 'formatting', user_df[[FORMATTING_ISSUES_COUNT]], index) - __write_dynamics(output_path / 'other', user_df[[OTHER_ISSUES_COUNT]], index) - - -def main() -> int: - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - solutions_file_path = args.solutions_file_path - extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) - solutions_df = get_solutions_df(extension, solutions_file_path) - - output_path = get_parent_folder(Path(solutions_file_path)) / 'dynamics' - output_path.mkdir(parents=True, exist_ok=True) - __get_users_statistics(solutions_df, output_path) - return 0 - - except FileNotFoundError: - logger.error('CSV-file with the specified name does not exists.') - return 2 - - except Exception: - logger.exception('An unexpected error.') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py deleted file mode 100644 index b38ec54e..00000000 --- a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py +++ /dev/null @@ -1,96 +0,0 @@ -import argparse -import sys -from collections import Counter -from pathlib import Path -from statistics import median -from typing import List - -import pandas as pd -import plotly.express as px -from src.python.evaluation.common.pandas_util import logger -from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn -from src.python.review.common.file_system import ( - Extension, extension_file_condition, get_all_file_system_items, get_parent_folder, -) - -MEDIAN_COLUMN = 'Median number of code quality issues in submissions' -FREQ_COLUMN = 'Number of users' -TYPE = 'Submissions\' type' -PERCENTAGE = 'percentage' - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('dynamics_folder_path', - type=lambda value: Path(value).absolute(), - help='Folder with dynamics after embedding tool') - - parser.add_argument('old_dynamics_folder_path', - type=lambda value: Path(value).absolute(), - help='Folder with dynamics before embedding tool') - - -def __get_medians(dynamics_folder_path: Path) -> List[float]: - dynamics_paths = get_all_file_system_items(dynamics_folder_path, extension_file_condition(Extension.CSV)) - medians = [] - for dynamic in dynamics_paths: - dynamic_df = pd.read_csv(dynamic) - medians.append(int(median(dynamic_df[DynamicsColumn.ISSUE_COUNT.value]))) - return medians - - -def __group_medians(path_to_dynamics: Path, dynamics_type: str, threshold: int = 8) -> pd.DataFrame: - medians = __get_medians(path_to_dynamics) - grouped_medians = dict(Counter(medians)) - more_threshold = sum([freq for m, freq in grouped_medians.items() if m > threshold]) - others = {str(m): freq for m, freq in grouped_medians.items() if m <= threshold} - others[f'> {threshold}'] = more_threshold - new_df = pd.DataFrame(others.items(), columns=[MEDIAN_COLUMN, FREQ_COLUMN]) - new_df[TYPE] = dynamics_type - all_users = sum(new_df[FREQ_COLUMN]) - new_df[PERCENTAGE] = new_df.apply(lambda row: f'{round(row[FREQ_COLUMN] / all_users * 100)}%', axis=1) - return new_df - - -def main() -> int: - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - old_df = __group_medians(args.old_dynamics_folder_path, 'Before embedding tool') - new_df = __group_medians(args.dynamics_folder_path, 'After embedding tool') - union_df = old_df.append(new_df).sort_values([MEDIAN_COLUMN, TYPE], ascending=[True, False]) - - fig = px.bar(union_df, x=MEDIAN_COLUMN, y=FREQ_COLUMN, width=1000, height=800, color=TYPE, - color_discrete_sequence=['rgb(253,251,220)', 'rgb(47,22,84)']) - fig.update_layout(legend={ - 'yanchor': 'top', - 'y': 0.99, - 'xanchor': 'right', - 'x': 0.99, - }, - font_size=22, - barmode='group', - paper_bgcolor='rgba(0,0,0,0)', - plot_bgcolor='rgba(0,0,0,0)', - ) - # Add borders around plot - fig.update_xaxes(showline=True, linewidth=1, linecolor='black', mirror=True) - fig.update_yaxes(showline=True, linewidth=1, linecolor='black', mirror=True) - # Add borders around bars - fig.update_traces(marker_line_color='black', marker_line_width=1.5, opacity=0.9, - textposition='outside') - fig.update_layout(uniformtext_minsize=11, uniformtext_mode='hide') - - output_path = get_parent_folder(args.old_dynamics_folder_path) / f'evaluation_chart{Extension.PDF.value}' - fig.write_image(str(output_path)) - fig.show() - return 0 - - except Exception: - logger.exception('An unexpected error.') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py deleted file mode 100644 index 149016d7..00000000 --- a/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py +++ /dev/null @@ -1,72 +0,0 @@ -import argparse -import sys -import uuid -from pathlib import Path -from typing import List - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import get_solutions_df, logger -from src.python.evaluation.common.util import ColumnName -from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension - - -''' -This scripts allows unpacking solutions to the solutions dataframe. -The initial dataframe has only several obligatory columns user_id,times,codes, -where is an array with times separated by ; symbol and - is an array with code fragments separated by ₣ symbol. -The and arrays have to has the same length. -The resulting dataset will have several: columns user_id,time,code, -where each row contains obly one time and one code fragment -''' - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help='Path to the compressed solutions') - - -def __parse_time_and_solutions(times_str: str, solutions_str: str) -> pd.DataFrame: - times = times_str.split(',') - solutions = solutions_str.split('₣') - time_to_solution = dict(zip(times, solutions)) - user_df = pd.DataFrame(time_to_solution.items(), columns=[ColumnName.TIME.value, ColumnName.CODE.value]) - user_df[ColumnName.USER.value] = uuid.uuid4() - return user_df - - -def __add_user_df(user_df_list: List[pd.DataFrame], user_df: pd.DataFrame): - user_df_list.append(user_df) - - -def main() -> int: - parser = argparse.ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - solutions_file_path = args.solutions_file_path - extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) - solutions_df = get_solutions_df(extension, solutions_file_path) - user_df_list = [] - solutions_df.apply(lambda row: __add_user_df(user_df_list, - __parse_time_and_solutions(row['times'], row['codes'])), axis=1) - unpacked_solutions = pd.concat(user_df_list) - output_path = get_parent_folder(Path(solutions_file_path)) / f'unpacked_solutions{Extension.CSV.value}' - write_dataframe_to_csv(output_path, unpacked_solutions) - return 0 - - except FileNotFoundError: - logger.error('CSV-file with the specified name does not exists.') - return 2 - - except Exception: - logger.exception('An unexpected error.') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py b/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py deleted file mode 100644 index 7b1d4091..00000000 --- a/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py +++ /dev/null @@ -1,23 +0,0 @@ -from dataclasses import dataclass -from enum import Enum, unique -from typing import Dict, List - -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue - - -@unique -class DynamicsColumn(Enum): - ALL_ISSUES_COUNT = 'all_issues_count' - FORMATTING_ISSUES_COUNT = 'formatting_issues_count' - OTHER_ISSUES_COUNT = 'other_issues_count' - - ISSUE_COUNT = 'issue_count' - - -@dataclass -class UserStatistics: - traceback: List[List[PenaltyIssue]] - top_issues: Dict[str, int] - - def get_traceback_dynamics(self) -> List[int]: - return list(map(lambda i_l: len(i_l), self.traceback)) diff --git a/src/python/evaluation/plots/README.md b/src/python/evaluation/plots/README.md deleted file mode 100644 index be7e8797..00000000 --- a/src/python/evaluation/plots/README.md +++ /dev/null @@ -1,160 +0,0 @@ -# Hyperstyle evaluation: plots -This module allows you to visualize the data. - -## Diffs plotter -This script allows you to visualize a dataset obtained with [diffs_between_df.py](../inspectors/diffs_between_df.py). - -The script can build the following charts: -* number of unique issues by category ([Example](#number-of-unique-issues-by-category)) -* number of issues by category ([Example](#number-of-issues-by-category)) -* number of unique penalty issues by category ([Example](#number-of-unique-penalty-issues-by-category)) -* number of penalty issues by category ([Example](#number-of-penalty-issues-by-category)) -* median penalty influence by category ([Example](#median-influence-on-penalty-by-category)) -* distribution of penalty influence by category ([Example](#distribution-of-influence-on-penalty-by-category)) - -### Usage -Run the [diffs_plotter.py](diffs_plotter.py) with the arguments from command line. - -**Required arguments**: -1. `diffs_file_path` — path to a file with serialized diffs that were founded by [diffs_between_df.py](../inspectors/diffs_between_df.py). -2. `save_dir` — directory where the plotted charts will be saved. -3. `config_path` — path to the yaml file containing information about the charts to be plotted. A description of the config and its example is provided in [this section](#config). - - -**Optional arguments**: - -Argument | Description ---- | --- -**‑‑file‑extension** | Allows you to select the extension of output files. Available extensions: `.png`, `.jpg`, `.jpeg`, `.webp`, `.svg`, `.pdf`, `.eps`, `.json`. Default is `.svg`. - -### Config -The configuration file is a dictionary in yaml format, where each chart you want to build has its parameters. - -**Possible values of the charts**: -* `unique_issues_by_category` to plot the number of unique issues by category -* `issues_by_category` to plot the number of issues by category -* `unique_penalty_issues_by_category` to plot the number of unique penalty issues by category -* `penalty_issues_by_category` to plot the number of penalty issues by category -* `median_penalty_influence_by_category` to plot the median penalty influence by category -* `penalty_influence_distribution` to plot the distribution of penalty influence by category - -**Possible parameters**: -Parametr | Description ----|--- -**x_axis_name** | Name of the x-axis. The default value depends on the type of chart. -**y_axis_name** | Name of the y-axis. The default value depends on the type of chart. -**limit** | A value that allows you to filter the data before displaying them.

For charts `unique_issues_by_category`, `issues_by_category`, `unique_penalty_issues_by_category` and `penalty_issues_by_category` only those categories will be shown where the number of issues is greater than or equal to the limit.

For chart `penalty_issues_by_category` only those categories will be shown where the number of median value is greater than or equal to the limit.

For chart `penalty_influence_distribution` only those categories will be shown where the number of values is greater than or equal to the limit.

The default value depends on the type of chart. -**margin** | Defines the outer margin on all four sides of the chart. The available values are specified in the Enum class `MARGIN` from [plots const file](./common/plotly_consts.py). If not specified, the default value provided by Plotly is used. -**sort_order** | Defines the sorting order of the chart. The available values are specified in the Enum class `SORT_ORDER` from [plots const file](./common/plotly_consts.py). If not specified, the default value provided by Plotly is used. -**color** | Defines the color of the chart. The available values are specified in the Enum class `COLOR` from [plots const file](./common/plotly_consts.py). If not specified, the default value provided by Plotly is used. - -#### Example of config -```yaml -unique_issues_by_category: - margin: "ZERO" - limit: 10 - sort_order: "total descending" - color: "RED" -unique_penalty_issues_by_category: - limit: 30 - sort_order: "category ascending" -median_penalty_influence_by_category: -penalty_influence_distribution: -``` - -The result will be four graphs (`unique_issues_by_category`, `unique_penalty_issues_by_category`, `median_penalty_influence_by_category`, `penalty_influence_distribution`) with the corresponding parameters. - -### Examples - -#### Number of unique issues by category - - -#### Number of issues by category - - -#### Number of unique penalty issues by category - - -#### Number of penalty issues by category - - -#### Median influence on penalty by category - - -#### Distribution of influence on penalty by category - - -## Raw issues statistics plotter -This script allows you to visualize a dataset obtained with [get_raw_issues_statistics.py](../issues_statistics/get_raw_issues_statistics.py). - -The script can build the following charts: -* Line chart ([Example](#line-chart)) -* Box plot ([Example](#box-plot)) -* Histogram ([Example](#histogram)) - -### Usage -Run the [raw_issues_statistics_plotter.py](raw_issues_statistics_plotter.py) with the arguments from command line. - -**Required arguments**: -1. `config_path` — path to the yaml file containing information about the charts to be plotted. A description of the config and its example is provided in [this section](#config-1). -2. `save_dir` — directory where the plotted charts will be saved. - -**Optional arguments**: - -Argument | Description ---- | --- -**‑‑file‑extension** | Allows you to select the extension of output files. Available extensions: `.png`, `.jpg`, `.jpeg`, `.webp`, `.svg`, `.pdf`, `.eps`, `.json`. Default is `.svg`. -**‑‑group‑stats** | If present, there will be several languages on the charts at once. - -### Config -The configuration file is a dictionary in yaml format, where -1) paths to datasets with statistics are specified -2) for each column of the original dataset, the types of graphs to be plotted are specified. You can also put the general parameters when plotting multiple graphs for one column in a separate `common` group. - -**Possible values of the charts**: -* `line_chart` -* `histogram` -* `box_plot` - -**Possible parameters**: -Parametr | Description ----|--- -**x_axis_name** | Name of the x-axis. The default value depends on the type of chart. -**y_axis_name** | Name of the y-axis. The default value depends on the type of chart. -**boundaries** | Dictionary consisting of pairs `boundary value`: `boundary name` (boundary name may not exist). Allows to draw vertical or horizontal lines on graphs (depending on the type of plot). By default, the boundaries are not drawn. -**range_of_values** | Allows you to filter the values. It is an array of two values: a and b. Only values that belong to the range [a, b) are taken into account when plotting. By default, all values are taken into account when plotting. -**margin** | Defines the outer margin on all four sides of the chart. The available values are specified in the Enum class `MARGIN` from [plots const file](./common/plotly_consts.py). If not specified, the default value provided by Plotly is used. -**sort_order** | Defines the sorting order of the chart. The available values are specified in the Enum class `SORT_ORDER` from [plots const file](./common/plotly_consts.py). If not specified, the default value provided by Plotly is used. -**color** | Defines the color of the chart. The available values are specified in the Enum class `COLOR` from [plots const file](./common/plotly_consts.py). If not specified, the default value provided by Plotly is used. -**n_bins** | Allows you to adjust the number of bins when plotting a box plot. By default, this value is set by Plotly. - -#### Example of config -```yaml -CYCLOMATIC_COMPLEXITY: - line_chart: - x_axis_name: Cyclomatic complexity value - histigram: - common: - range_of_values: [0, 20] -``` - -The result will be two graphs: line chart and histogram. The values in both charts will be between 0 and 19 inclusive. In the line chart the x-axis will be named "Cyclomatic complexity value". - -### Examples - -#### Line chart -

- - -

- -#### Box plot -

- - -

- -#### Histogram -

- -

diff --git a/src/python/evaluation/plots/__init__.py b/src/python/evaluation/plots/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/plots/common/__init__.py b/src/python/evaluation/plots/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/plots/common/plotly_consts.py b/src/python/evaluation/plots/common/plotly_consts.py deleted file mode 100644 index 5f0e6d85..00000000 --- a/src/python/evaluation/plots/common/plotly_consts.py +++ /dev/null @@ -1,57 +0,0 @@ -from enum import Enum - -import plotly.express as px - - -class MARGIN(Enum): - ZERO = {'l': 0, 'r': 0, 'b': 0, 't': 0} - - -class SORT_ORDER(Enum): # noqa: N801 - CATEGORY_ASCENDING = 'category ascending' - CATEGORY_DESCENDING = 'category descending' - TOTAL_ASCENDING = 'total ascending' - TOTAL_DESCENDING = 'total descending' - - -class COLOR(Enum): - """ - Colors from px.colors.DEFAULT_PLOTLY_COLORS - """ - - BLUE = "rgb(31, 119, 180)" - ORANGE = "rgb(255, 127, 14)" - GREEN = "rgb(44, 160, 44)" - RED = "rgb(214, 39, 40)" - PURPLE = "rgb(148, 103, 189)" - BROWN = "rgb(140, 86, 75)" - PINK = "rgb(227, 119, 194)" - GRAY = "rgb(127, 127, 127)" - YELLOW = "rgb(188, 189, 34)" - CYAN = "rgb(23, 190, 207)" - - -class COLORWAY(Enum): # noqa: N801 - """ - Colors from px.colors.qualitative - """ - - PLOTLY = px.colors.qualitative.Plotly - D3 = px.colors.qualitative.D3 - G10 = px.colors.qualitative.G10 - T10 = px.colors.qualitative.T10 - ALPHABET = px.colors.qualitative.Alphabet - DARK24 = px.colors.qualitative.Dark24 - LIGHT24 = px.colors.qualitative.Light24 - SET1 = px.colors.qualitative.Set1 - PASTEL1 = px.colors.qualitative.Pastel1 - DARK2 = px.colors.qualitative.Dark2 - SET2 = px.colors.qualitative.Set2 - PASTEL2 = px.colors.qualitative.Pastel2 - SET3 = px.colors.qualitative.Set3 - ANTIQUE = px.colors.qualitative.Antique - BOLD = px.colors.qualitative.Bold - PASTEL = px.colors.qualitative.Pastel - PRISM = px.colors.qualitative.Prism - SAFE = px.colors.qualitative.Safe - VIVID = px.colors.qualitative.Vivid diff --git a/src/python/evaluation/plots/common/utils.py b/src/python/evaluation/plots/common/utils.py deleted file mode 100644 index 4ee1fb5c..00000000 --- a/src/python/evaluation/plots/common/utils.py +++ /dev/null @@ -1,188 +0,0 @@ -import os -from pathlib import Path -from typing import Dict, List, Optional - -import pandas as pd -import plotly.express as px -import plotly.graph_objects as go -from src.python.evaluation.plots.common import plotly_consts -from src.python.review.common.file_system import Extension - -COLOR = Optional[plotly_consts.COLOR] -COLORWAY = Optional[plotly_consts.COLORWAY] -MARGIN = Optional[plotly_consts.MARGIN] -SORT_ORDER = Optional[plotly_consts.SORT_ORDER] -LINES = Optional[Dict[int, Optional[str]]] - - -def get_supported_extensions() -> List[str]: - extensions = Extension.get_image_extensions() - extensions.append(Extension.JSON) - extensions.append(Extension.HTML) - return [extension.value for extension in extensions] - - -def create_bar_plot( - df: pd.DataFrame, - *, - x_axis: str, - y_axis: str, - margin: MARGIN = None, - sort_order: SORT_ORDER = None, - color: COLOR = None, -) -> go.Figure: - fig = px.bar(df, x=x_axis, y=y_axis, text=y_axis) - update_figure(fig, margin=margin, sort_order=sort_order, color=color) - return fig - - -def create_box_trace( - df: pd.DataFrame, - *, - x_column: Optional[str] = None, - y_column: Optional[str] = None, - color: COLOR = None, -) -> go.Box: - return go.Box( - x=df[x_column] if x_column is not None else None, - y=df[y_column] if y_column is not None else None, - line={'color': color.value if color is not None else None}, - ) - - -def create_box_plot( - df: pd.DataFrame, - *, - x_axis: Optional[str], - y_axis: Optional[str], - margin: MARGIN = None, - sort_order: SORT_ORDER = None, - color: COLOR = None, - horizontal_lines: LINES = None, -) -> go.Figure: - fig = go.Figure(create_box_trace(df, x_column=x_axis, y_column=y_axis, color=color)) - update_figure( - fig, - margin=margin, - sort_order=sort_order, - horizontal_lines=horizontal_lines, - x_axis_name=x_axis, - y_axis_name=y_axis, - ) - return fig - - -def create_scatter_trace( - df: pd.DataFrame, - *, - x_column: str, - y_column: str, - color: COLOR = None, -) -> go.Scatter: - return go.Scatter( - x=df[x_column], - y=df[y_column], - line={'color': color.value if color is not None else None}, - ) - - -def create_line_chart( - df: pd.DataFrame, - *, - x_axis: str, - y_axis: str, - margin: MARGIN = None, - color: COLOR = None, - vertical_lines: LINES = None, -) -> go.Figure: - fig = go.Figure(create_scatter_trace(df, x_column=x_axis, y_column=y_axis, color=color)) - update_figure( - fig, - margin=margin, - vertical_lines=vertical_lines, - x_axis_name=x_axis, - y_axis_name=y_axis, - ) - return fig - - -def create_histogram( - df: pd.DataFrame, - x_axis: str, - y_axis: str, - n_bins: Optional[int] = None, - margin: MARGIN = None, - color: COLOR = None, - vertical_lines: LINES = None, -) -> go.Figure: - fig = px.histogram(df, x=x_axis, y=y_axis, nbins=n_bins) - update_figure( - fig, - margin=margin, - color=color, - vertical_lines=vertical_lines, - x_axis_name=x_axis, - y_axis_name=y_axis, - ) - return fig - - -def update_figure( - fig: go.Figure, - *, - margin: MARGIN = None, - sort_order: SORT_ORDER = None, - color: COLOR = None, - colorway: COLORWAY = None, - horizontal_lines: LINES = None, - vertical_lines: LINES = None, - x_axis_name: Optional[str] = None, - y_axis_name: Optional[str] = None, -) -> None: - new_layout = {} - - if margin is not None: - new_layout["margin"] = margin.value - - if sort_order is not None: - new_layout["xaxis"] = {"categoryorder": sort_order.value} - - if x_axis_name is not None: - new_layout['xaxis_title'] = x_axis_name - - if y_axis_name is not None: - new_layout['yaxis_title'] = y_axis_name - - if colorway is not None: - new_layout['colorway'] = colorway.value - - fig.update_layout(**new_layout) - - new_trace = {} - - if color is not None: - new_trace["marker"] = {"color": color.value} - - fig.update_traces(**new_trace) - - if horizontal_lines is not None: - for y, annotation in horizontal_lines.items(): - fig.add_hline(y=y, annotation_text=annotation) - - if vertical_lines is not None: - for x, annotation in vertical_lines.items(): - fig.add_vline(x=x, annotation_text=annotation, annotation_textangle=90) - - -def save_plot( - fig: go.Figure, - dir_path: Path, - plot_name: str = "result_plot", - extension: Extension = Extension.SVG, -) -> None: - os.makedirs(dir_path, exist_ok=True) - file = dir_path / f"{plot_name}{extension.value}" - if extension == Extension.HTML: - fig.write_html(str(file)) - else: - fig.write_image(str(file)) diff --git a/src/python/evaluation/plots/diffs_plotter.py b/src/python/evaluation/plots/diffs_plotter.py deleted file mode 100644 index 5ecce6f7..00000000 --- a/src/python/evaluation/plots/diffs_plotter.py +++ /dev/null @@ -1,172 +0,0 @@ -import argparse -import sys -from enum import Enum, unique -from pathlib import Path -from typing import Any, Callable, Dict, Union - -sys.path.append('../../../..') - -import plotly.graph_objects as go -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.inspectors.common.statistics import ( - GeneralInspectorsStatistics, - IssuesStatistics, - PenaltyInfluenceStatistics, -) -from src.python.evaluation.inspectors.print_inspectors_statistics import gather_statistics -from src.python.evaluation.plots.common import plotly_consts -from src.python.evaluation.plots.common.utils import get_supported_extensions, save_plot -from src.python.evaluation.plots.plotters.diffs_plotters import ( - get_issues_by_category, - get_median_penalty_influence_by_category, - get_penalty_influence_distribution, - get_unique_issues_by_category, -) -from src.python.review.common.file_system import deserialize_data_from_file, Extension, parse_yaml - - -@unique -class ConfigFields(Enum): - X_AXIS_NAME = 'x_axis_name' - Y_AXIS_NAME = 'y_axis_name' - LIMIT = 'limit' - MARGIN = 'margin' - SORT_ORDER = 'sort_order' - COLOR = 'color' - - -X_AXIS_NAME = ConfigFields.X_AXIS_NAME.value -Y_AXIS_NAME = ConfigFields.Y_AXIS_NAME.value -LIMIT = ConfigFields.LIMIT.value -MARGIN = ConfigFields.MARGIN.value -SORT_ORDER = ConfigFields.SORT_ORDER.value -COLOR = ConfigFields.COLOR.value - - -@unique -class PlotTypes(Enum): - UNIQUE_ISSUES_BY_CATEGORY = 'unique_issues_by_category' - ISSUES_BY_CATEGORY = 'issues_by_category' - UNIQUE_PENALTY_ISSUES_BY_CATEGORY = 'unique_penalty_issues_by_category' - PENALTY_ISSUES_BY_CATEGORY = 'penalty_issues_by_category' - MEDIAN_PENALTY_INFLUENCE_BY_CATEGORY = 'median_penalty_influence_by_category' - PENALTY_INFLUENCE_DISTRIBUTION = 'penalty_influence_distribution' - - def to_plotter_function(self) -> Callable[..., go.Figure]: - type_to_function = { - PlotTypes.UNIQUE_ISSUES_BY_CATEGORY: get_unique_issues_by_category, - PlotTypes.ISSUES_BY_CATEGORY: get_issues_by_category, - PlotTypes.UNIQUE_PENALTY_ISSUES_BY_CATEGORY: get_unique_issues_by_category, - PlotTypes.PENALTY_ISSUES_BY_CATEGORY: get_issues_by_category, - PlotTypes.MEDIAN_PENALTY_INFLUENCE_BY_CATEGORY: get_median_penalty_influence_by_category, - PlotTypes.PENALTY_INFLUENCE_DISTRIBUTION: get_penalty_influence_distribution, - } - - return type_to_function[self] - - def extract_statistics( - self, - statistics: GeneralInspectorsStatistics, - ) -> Union[IssuesStatistics, PenaltyInfluenceStatistics]: - type_to_statistics = { - PlotTypes.UNIQUE_ISSUES_BY_CATEGORY: statistics.new_issues_stat, - PlotTypes.ISSUES_BY_CATEGORY: statistics.new_issues_stat, - PlotTypes.UNIQUE_PENALTY_ISSUES_BY_CATEGORY: statistics.penalty_issues_stat, - PlotTypes.PENALTY_ISSUES_BY_CATEGORY: statistics.penalty_issues_stat, - PlotTypes.MEDIAN_PENALTY_INFLUENCE_BY_CATEGORY: statistics.penalty_influence_stat, - PlotTypes.PENALTY_INFLUENCE_DISTRIBUTION: statistics.penalty_influence_stat, - } - - return type_to_statistics[self] - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - RunToolArgument.DIFFS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.DIFFS_FILE_PATH.value.description, - ) - - parser.add_argument( - 'save_dir', - type=lambda value: Path(value).absolute(), - help='The directory where the plotted charts will be saved', - ) - - parser.add_argument( - 'config_path', - type=lambda value: Path(value).absolute(), - help='Path to the yaml file containing information about the graphs to be plotted.', - ) - - parser.add_argument( - '--file-extension', - type=str, - default=Extension.SVG.value, - choices=get_supported_extensions(), - help='Allows you to select the extension of output files', - ) - - -def get_plot_params(config: Dict, plot_type: PlotTypes) -> Dict[str, Any]: - config_params = config.get(plot_type.value) - params = {} - - if config_params is None: - return params - - if config_params.get(MARGIN) is not None: - margin_value = config_params.get(MARGIN).upper() - params[MARGIN] = plotly_consts.MARGIN[margin_value] - - if config_params.get(SORT_ORDER) is not None: - sort_order_value = config_params.get(SORT_ORDER) - params[SORT_ORDER] = plotly_consts.SORT_ORDER(sort_order_value) - - if config_params.get(LIMIT) is not None: - params[LIMIT] = config_params.get(LIMIT) - - if config_params.get(X_AXIS_NAME) is not None: - params[X_AXIS_NAME] = config_params.get(X_AXIS_NAME) - - if config_params.get(Y_AXIS_NAME) is not None: - params[Y_AXIS_NAME] = config_params.get(Y_AXIS_NAME) - - if config_params.get(COLOR) is not None: - color_value = config_params.get(COLOR) - params[COLOR] = plotly_consts.COLOR[color_value] - - return params - - -def plot_and_save( - config: Dict, - general_statistics: GeneralInspectorsStatistics, - save_dir: Path, - extension: Extension, -) -> None: - for plot_type in PlotTypes: - if plot_type.value in config: - params = get_plot_params(config, plot_type) - plotter_function = plot_type.to_plotter_function() - statistics = plot_type.extract_statistics(general_statistics) - plot = plotter_function(statistics, **params) - save_plot(plot, save_dir, plot_name=plot_type.value, extension=extension) - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - diffs = deserialize_data_from_file(args.diffs_file_path) - general_statistics = gather_statistics(diffs) - - extension = Extension(args.file_extension) - config = parse_yaml(args.config_path) - - plot_and_save(config, general_statistics, args.save_dir, extension) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/plots/examples/BEST_PRACTICES_box_plot.png b/src/python/evaluation/plots/examples/BEST_PRACTICES_box_plot.png deleted file mode 100644 index 94b59e74..00000000 Binary files a/src/python/evaluation/plots/examples/BEST_PRACTICES_box_plot.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/BEST_PRACTICES_box_plot_grouped.png b/src/python/evaluation/plots/examples/BEST_PRACTICES_box_plot_grouped.png deleted file mode 100644 index 7965aa25..00000000 Binary files a/src/python/evaluation/plots/examples/BEST_PRACTICES_box_plot_grouped.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/CODE_STYLE_ratio_histogram.png b/src/python/evaluation/plots/examples/CODE_STYLE_ratio_histogram.png deleted file mode 100644 index e3dde849..00000000 Binary files a/src/python/evaluation/plots/examples/CODE_STYLE_ratio_histogram.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/CYCLOMATIC_COMPLEXITY_line_chart.png b/src/python/evaluation/plots/examples/CYCLOMATIC_COMPLEXITY_line_chart.png deleted file mode 100644 index 17673eb6..00000000 Binary files a/src/python/evaluation/plots/examples/CYCLOMATIC_COMPLEXITY_line_chart.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/CYCLOMATIC_COMPLEXITY_line_chart_grouped.png b/src/python/evaluation/plots/examples/CYCLOMATIC_COMPLEXITY_line_chart_grouped.png deleted file mode 100644 index 044e92c8..00000000 Binary files a/src/python/evaluation/plots/examples/CYCLOMATIC_COMPLEXITY_line_chart_grouped.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/issues_by_category.png b/src/python/evaluation/plots/examples/issues_by_category.png deleted file mode 100644 index 3e55aa12..00000000 Binary files a/src/python/evaluation/plots/examples/issues_by_category.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/median_penalty_influence_by_category.png b/src/python/evaluation/plots/examples/median_penalty_influence_by_category.png deleted file mode 100644 index 94d5baf9..00000000 Binary files a/src/python/evaluation/plots/examples/median_penalty_influence_by_category.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/penalty_influence_distribution.png b/src/python/evaluation/plots/examples/penalty_influence_distribution.png deleted file mode 100644 index 6fbdce22..00000000 Binary files a/src/python/evaluation/plots/examples/penalty_influence_distribution.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/penalty_issues_by_category.png b/src/python/evaluation/plots/examples/penalty_issues_by_category.png deleted file mode 100644 index 3e55aa12..00000000 Binary files a/src/python/evaluation/plots/examples/penalty_issues_by_category.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/unique_issues_by_category.png b/src/python/evaluation/plots/examples/unique_issues_by_category.png deleted file mode 100644 index ab69b22f..00000000 Binary files a/src/python/evaluation/plots/examples/unique_issues_by_category.png and /dev/null differ diff --git a/src/python/evaluation/plots/examples/unique_penalty_issues_by_category.png b/src/python/evaluation/plots/examples/unique_penalty_issues_by_category.png deleted file mode 100644 index ab69b22f..00000000 Binary files a/src/python/evaluation/plots/examples/unique_penalty_issues_by_category.png and /dev/null differ diff --git a/src/python/evaluation/plots/plotters/__init__.py b/src/python/evaluation/plots/plotters/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/plots/plotters/diffs_plotters.py b/src/python/evaluation/plots/plotters/diffs_plotters.py deleted file mode 100644 index 0ff94b64..00000000 --- a/src/python/evaluation/plots/plotters/diffs_plotters.py +++ /dev/null @@ -1,161 +0,0 @@ -from statistics import median -from typing import Any, Callable, Dict, Optional - -import pandas as pd -import plotly.graph_objects as go -from src.python.evaluation.inspectors.common.statistics import IssuesStatistics, PenaltyInfluenceStatistics -from src.python.evaluation.plots.common import plotly_consts -from src.python.evaluation.plots.common.utils import create_bar_plot, create_box_plot -from src.python.review.inspectors.issue import IssueType - - -def _get_dataframe_from_dict( - data_dict: Dict[Any, Any], - key_name: str, - value_name: str, - key_mapper: Callable = lambda x: x, - value_mapper: Callable = lambda y: y, -): - """ - Converts 'data_dict' to a dataframe consisting of two columns: 'key_name', 'value_name'. - 'key_name' contains all keys of 'data_dict', 'value_name' contains all corresponding - values of 'data_dict'. With the functions 'key_mapper' and 'value_mapper' you can - additionally convert keys and values respectively. - """ - converted_dict = { - key_name: list(map(key_mapper, data_dict.keys())), - value_name: list(map(value_mapper, data_dict.values())), - } - - return pd.DataFrame.from_dict(converted_dict) - - -def _extract_stats_from_issues_statistics( - statistics: IssuesStatistics, - limit: int, - only_unique: bool, -) -> Dict[IssueType, int]: - categorized_statistics = statistics.get_short_categorized_statistics() - - # If you want to get only unique issues, you should use position 0 of the tuple, otherwise 1. - position = int(not only_unique) - - return { - issue_type: stat[position] for issue_type, stat in categorized_statistics.items() if stat[position] >= limit - } - - -def get_unique_issues_by_category( - statistics: IssuesStatistics, - x_axis_name: str = 'Categories', - y_axis_name: str = 'Number of unique issues', - limit: int = 0, - margin: Optional[plotly_consts.MARGIN] = None, - sort_order: Optional[plotly_consts.SORT_ORDER] = None, - color: Optional[plotly_consts.COLOR] = None, -) -> go.Figure: - filtered_stats = _extract_stats_from_issues_statistics(statistics, limit, only_unique=True) - - df = _get_dataframe_from_dict( - filtered_stats, - key_name=x_axis_name, - value_name=y_axis_name, - key_mapper=lambda issue_type: issue_type.name, - ) - - return create_bar_plot( - df, - x_axis=x_axis_name, - y_axis=y_axis_name, - margin=margin, - sort_order=sort_order, - color=color, - ) - - -def get_issues_by_category( - statistics: IssuesStatistics, - x_axis_name: str = 'Categories', - y_axis_name: str = 'Number of issues', - limit: int = 0, - margin: Optional[plotly_consts.MARGIN] = None, - sort_order: Optional[plotly_consts.SORT_ORDER] = None, - color: Optional[plotly_consts.COLOR] = None, -) -> go.Figure: - filtered_stats = _extract_stats_from_issues_statistics(statistics, limit, only_unique=False) - - df = _get_dataframe_from_dict( - filtered_stats, - key_name=x_axis_name, - value_name=y_axis_name, - key_mapper=lambda issue_type: issue_type.name, - ) - - return create_bar_plot( - df, - x_axis=x_axis_name, - y_axis=y_axis_name, - margin=margin, - sort_order=sort_order, - color=color, - ) - - -def get_median_penalty_influence_by_category( - statistics: PenaltyInfluenceStatistics, - x_axis_name: str = 'Categories', - y_axis_name: str = 'Penalty influence (%)', - limit: int = 0, - margin: Optional[plotly_consts.MARGIN] = None, - sort_order: Optional[plotly_consts.SORT_ORDER] = None, - color: Optional[plotly_consts.COLOR] = None, -) -> go.Figure: - stat = statistics.stat - filtered_stats = {issue_type: influence for issue_type, influence in stat.items() if median(influence) >= limit} - - df = _get_dataframe_from_dict( - filtered_stats, - key_name=x_axis_name, - value_name=y_axis_name, - key_mapper=lambda issue_type: issue_type.name, - value_mapper=lambda influence: median(influence), - ) - - return create_bar_plot( - df, - x_axis=x_axis_name, - y_axis=y_axis_name, - margin=margin, - sort_order=sort_order, - color=color, - ) - - -def get_penalty_influence_distribution( - statistics: PenaltyInfluenceStatistics, - x_axis_name: str = 'Categories', - y_axis_name: str = 'Penalty influence (%)', - limit: int = 0, - margin: Optional[plotly_consts.MARGIN] = None, - sort_order: Optional[plotly_consts.SORT_ORDER] = None, - color: Optional[plotly_consts.COLOR] = None, -): - stat = statistics.stat - filtered_stats = {issue_type: influence for issue_type, influence in stat.items() if len(influence) >= limit} - - df = _get_dataframe_from_dict( - filtered_stats, - key_name=x_axis_name, - value_name=y_axis_name, - key_mapper=lambda issue_type: issue_type.name, - ) - df = df.explode(y_axis_name) - - return create_box_plot( - df, - x_axis=x_axis_name, - y_axis=y_axis_name, - margin=margin, - sort_order=sort_order, - color=color, - ) diff --git a/src/python/evaluation/plots/plotters/raw_issues_statistics_plotters.py b/src/python/evaluation/plots/plotters/raw_issues_statistics_plotters.py deleted file mode 100644 index 4e316e06..00000000 --- a/src/python/evaluation/plots/plotters/raw_issues_statistics_plotters.py +++ /dev/null @@ -1,234 +0,0 @@ -import logging -from dataclasses import dataclass -from enum import Enum, unique -from typing import Callable, Dict, Optional, Tuple - -import numpy as np -import pandas as pd -import plotly.graph_objects as go -from src.python.evaluation.issues_statistics.get_raw_issues_statistics import VALUE -from src.python.evaluation.plots.common.utils import ( - COLOR, - COLORWAY, - create_box_plot, - create_box_trace, - create_histogram, - create_line_chart, - create_scatter_trace, - LINES, - MARGIN, - update_figure, -) - -logger = logging.getLogger(__name__) - - -@unique -class PlotTypes(Enum): - LINE_CHART = 'line_chart' - HISTOGRAM = 'histogram' - BOX_PLOT = 'box_plot' - - def to_plotter_function(self) -> Callable[[Dict[str, pd.DataFrame], 'PlotConfig', bool], Dict[str, go.Figure]]: - type_to_function = { - PlotTypes.LINE_CHART: plot_line_chart, - PlotTypes.HISTOGRAM: plot_histogram, - PlotTypes.BOX_PLOT: plot_box_plot, - } - - return type_to_function[self] - - -@dataclass -class PlotConfig: - column: str - type: PlotTypes - x_axis_name: Optional[str] = None - y_axis_name: Optional[str] = None - margin: MARGIN = None - color: COLOR = None - colorway: COLORWAY = None - boundaries: LINES = None - range_of_values: Optional[range] = None - n_bins: Optional[int] = None - - -def prepare_stats( - stats: pd.DataFrame, - column: str, - range_of_values: Optional[range], - x_axis_name: str, - y_axis_name: str, -) -> pd.DataFrame: - result_df = stats[[VALUE, column]] - - if range_of_values is not None: - result_df = result_df[result_df[VALUE].isin(range_of_values)] - - result_df.set_index(VALUE, inplace=True) - - # Trim trailing zeros - result_df = result_df.apply(lambda column: np.trim_zeros(column, trim='b')).dropna() - - # Fill in the missing intermediate values with zeros - min_index, max_index = result_df.index.min(), result_df.index.max() - if pd.isna(min_index) or pd.isna(max_index): - logger.warning(f'{column}: no data') - else: - result_df = result_df.reindex(range(min_index, max_index + 1), fill_value=0) - - result_df.reset_index(inplace=True) - - return result_df.rename(columns={VALUE: x_axis_name, column: y_axis_name}) - - -def _get_axis_names( - *, - x_axis_name: Optional[str], - y_axis_name: Optional[str], - default_x_axis_name: str, - default_y_axis_name: str, -) -> Tuple[str, str]: - new_x_axis_name = default_x_axis_name - if x_axis_name is not None: - new_x_axis_name = x_axis_name - - new_y_axis_name = default_y_axis_name - if y_axis_name is not None: - new_y_axis_name = y_axis_name - - return new_x_axis_name, new_y_axis_name - - -def plot_line_chart( - stats_by_lang: Dict[str, pd.DataFrame], - config: PlotConfig, - group_stats: bool, -) -> Dict[str, go.Figure]: - x_axis_name, y_axis_name = _get_axis_names( - x_axis_name=config.x_axis_name, - y_axis_name=config.y_axis_name, - default_x_axis_name='Value', - default_y_axis_name='Quantity', - ) - - if not group_stats: - plots = {} - for lang, stats in stats_by_lang.items(): - stats = prepare_stats(stats, config.column, config.range_of_values, x_axis_name, y_axis_name) - plots[lang] = create_line_chart( - stats, - x_axis=x_axis_name, - y_axis=y_axis_name, - color=config.color, - margin=config.margin, - vertical_lines=config.boundaries, - ) - return plots - - plot = go.Figure() - for lang, stats in stats_by_lang.items(): - stats = prepare_stats(stats, config.column, config.range_of_values, x_axis_name, y_axis_name) - trace = create_scatter_trace(stats, x_column=x_axis_name, y_column=y_axis_name) - trace.name = lang - plot.add_trace(trace) - - update_figure( - plot, - margin=config.margin, - vertical_lines=config.boundaries, - x_axis_name=x_axis_name, - y_axis_name=y_axis_name, - colorway=config.colorway, - ) - - return {'grouped': plot} - - -def plot_histogram( - stats_by_lang: Dict[str, pd.DataFrame], - config: PlotConfig, - group_stats: bool, -) -> Dict[str, go.Figure]: - x_axis_name, y_axis_name = _get_axis_names( - x_axis_name=config.x_axis_name, - y_axis_name=config.y_axis_name, - default_x_axis_name='Value', - default_y_axis_name='Quantity', - ) - - if group_stats: - logger.info(f'{config.column}: the histogram cannot be grouped.') - - plots = {} - for lang, stats in stats_by_lang.items(): - stats = prepare_stats(stats, config.column, config.range_of_values, x_axis_name, y_axis_name) - plots[lang] = create_histogram( - stats, - x_axis_name, - y_axis_name, - margin=config.margin, - color=config.color, - n_bins=config.n_bins, - vertical_lines=config.boundaries, - ) - - return plots - - -def _get_values_df(stats: pd.DataFrame, config: PlotConfig, x_axis_name: str, y_axis_name: str): - values = [] - stats.apply(lambda row: values.extend([row[VALUE]] * row[config.column]), axis=1) - - if config.range_of_values is not None: - values = [elem for elem in values if elem in config.range_of_values] - - return pd.DataFrame.from_dict({x_axis_name: config.column, y_axis_name: values}) - - -def plot_box_plot( - stats_by_lang: Dict[str, pd.DataFrame], - config: PlotConfig, - group_stats: bool, -) -> Dict[str, go.Figure]: - x_axis_name, y_axis_name = _get_axis_names( - x_axis_name=config.x_axis_name, - y_axis_name=config.y_axis_name, - default_x_axis_name='Category', - default_y_axis_name='Values', - ) - - if not group_stats: - plots = {} - for lang, stats in stats_by_lang.items(): - values_df = _get_values_df(stats, config, x_axis_name, y_axis_name) - - plots[lang] = create_box_plot( - values_df, - x_axis=x_axis_name, - y_axis=y_axis_name, - color=config.color, - margin=config.margin, - horizontal_lines=config.boundaries, - ) - return plots - - plot = go.Figure() - for lang, stats in stats_by_lang.items(): - values_df = _get_values_df(stats, config, x_axis_name, y_axis_name) - - trace = create_box_trace(values_df, y_column=y_axis_name) - trace.name = lang - - plot.add_trace(trace) - - update_figure( - plot, - margin=config.margin, - horizontal_lines=config.boundaries, - x_axis_name=x_axis_name, - y_axis_name=y_axis_name, - colorway=config.colorway, - ) - - return {'grouped': plot} diff --git a/src/python/evaluation/plots/raw_issues_statistics_plotter.py b/src/python/evaluation/plots/raw_issues_statistics_plotter.py deleted file mode 100644 index 6d8295ff..00000000 --- a/src/python/evaluation/plots/raw_issues_statistics_plotter.py +++ /dev/null @@ -1,150 +0,0 @@ -import argparse -import logging -import sys -from enum import Enum, unique -from pathlib import Path -from typing import Dict, List, Optional - -sys.path.append('../../../..') - -import plotly.graph_objects as go -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.plots.common import plotly_consts -from src.python.evaluation.plots.common.utils import ( - get_supported_extensions, - save_plot, -) -from src.python.evaluation.plots.plotters.raw_issues_statistics_plotters import PlotConfig, PlotTypes -from src.python.review.common.file_system import Extension, parse_yaml - - -@unique -class ConfigFields(Enum): - X_AXIS_NAME = 'x_axis_name' - Y_AXIS_NAME = 'y_axis_name' - MARGIN = 'margin' - COLOR = 'color' - COLORWAY = 'colorway' - BOUNDARIES = 'boundaries' - COMMON = 'common' - STATS = 'stats' - RANGE_OF_VALUES = 'range_of_values' - N_BINS = 'n_bins' - - -X_AXIS_NAME = ConfigFields.X_AXIS_NAME.value -Y_AXIS_NAME = ConfigFields.Y_AXIS_NAME.value -MARGIN = ConfigFields.MARGIN.value -COLOR = ConfigFields.COLOR.value -COLORWAY = ConfigFields.COLORWAY.value -BOUNDARIES = ConfigFields.BOUNDARIES.value -COMMON = ConfigFields.COMMON.value -STATS = ConfigFields.STATS.value -RANGE_OF_VALUES = ConfigFields.RANGE_OF_VALUES.value -N_BINS = ConfigFields.N_BINS.value - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - 'config_path', - type=lambda value: Path(value).absolute(), - help='Path to the yaml file containing information about the graphs to be plotted.', - ) - - parser.add_argument( - 'save_dir', - type=lambda value: Path(value).absolute(), - help='The directory where the plotted charts will be saved.', - ) - - parser.add_argument( - '--file-extension', - type=str, - default=Extension.SVG.value, - choices=get_supported_extensions(), - help='Allows you to select the extension of output files.', - ) - - parser.add_argument( - '--group-stats', - action='store_true', - help='If present, there will be several languages on the charts at once.', - ) - - -def _get_plot_config( - column_name: str, - plot_type: str, - plot_config: Optional[Dict], - common: Optional[Dict], -) -> PlotConfig: - params = {'column': column_name, 'type': PlotTypes(plot_type.lower())} - - if common is not None: - params.update(common) - - if plot_config is not None: - params.update(plot_config) - - if MARGIN in params: - margin_value = params.get(MARGIN).upper() - params[MARGIN] = plotly_consts.MARGIN[margin_value] - - if COLOR in params: - color_value = params.get(COLOR).upper() - params[COLOR] = plotly_consts.COLOR[color_value] - - if RANGE_OF_VALUES in params: - params[RANGE_OF_VALUES] = range(*params[RANGE_OF_VALUES]) - - if COLORWAY in params: - colorway_value = params.get(COLORWAY).upper() - params[COLORWAY] = plotly_consts.COLORWAY[colorway_value] - - return PlotConfig(**params) - - -def get_plot_configs(column_name: str, column_config: Dict) -> List[PlotConfig]: - common = column_config.pop(COMMON, None) - - plot_configs = [] - for plot_type, plot_config in column_config.items(): - plot_configs.append(_get_plot_config(column_name, plot_type, plot_config, common)) - - return plot_configs - - -def _save_plots(plots: Dict[str, go.Figure], save_dir: Path, extension: Extension, column: str, plot_type: str) -> None: - for output_name, plot in plots.items(): - subdir = save_dir / column - save_plot(plot, subdir, plot_name=f'{column}_{plot_type}_{output_name}', extension=extension) - - -def plot_and_save(config: Dict, save_dir: Path, extension: Extension, group_stats: bool) -> None: - stats_by_lang = { - lang: get_solutions_df_by_file_path(Path(lang_stats)) for lang, lang_stats in config.pop(STATS).items() - } - - for column_name, column_config in config.items(): - plot_configs = get_plot_configs(column_name, column_config) - for plot_config in plot_configs: - plotter_function = plot_config.type.to_plotter_function() - plots = plotter_function(stats_by_lang, plot_config, group_stats) - _save_plots(plots, save_dir, extension, plot_config.column, plot_config.type.value) - - -def main(): - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO) - - extension = Extension(args.file_extension) - config = parse_yaml(args.config_path) - - plot_and_save(config, args.save_dir, extension, args.group_stats) - - -if __name__ == "__main__": - main() diff --git a/src/python/evaluation/qodana/README.md b/src/python/evaluation/qodana/README.md deleted file mode 100644 index 0e81af0f..00000000 --- a/src/python/evaluation/qodana/README.md +++ /dev/null @@ -1,311 +0,0 @@ -# Dataset labelling -This script allows you to label a dataset using the found [Qodana](https://github.com/JetBrains/Qodana) inspections. - -The dataset must contain at least three columns: `id`, `code` and `lang`, where `id` is a unique solution number, `lang` is the language in which the code is written in the `code` column. The `lang` must belong to one of the following values: `java7`, `java8`, `java9`, `java11`, `python3`, `kotlin`. If `lang` is not equal to any of the values, the row will be skipped. - -The dataset must have the format `csv`. The labeled dataset is also in `csv` format, with a new column `inspections` added, which contains a list of all found inspections. - -# Usage -Run the [dataset_labeling.py](dataset_labeling.py) with the arguments from command line. - -### Required arguments - -`dataset_path` — path to dataset. - -### Optional arguments -| Argument | Description | -|-|-| -| **‑c**, **‑‑config** | Path to qodana.yaml. If the path is not specified, Qodana will start without a configuration file. | -| **‑l**, **‑‑limit** | Allows you to read only the specified number of first rows from the dataset. If no limit is specified, the whole dataset will be processed. | -| **‑s**, **‑‑chunk‑size** | The number of files that Qodana will process at a time. Default is `5000`. | -| **‑o**, **‑‑output‑path** | The path where the labeled dataset will be saved. If not specified, the original dataset will be overwritten. | - ---- - -# Preprocessing - -The model that imitates Qodana analysis gets input from a dataset in a special format. -This module allows preparing datasets that were graded by [dataset_labeling.py](dataset_labeling.py) script. - -Data processing consists of several stages: -- union several `csv` files that were graded by [dataset_labeling.py](dataset_labeling.py) script - and filter inspections list if it is necessary; -- get all unique inspections from the dataset; -- convert `csv` file into a special format. - -## Filter inspections - -This stage allow you to union several `csv` files that were graded by [dataset_labeling.py](dataset_labeling.py) script - and filter inspections list if it is necessary. - -Please, note that your all input files must be graded by [dataset_labeling.py](dataset_labeling.py) script -and have `inspections` column. - -Output file is a new `csv` file with the all columns from the input files. - -#### Usage - -Run the [filter_inspections.py](filter_inspections.py) with the arguments from command line. - -Required arguments: - -`dataset_folder` — path to a folder with csv files graded by Qodana. Each file must have `inspections` column. - -Optional arguments: -Argument | Description ---- | --- -|**‑i**, **‑‑inspections**| Set of inspections ids to exclude from the dataset separated by comma. By default all inspections remain. | - -The resulting file will be stored in the `dataset_folder`. - -___ - -## Get all unique inspections - -This stage allow you to get all unique inspections from a `csv` file graded by Qodana. -Please, note that your input file must be graded by [dataset_labeling.py](dataset_labeling.py) script -and has `inspections` column. - -Output file is a new `csv` file with four columns: `id`, `inspection_id`, `count_all`, `count_uniq`. -`id` is unique number for each inspection, minimal value is 1. -`inspection_id` is unique Qoadana id for each inspection. -`count_all` count all fragments where was this inspection (with duplicates). -`count_uniq` count all fragments where was this inspection (without duplicates). - -#### Usage - -Run the [get_unique_inspectors.py](get_unique_inspectors.py) with the arguments from command line. - -Required arguments: - -`solutions_file_path` — path to csv-file with code samples graded by [dataset_labeling.py](dataset_labeling.py) script. - -Optional arguments: -Argument | Description ---- | --- -|**‑‑uniq**| To count all fragments for each inspection where was this inspection (without duplicates). By default it disabled. | - -The resulting file will be stored in the same folder as the input file. - -An example of the output file: - -```json -id | inspection_id | count_all | count_unique ------|---------------------|--------------|-------------- -1 | SystemOutErr | 5 | 2 -2 | ConstantExpression | 1 | 1 -``` - -___ - -#### Convert `csv` file into a special format - -This block describes what format can be converted csv-file with code samples -graded by [dataset_labeling.py](dataset_labeling.py) script. - -We have two different formats: -- fragment to inspections list; -- fragment to inspections list with positions. - - -#### Fragment to inspections list - -This data representation match code fragments to a list with ids of inspections. - -Please, note that your input file must be graded by [dataset_labeling.py](dataset_labeling.py) script -and has `inspections` column. - -Output file is a new `csv` file with a new `inspections` column with list with ids of inspections. -If the list of inspections for the fragment is empty, then write 0. - -#### Usage - -Run the [fragment_to_inspections_list.py](fragment_to_inspections_list.py) with the arguments from command line. - -Required arguments: - -- `solutions_file_path` — path to csv-file with code samples graded by [dataset_labeling.py](dataset_labeling.py) script, -- `inspections_path` — path to csv-file with inspections list from the input file. You can get this file by [get_unique_inspectors.py](get_unique_inspectors.py) script. - -Optional arguments: -Argument | Description ---- | --- -|**‑‑remove‑duplicates**| Remove duplicates around inspections in each row. Default value is `False`. | - -The resulting file will be stored in the same folder as the input file. - -An example of the input file: - -```json -id | code | lang | inspections ------|-------------------|---------------|----------------- -2 | "// some code" | java11 | "{""issues"": []}" -3 | "// some code" | java11 | "{""issues"": [""{\"... \""problem_id\"": \""SystemOutErr\""}""]}" -0 | "// some code" | java11 | "{""issues"": [""{\"...\""problem_id\"": \""ConstantExpression\""}"",""{\"...\""problem_id\"": \""ConstantExpression\""}""]}" -1 | "// some code" | java11 | "{""issues"": []}" -``` - -with the inspections file: - -```json -id | inspection_id ------|------------------- -1 | SystemOutErr -2 | ConstantExpression -``` - -An example of the output file: - -```json -id | code | lang | inspections ------|-------------------|---------------|----------------- -2 | "// some code" | java11 | 0 -3 | "// some code" | java11 | 1 -0 | "// some code" | java11 | 2,2 -1 | "// some code" | java11 | 0 - -``` - ---- - -#### Fragment to inspections list with positions - -This data representation match each line in code fragments to a list with ids of inspections in this line. - -Please, note that your input file must be graded by [dataset_labeling.py](dataset_labeling.py) script -and has `inspections` column. - -Output file is a new `csv` file with a new `inspections` column with list with ids of inspections. -If the list of inspections for the fragment is empty, then write 0. -Note, that each line in code fragments in the new file is stored in a separate row. -All indents as well as blank lines are keeped. - -#### Usage - -Run the [fragment_to_inspections_list_line_by_line.py](fragment_to_inspections_list_line_by_line.py) with the arguments from command line. - -Required arguments: - -- `solutions_file_path` — path to csv-file with code samples graded by [dataset_labeling.py](dataset_labeling.py) script, -- `inspections_path` — path to csv-file with inspections list from the input file. You can get this file by [get_unique_inspectors.py](get_unique_inspectors.py) script. - -Optional arguments: -Argument | Description ---- | --- -|**‑‑remove‑duplicates**| Remove duplicates around inspections in each row. Default value is `False`. | - -The resulting file will be stored in the same folder as the input file. - -An example of the input file: - -```json -id | code | lang | inspections ------|-------------------|---------------|----------------- -2 | "// some code" | java11 | "{""issues"": []}" -3 | "// some code" | java11 | "{""issues"": [""{\"... \""problem_id\"": \""SystemOutErr\""}""]}" -0 | "// some code" | java11 | "{""issues"": [""{\"...\""problem_id\"": \""ConstantExpression\""}"",""{\"...\""problem_id\"": \""ConstantExpression\""}""]}" -1 | "// some code" | java11 | "{""issues"": []}" -``` - -with the inspections file: - -```json -id | inspection_id ------|------------------- -1 | SystemOutErr -2 | ConstantExpression -``` - -An example of the output file: - -```json -id | code | lang | inspections ------|----------------------------------------|---------------|----------------- -2 | "// first line from code with id 2" | java11 | 0 -2 | "// second line from code with id 2" | java11 | 0 -3 | "// first line from code with id 3" | java11 | 1 -3 | "// second line from code with id 3" | java11 | 0 -0 | "// first line from code with id 0" | java11 | 0 -0 | "// second line from code with id 0" | java11 | 2,2 -1 | "// first line from code with id 1" | java11 | 0 -1 | "// second line from code with id 1" | java11 | 0 - -``` - -# Postprocessing - -At this stage, you can convert the data received by the Qodana into the format of the Hyperstyle tool for -analysis and statistics gathering. - -## Convert Qodana inspections into Hyperstyle inspections - -This stage allows you to convert the `inspections` column from `csv` marked by Qodana into -`traceback` column with the Hyperstyle tool format. - -This stage includes: -- keep only unique code fragments in both datasets (Qodana and Hyperstyle); -- keep only fragments in both datasets that have same ids and same code fragments; -- add a `grade` column into Qodana dataset corresponding to the `grade` column from Hyperstyle dataset; -- add a `traceback` column in the Hyperstyle format into Qodana dataset with inspection from the `inspections` column. - -Please, note that your Qodana input file must be graded by [dataset_labeling.py](dataset_labeling.py) script -and have `inspections` column. Your Hyperstyle input file must be graded by [evaluation_run_tool.py](../evaluation_run_tool.py) script -and have `traceback` and `grade` columns. - -Output files is two new `csv` files. - -#### Usage - -Run the [convert_to_hyperstyle_inspections.py](convert_to_hyperstyle_inspections.py) with the arguments from command line. - -Required arguments: - -- `solutions_file_path_hyperstyle` — path to a `csv` file labelled by Hyperstyle; -- `solutions_file_path_qodana` — path to a `csv` file labelled by Qodana. - -Optional arguments: -Argument | Description ---- | --- -|**‑i**, **‑‑issues-to-keep**| Set of issues ids to keep in the dataset separated by comma. By default all issues are deleted. | - -The Hyperstyle resulting file will be stored in the same folder with `solutions_file_path_hyperstyle`. -The Qodana resulting file will be stored in the same folder with `solutions_file_path_qodana`. - -An example of the Qodana inspections before and after this processing: - -1. Before: - -```json -{ - "issues": [ - { - "fragment_id": 0, - "line": 8, - "offset": 8, - "length": 10, - "highlighted_element": "System.out", - "description": "Uses of System.out should probably be replaced with more robust logging #loc", - "problem_id": "SystemOutErr" - } - ] -} -``` - -2. After: - -```json -{ - "issues": [ - { - "code": "SystemOutErr", - "text": "Uses of System.out should probably be replaced with more robust logging #loc", - "line": "", - "line_number": 8, - "column_number": 8, - "category": "INFO", - "influence_on_penalty": 0 - } - ] -} -``` -___ diff --git a/src/python/evaluation/qodana/__init__.py b/src/python/evaluation/qodana/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/qodana/convert_to_hyperstyle_inspections.py b/src/python/evaluation/qodana/convert_to_hyperstyle_inspections.py deleted file mode 100644 index 5d7530a3..00000000 --- a/src/python/evaluation/qodana/convert_to_hyperstyle_inspections.py +++ /dev/null @@ -1,123 +0,0 @@ -import argparse -import json -from pathlib import Path -from typing import Iterable, Set - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.pandas_util import ( - drop_duplicates, filter_df_by_iterable_value, get_solutions_df_by_file_path, write_df_to_file, -) -from src.python.evaluation.common.util import ColumnName, parse_set_arg -from src.python.evaluation.qodana.util.issue_types import QODANA_CLASS_NAME_TO_ISSUE_TYPE -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue -from src.python.review.common.file_system import Extension, get_parent_folder -from src.python.review.inspectors.inspector_type import InspectorType -from src.python.review.inspectors.issue import BaseIssue, IssueType -from src.python.review.reviewers.utils.print_review import convert_issue_to_json - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name}_hyperstyle', - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be graded by hyperstyle tool' - f'(file contains traceback column)') - - parser.add_argument(f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name}_qodana', - type=lambda value: Path(value).absolute(), - help=f'{RunToolArgument.SOLUTIONS_FILE_PATH.value.description}' - f'\nAll code fragments from this file must be graded by qodana' - f'(file contains inspections column)') - - parser.add_argument('-i', '--issues-to-keep', - help='Set of issues to keep', - default='') - - -# Drop duplicates in the CODE column and delete rows that have ids from value_to_filter -# The new dataframe will be sorted by the ID column -def __preprocess_df(df: pd.DataFrame, ids_to_filter: Iterable) -> pd.DataFrame: - df = drop_duplicates(df) - df = filter_df_by_iterable_value(df, ColumnName.ID.value, ids_to_filter) - return df.sort_values(ColumnName.ID.value).set_index(ColumnName.ID.value, drop=False) - - -# Check if all code fragments with the same ids are equal -def __check_code_by_ids(qodana_df: pd.DataFrame, hyperstyle_df: pd.DataFrame) -> None: - assert qodana_df.shape[0] == hyperstyle_df.shape[0], ( - f'rows count {qodana_df.shape[0]} in the qodana df does not equal rows ' - f'count {hyperstyle_df.shape[0]} in the hyperstyle df' - ) - for i in range(0, qodana_df.shape[0]): - if qodana_df.iloc[i][ColumnName.CODE.value] != hyperstyle_df.iloc[i][ColumnName.CODE.value]: - raise ValueError(f'Code fragments in the {i}th row do not equal!') - - -# Convert qodana inspections output to hyperstyle output -# Note: keep only json field in the result -def __qodana_to_hyperstyle_output(qodana_output: str, issues_to_keep: Set[str]) -> str: - qodana_issues = QodanaIssue.parse_list_issues_from_json(qodana_output) - filtered_issues = filter(lambda issue: issue.problem_id in issues_to_keep, qodana_issues) - hyperstyle_issues = map(lambda issue: - BaseIssue(origin_class=issue.problem_id, - type=QODANA_CLASS_NAME_TO_ISSUE_TYPE.get(issue.problem_id, IssueType.INFO), - description=issue.description, - file_path=Path(), - line_no=issue.line, - column_no=issue.offset, - inspector_type=InspectorType.QODANA), - filtered_issues) - hyperstyle_json = {'issues': list(map(lambda issue: convert_issue_to_json(issue), hyperstyle_issues))} - - return json.dumps(hyperstyle_json) - - -# Resort all fields in the qodana dataframe according to the hyperstyle dataframe -# Add column with hyperstyle output (convert qodana output to hyperstyle output) -# Add grade column with grades from hyperstyle dataframe (to gather statistics by diffs_between_df.py script) -def __prepare_qodana_df(qodana_df: pd.DataFrame, hyperstyle_df: pd.DataFrame, - issues_to_keep: Set[str]) -> pd.DataFrame: - qodana_df = __preprocess_df(qodana_df, hyperstyle_df[ColumnName.ID.value]) - __check_code_by_ids(qodana_df, hyperstyle_df) - - qodana_df[ColumnName.TRACEBACK.value] = qodana_df.apply( - lambda row: __qodana_to_hyperstyle_output(row[QodanaColumnName.INSPECTIONS.value], issues_to_keep), axis=1) - - qodana_df[ColumnName.GRADE.value] = hyperstyle_df[ColumnName.GRADE.value] - return qodana_df - - -def __write_updated_df(old_df_path: Path, df: pd.DataFrame, name_prefix: str) -> None: - output_path = get_parent_folder(Path(old_df_path)) - write_df_to_file(df, output_path / f'{name_prefix}_updated{Extension.CSV.value}', Extension.CSV) - - -def __reassign_ids(df: pd.DataFrame) -> pd.DataFrame: - df = df.sort_values(ColumnName.CODE.value) - df[ColumnName.ID.value] = df.index - return df - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - issues_to_keep = parse_set_arg(args.issues_to_keep) - - qodana_solutions_file_path = args.solutions_file_path_qodana - qodana_solutions_df = __reassign_ids(get_solutions_df_by_file_path(qodana_solutions_file_path)) - - hyperstyle_solutions_file_path = args.solutions_file_path_hyperstyle - hyperstyle_solutions_df = __reassign_ids(get_solutions_df_by_file_path(hyperstyle_solutions_file_path)) - hyperstyle_solutions_df = __preprocess_df(hyperstyle_solutions_df, qodana_solutions_df[ColumnName.ID.value]) - - qodana_solutions_df = __prepare_qodana_df(qodana_solutions_df, hyperstyle_solutions_df, issues_to_keep) - - __write_updated_df(qodana_solutions_file_path, qodana_solutions_df, 'qodana') - __write_updated_df(hyperstyle_solutions_file_path, hyperstyle_solutions_df, 'hyperstyle') - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/qodana/dataset_labeling.py b/src/python/evaluation/qodana/dataset_labeling.py deleted file mode 100644 index f11de467..00000000 --- a/src/python/evaluation/qodana/dataset_labeling.py +++ /dev/null @@ -1,280 +0,0 @@ -import json -import logging -import os -import re -import sys -import traceback -from argparse import ArgumentParser, Namespace -from collections import defaultdict -from math import ceil -from pathlib import Path -from typing import Dict, List, Optional - -sys.path.append('../../../..') - -import numpy as np -import pandas as pd -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue -from src.python.evaluation.qodana.util.util import to_json -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import ( - copy_directory, - copy_file, - create_file, - Extension, - get_all_file_system_items, - get_content_from_file, - get_name_from_path, - get_parent_folder, - match_condition, - remove_directory, -) -from src.python.review.common.subprocess_runner import run_and_wait -from src.python.review.run_tool import positive_int - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - -TEMPLATE_FOLDER = Path(__file__).parents[3] / 'resources' / 'evaluation' / 'qodana' / 'project_templates' - - -def configure_arguments(parser: ArgumentParser) -> None: - parser.add_argument( - 'dataset_path', - type=lambda value: Path(value).absolute(), - help=f"Dataset path. The dataset must contain at least three columns: '{ColumnName.ID.value}', " - f"'{ColumnName.CODE.value}' and '{ColumnName.LANG.value}', where '{ColumnName.ID.value}' is a unique " - f"solution number, '{ColumnName.LANG.value}' is the language in which the code is written in the " - f"'{ColumnName.CODE.value}' column. The '{ColumnName.LANG.value}' must belong to one of the following " - f"values: {', '.join(LanguageVersion.values())}. " - f"If '{ColumnName.LANG.value}' is not equal to any of the values, the row will be skipped.", - ) - - parser.add_argument('-c', '--config', type=lambda value: Path(value).absolute(), help='Path to qodana.yaml') - - parser.add_argument( - '-l', - '--limit', - type=positive_int, - help='Allows you to read only the specified number of first rows from the dataset.', - ) - - parser.add_argument( - '-s', - '--chunk-size', - type=positive_int, - help='The number of files that qodana will process at a time.', - default=5000, - ) - - parser.add_argument( - '-o', - '--output-path', - type=lambda value: Path(value).absolute(), - help='The path where the labeled dataset will be saved. ' - 'If not specified, the labeled dataset will be saved next to the original one.', - ) - - -class DatasetLabel: - """ - DatasetLabel allows you to label a dataset using the found Qodana inspections. - Accepts dataset_path, config, limit, chunk_size and output_path. - """ - - dataset_path: Path - config: Optional[Path] - limit: Optional[int] - chunk_size: Optional[int] - inspection_to_id: Dict[str, int] - output_path: Path - - def __init__(self, args: Namespace): - self.dataset_path = args.dataset_path - self.config = args.config - self.limit = args.limit - self.chunk_size = args.chunk_size - - self.output_path = args.output_path - if self.output_path is None: - output_dir = get_parent_folder(self.dataset_path) - dataset_name = get_name_from_path(self.dataset_path) - self.output_path = output_dir / f'labeled_{dataset_name}' - - def label(self) -> None: - """ - Runs Qodana on each row of the dataset and writes the found inspections in the 'inspections' column. - """ - dataset = pd.read_csv(self.dataset_path, nrows=self.limit) - - group_by_lang = dataset.groupby(ColumnName.LANG.value) - unique_languages = dataset[ColumnName.LANG.value].unique() - - logger.info(f'Unique languages: {unique_languages}') - - groups = [] - for language in unique_languages: - lang_group = group_by_lang.get_group(language) - - if language in LanguageVersion.values(): - # TODO: languages need implementation - try: - logger.info(f'Processing the language: {language}') - groups.append(self._label_language(lang_group, LanguageVersion(language))) - except NotImplementedError: - # If we find a language that is in the LanguageVersion, - # but is not supported in this script, we should skip this fragment. - logger.warning(f'{language} needs implementation') - groups.append(lang_group) - else: - logger.warning(f'Unknown language: {language}') - groups.append(lang_group) - - logger.info('Dataset processing finished') - - dataset = pd.concat(groups) - - logger.info('Writing the dataset to a file.') - write_dataframe_to_csv(self.output_path, dataset) - - def _label_language(self, df: pd.DataFrame, language: LanguageVersion) -> pd.DataFrame: - number_of_chunks = 1 - if self.chunk_size is not None: - number_of_chunks = ceil(df.shape[0] / self.chunk_size) - - chunks = np.array_split(df, number_of_chunks) - labeled_chunks = [] - # Todo: run this in parallel - for index, chunk in enumerate(chunks): - logger.info(f'Processing chunk: {index + 1} / {number_of_chunks}') - labeled_chunks.append(self._label_chunk(chunk, language, index)) - - logger.info(f'{language} processing finished.') - result = pd.concat(labeled_chunks) - return result - - @classmethod - def _extract_fragment_id(cls, folder_name: str) -> int: - numbers = re.findall(r'\d+', folder_name) - if len(numbers) != 1: - raise ValueError(f'Can not extract fragment id from {folder_name}') - return numbers[0] - - @classmethod - def _get_fragment_id_from_fragment_file_path(cls, fragment_file_path: str) -> int: - folder_name = get_name_from_path(get_parent_folder(fragment_file_path), with_extension=False) - return cls._extract_fragment_id(folder_name) - - @classmethod - def _parse_inspections_files(cls, inspections_files: List[Path]) -> Dict[int, List[QodanaIssue]]: - id_to_issues: Dict[int, List[QodanaIssue]] = defaultdict(list) - for file in inspections_files: - issues = json.loads(get_content_from_file(file))['problems'] - for issue in issues: - fragment_id = int(cls._get_fragment_id_from_fragment_file_path(issue['file'])) - qodana_issue = QodanaIssue( - line=issue['line'], - offset=issue['offset'], - length=issue['length'], - highlighted_element=issue['highlighted_element'], - description=issue['description'], - fragment_id=fragment_id, - problem_id=issue['problem_class']['id'], - ) - id_to_issues[fragment_id].append(qodana_issue) - return id_to_issues - - def _label_chunk(self, chunk: pd.DataFrame, language: LanguageVersion, chunk_id: int) -> pd.DataFrame: - tmp_dir_path = self.dataset_path.parent.absolute() / f'qodana_project_{chunk_id}' - os.makedirs(tmp_dir_path, exist_ok=True) - - project_dir = tmp_dir_path / 'project' - results_dir = tmp_dir_path / 'results' - - logger.info('Copying the template') - self._copy_template(project_dir, language) - - if self.config: - logger.info('Copying the config') - copy_file(self.config, project_dir) - - logger.info('Creating main files') - self._create_main_files(project_dir, chunk, language) - - logger.info('Running qodana') - self._run_qodana(project_dir, results_dir) - - logger.info('Getting inspections') - inspections_files = self._get_inspections_files(results_dir) - inspections = self._parse_inspections_files(inspections_files) - - logger.info('Write inspections') - chunk[QodanaColumnName.INSPECTIONS.value] = chunk.apply( - lambda row: to_json(inspections.get(row[ColumnName.ID.value], [])), axis=1, - ) - - remove_directory(tmp_dir_path) - return chunk - - @staticmethod - def _copy_template(project_dir: Path, language: LanguageVersion) -> None: - if language.is_java(): - java_template = TEMPLATE_FOLDER / "java" - copy_directory(java_template, project_dir) - else: - raise NotImplementedError(f'{language} needs implementation.') - - def _create_main_files(self, project_dir: Path, chunk: pd.DataFrame, language: LanguageVersion) -> None: - if language.is_java(): - working_dir = project_dir / 'src' / 'main' / 'java' - - chunk.apply( - lambda row: next( - create_file( - file_path=(working_dir / f'solution{row[ColumnName.ID.value]}' / f'Main{Extension.JAVA.value}'), - content=row[ColumnName.CODE.value], - ), - ), - axis=1, - ) - else: - raise NotImplementedError(f'{language} needs implementation.') - - @staticmethod - def _run_qodana(project_dir: Path, results_dir: Path) -> None: - results_dir.mkdir() - command = [ - 'docker', 'run', - '-u', str(os.getuid()), - '--rm', - '-v', f'{project_dir}/:/data/project/', - '-v', f'{results_dir}/:/data/results/', - 'jetbrains/qodana', - ] - run_and_wait(command) - - @staticmethod - def _get_inspections_files(results_dir: Path) -> List[Path]: - condition = match_condition(r'\w*.json') - return get_all_file_system_items(results_dir, condition, without_subdirs=True) - - -def main(): - parser = ArgumentParser() - configure_arguments(parser) - - try: - args = parser.parse_args() - dataset_label = DatasetLabel(args) - dataset_label.label() - - except Exception: - traceback.print_exc() - logger.exception('An unexpected error') - return 2 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/qodana/filter_inspections.py b/src/python/evaluation/qodana/filter_inspections.py deleted file mode 100644 index 9321a7eb..00000000 --- a/src/python/evaluation/qodana/filter_inspections.py +++ /dev/null @@ -1,58 +0,0 @@ -import argparse -from pathlib import Path -from typing import List - -import pandas as pd -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.common.util import parse_set_arg -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue -from src.python.evaluation.qodana.util.util import to_json -from src.python.review.common.file_system import Extension, extension_file_condition, get_all_file_system_items - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('dataset_folder', - type=lambda value: Path(value).absolute(), - help='Path to a folder with csv files graded by Qodana. ' - 'Each file must have "inspections" column.') - - parser.add_argument('-i', '--inspections', - help='Set of inspections ids to exclude from the dataset', - type=str, - default='') - - -def __get_qodana_dataset(root: Path) -> pd.DataFrame: - if not root.is_dir(): - raise ValueError(f'The {root} is not a directory') - dataset_files = get_all_file_system_items(root, extension_file_condition(Extension.CSV)) - datasets = [] - for file_path in dataset_files: - datasets.append(get_solutions_df_by_file_path(file_path)) - return pd.concat(datasets) - - -def __filter_inspections(json_issues: str, inspections_to_keep: List[str]) -> str: - issues_list = QodanaIssue.parse_list_issues_from_json(json_issues) - filtered_issues = list(filter(lambda i: i.problem_id not in inspections_to_keep, issues_list)) - return to_json(filtered_issues) - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - dataset_folder = args.dataset_folder - full_dataset = __get_qodana_dataset(dataset_folder) - inspections_to_keep = parse_set_arg(args.inspections) - - full_dataset[QodanaColumnName.INSPECTIONS.value] = full_dataset.apply( - lambda row: __filter_inspections(row[QodanaColumnName.INSPECTIONS.value], inspections_to_keep), axis=1) - - write_dataframe_to_csv(dataset_folder / f'filtered_issues{Extension.CSV.value}', full_dataset) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/qodana/fragment_to_inspections_list.py b/src/python/evaluation/qodana/fragment_to_inspections_list.py deleted file mode 100644 index 42fe3ec6..00000000 --- a/src/python/evaluation/qodana/fragment_to_inspections_list.py +++ /dev/null @@ -1,33 +0,0 @@ -import argparse -from pathlib import Path - -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue -from src.python.evaluation.qodana.util.util import ( - configure_model_converter_arguments, get_inspections_dict, replace_inspections_on_its_ids, -) -from src.python.review.common.file_system import Extension, get_parent_folder - -INSPECTIONS = QodanaColumnName.INSPECTIONS.value - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_model_converter_arguments(parser) - args = parser.parse_args() - - solutions_file_path = args.solutions_file_path - solutions_df = get_solutions_df_by_file_path(solutions_file_path) - inspections_dict = get_inspections_dict(args.inspections_path) - - solutions_df[INSPECTIONS] = solutions_df.apply( - lambda row: replace_inspections_on_its_ids(QodanaIssue.parse_list_issues_from_json(row[INSPECTIONS]), - inspections_dict, args.remove_duplicates), axis=1) - - output_path = get_parent_folder(Path(solutions_file_path)) - write_dataframe_to_csv(output_path / f'numbered_ids{Extension.CSV.value}', solutions_df) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/qodana/fragment_to_inspections_list_line_by_line.py b/src/python/evaluation/qodana/fragment_to_inspections_list_line_by_line.py deleted file mode 100644 index c70d9ba1..00000000 --- a/src/python/evaluation/qodana/fragment_to_inspections_list_line_by_line.py +++ /dev/null @@ -1,62 +0,0 @@ -import argparse -import os -from itertools import groupby -from pathlib import Path -from typing import Dict, List - -import pandas as pd -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue -from src.python.evaluation.qodana.util.util import ( - configure_model_converter_arguments, get_inspections_dict, replace_inspections_on_its_ids, -) -from src.python.review.common.file_system import Extension, get_parent_folder - - -INSPECTIONS = QodanaColumnName.INSPECTIONS.value -CODE = ColumnName.CODE.value - - -# Make a new dataframe where code fragment is separated line by line and inspections are grouped line by line -def __replace_inspections_to_its_ids_in_row(row: pd.Series, inspections_dict: Dict[str, int], - to_remove_duplicates: bool) -> pd.DataFrame: - row_df = pd.DataFrame(row).transpose() - fragment_lines = row_df.iloc[0][CODE].split(os.linesep) - fragment_df = row_df.loc[row_df.index.repeat(len(fragment_lines))].reset_index(drop=True) - - issues_list = QodanaIssue.parse_list_issues_from_json(row_df.iloc[0][INSPECTIONS]) - line_number_to_issues = {k: list(v) for k, v in groupby(issues_list, key=lambda i: i.line)} - for index, fragment_line in enumerate(fragment_lines): - issues = line_number_to_issues.get(index + 1, []) - fragment_df.iloc[index][CODE] = fragment_line - fragment_df.iloc[index][INSPECTIONS] = replace_inspections_on_its_ids(issues, inspections_dict, - to_remove_duplicates) - return fragment_df - - -def __append_df(df: pd.DataFrame, df_list: List[pd.DataFrame]) -> None: - df_list.append(df) - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_model_converter_arguments(parser) - args = parser.parse_args() - - solutions_file_path = args.solutions_file_path - solutions_df = get_solutions_df_by_file_path(solutions_file_path) - inspections_dict = get_inspections_dict(args.inspections_path) - - fragment_df_list = [] - solutions_df.apply( - lambda row: __append_df(__replace_inspections_to_its_ids_in_row(row, inspections_dict, args.remove_duplicates), - fragment_df_list), axis=1) - - output_path = get_parent_folder(Path(solutions_file_path)) - write_dataframe_to_csv(output_path / f'numbered_ids_line_by_line{Extension.CSV.value}', pd.concat(fragment_df_list)) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/qodana/get_unique_inspectors.py b/src/python/evaluation/qodana/get_unique_inspectors.py deleted file mode 100644 index 35c32bdb..00000000 --- a/src/python/evaluation/qodana/get_unique_inspectors.py +++ /dev/null @@ -1,94 +0,0 @@ -import argparse -import itertools -from collections import defaultdict -from pathlib import Path -from typing import Dict, List, Optional - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue -from src.python.review.common.file_system import Extension, get_parent_folder - - -INSPECTION_ID = QodanaColumnName.INSPECTION_ID.value -INSPECTIONS = QodanaColumnName.INSPECTIONS.value -COUNT_ALL = QodanaColumnName.COUNT_ALL.value -COUNT_UNIQUE = QodanaColumnName.COUNT_UNIQUE.value -ID = QodanaColumnName.ID.value - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.QODANA_SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.QODANA_SOLUTIONS_FILE_PATH.value.description) - - parser.add_argument('--uniq', - help='If True, count fragments for eash inspection in which this inspection was.', - action='store_true') - - -def __get_inspections_ids(json_issues: str) -> List[str]: - issues_list = QodanaIssue.parse_list_issues_from_json(json_issues) - return list(map(lambda i: i.problem_id, issues_list)) - - -def __get_inspections_from_df(solutions_df: pd.DataFrame) -> List[str]: - inspections = solutions_df.apply(lambda row: __get_inspections_ids(row[INSPECTIONS]), axis=1) - return list(itertools.chain.from_iterable(inspections.values)) - - -def __count_uniq_inspections_in_fragment(json_issues: str, inspection_id_to_fragments: Dict[str, int]) -> None: - issues_list = set(__get_inspections_ids(json_issues)) - for issue in issues_list: - inspection_id_to_fragments[issue] += 1 - - -def __get_uniq_inspections_in_all_fragments(solutions_df: pd.DataFrame) -> Dict[str, int]: - inspection_id_to_fragments: Dict[str, int] = defaultdict(int) - solutions_df.apply(lambda row: __count_uniq_inspections_in_fragment(row[INSPECTIONS], inspection_id_to_fragments), - axis=1) - - return inspection_id_to_fragments - - -def __get_all_inspections_by_inspection_id(inspection_id: str, all_inspections: List[str]) -> List[str]: - return list(filter(lambda i: i == inspection_id, all_inspections)) - - -def __create_unique_inspections_df(inspections: List[str], - inspection_id_to_fragments: Optional[Dict[str, int]]) -> pd.DataFrame: - id_to_inspection = {} - for index, inspection in enumerate(set(inspections)): - id_to_inspection[index + 1] = inspection - inspections_df = pd.DataFrame(id_to_inspection.items(), columns=[ID, INSPECTION_ID]) - inspections_df[COUNT_ALL] = inspections_df.apply(lambda row: len(__get_all_inspections_by_inspection_id( - row[INSPECTION_ID], inspections)), axis=1) - if inspection_id_to_fragments is None: - inspections_df[COUNT_UNIQUE] = 0 - else: - inspections_df[COUNT_UNIQUE] = inspections_df.apply(lambda row: inspection_id_to_fragments.get( - row[INSPECTION_ID], 0), axis=1) - return inspections_df - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - solutions_file_path = args.solutions_file_path - solutions_df = get_solutions_df_by_file_path(solutions_file_path) - if args.uniq: - inspection_id_to_fragments = __get_uniq_inspections_in_all_fragments(solutions_df) - else: - inspection_id_to_fragments = None - inspections_df = __create_unique_inspections_df(__get_inspections_from_df(solutions_df), inspection_id_to_fragments) - - output_path = get_parent_folder(Path(solutions_file_path)) - write_dataframe_to_csv(output_path / f'inspections{Extension.CSV.value}', inspections_df) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/qodana/imitation_model/README.md b/src/python/evaluation/qodana/imitation_model/README.md deleted file mode 100644 index acb635f5..00000000 --- a/src/python/evaluation/qodana/imitation_model/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# Qodana imitation model -## Description -The general purpose of the model is to simulate the behavior of [`Qodana`](https://github.com/JetBrains/Qodana/tree/main) – -a code quality monitoring tool that identifies and suggests fixes for bugs, security vulnerabilities, duplications, and imperfections. - -Motivation for developing a model: -- acceleration of the code analysis process by training the model to recognize a certain class of errors; -- the ability to run the model on separate files without the need to create a project (for example, for the Java language) - - -## Architecture -[`RobertaForSequenceClassification`](https://huggingface.co/transformers/model_doc/roberta.html#robertaforsequenceclassification) model with [`BCEWithLogitsLoss`](https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html) solve multilabel classification task. - -Model outputs is a tensor of size: `batch_size` x `num_classes`. Where `batch_size` is the number of training examples utilized in one iteration, -and `num_classes` is the number of error types met in the dataset. By model class here, we mean a unique error type. -Class probabilities are received by taking `sigmoid` and final predictions are computed by comparing the probability of each class with the `threshold`. - -As classes might be unbalanced the used metric is `f1-score`. -## What it does - -Model has two use cases: -- It can be trained to predict a unique number of errors in a **block** of code, unfixed length. - -**Example**: - -code | inspections ---- | --- -|`import java.util.Scanner; class Main {public static void main(String[] args) {Scanner scanner = new Scanner(System.in);// put your code here int num = scanner.nextInt(); System.out.println((num / 10 ) % 10);}}`| 1, 2| - - -- It can be trained to predict a unique number of errors in a **line** of code. - -**Example** - -code | inspections ---- | --- -|`import java.util.Scanner;`| 0| -|`\n`|0| -|`class Main {`|1| -|`public static void main(String[] args`) {|1| -|`Scanner scanner = new Scanner(System.in);`|0| -|`// put your code here`|0| -|`int num = scanner.nextInt();`|0| -|`System.out.println((num / 10 ) % 10);`|2| -|`}`|0| -|`}`|0| - - -## Data preprocessing - -Please address to the [`following documentation`](src/python/evaluation/qodana) for labeling dataset and to the [`following documentation`](preprocessing) to preprocess data for model training and evaluation afterwards. - -After completing the 3d preprocessing step you should have 3 folders: -`train`, `val`, `test` with `train.csv`, `val.csv` and `test.csv` respectively. - -Each file has the same structure, it should consist of 4+ columns: -- `id` – solutions id; -- `code` – line od code or block of code; -- `lang` - language version; -- `0`, `1`, `2` ... `n` – several columns, equal to the unique number of errors detected by Qodana in the dataset. -The values in the columns are binary numbers: `1` if inspection is detected and `0` otherwise. - - -## How to train the model - -Run [`train.py`](train.py) script from the command line with the following arguments: - -Required arguments: - -- `train_dataset_path` ‑ path to the `train.csv` – file that consists of samples -that model will use for training. - -- `val_dataset_path` ‑ path to the `val.csv` – file that consists of samples -that model will use for evaluation during training. - -Both files are received by running [`split_dataset.py`](preprocessing/split_dataset.py) script and has the structure as described above. - -Optional arguments: - -Argument | Description ---- | --- -|**‑o**, **‑‑output_directory_path**| Path to the directory where model weights will be saved. If not set, folder will be created in the `train` folder where `train.csv` dataset is stored.| -|**‑c**, **‑‑context_length**| Sequence length or embedding size of tokenized samples. Available values are any `positive integers`. **Default is 40**.| -|**‑e**, **‑‑epoch**| Number of epochs to train model. **Default is 2**.| -|**‑bs**, **‑‑batch_size**| Batch size for training and validation dataset. Available values are any `positive integers`. **Default is 16**.| -|**‑lr**, **‑‑learning_rate**| Optimizer learning rate. **Default is 2e-5**.| -|**‑w**, **‑‑weight_decay**| Weight decay parameter for an optimizer. **Default is 0.01**.| -|**‑th**, **‑‑threshold**| Is used to compute predictions. Available values: 0 < `threshold` < 1. If the probability of inspection is greater than `threshold`, sample will be classified with the inspection. **Default is 0.5**.| -|**‑ws**, **‑‑warm_up_steps**| A number of steps when optimizer uses constant learning rate before applying scheduler policy. **Default is 300**.| -|**‑sl**, **‑‑save_limit**| Total amount of checkpoints limit. Default is 1.| - -To inspect the rest of default training parameters please, address to the [`TrainingArguments`](common/train_config.py). - -## How to evaluate model - -Run [`evaluation.py`](evaluation.py) script from the command line with the following arguments: - -Required arguments: - -`test_dataset_path` ‑ path to the `test.csv` received by running [`split_dataset.py`](preprocessing/split_dataset.py) script. - -`model_weights_directory_path` ‑ path to the folder where trained model weights are saved. - -Optional arguments: - -Argument | Description ---- | --- -|**‑o**, **‑‑output_directory_path**| Path to the directory where labeled dataset will be saved. Default is the `test` folder.| -|**‑c**, **‑‑context_length**| Sequence length or embedding size of tokenized samples. Available values are any `positive integers`. **Default is 40**.| -|**‑sf**, **‑‑save_f1_score**| If enabled report with f1 scores by classes will be saved to the `csv` file in the parent directory of labeled dataset. **Disabled by default**.| -|**‑bs**, **‑‑batch_size**| The number of training examples utilized in one training and validation iteration. Available values are any `positive integers`. **Default is 16**.| -|**‑th**, **‑‑threshold**| Is used to compute predictions. Available values: 0 < `threshold` < 1. If the probability of inspection is greater than `threshold`, sample will be classified with the inspection. **Default is 0.5**.| - -Output is a `predictions.csv` file with the column names matches the number of classes. Each sample has a binary label: - -- `0` ‑ if the model didn't found an error in a sample. - -- `1` ‑ if the error was found in a sample. - - -## How to use model, pretrained on Java code snippets from Stepik - -There are 2 trained models available for the usage and 2 datasets on which models were trained and evaluated. -Access to the datasets is restricted. -### Model that uses program text as an input: -- [`train_dataset`](https://drive.google.com/drive/folders/1bdLExLIbY53SVobT0y4Lnz9oeZENqLmt?usp=sharing) – private access; -- [`evaluation_dataset`](https://drive.google.com/file/d/1hZlP7q3gVoIl8vmOur0UFpEyFDYyVZko/view?usp=sharing) – private access; -- [`test_dataset`](https://drive.google.com/file/d/1oappcDcH-p-2LwjdOfZHRSiRB9Vi39mc/view?usp=sharing) – private access; -- [`model_weights`](https://drive.google.com/file/d/1PFVHVd4JDjFUD3b5fDSGXoYBWEDlaEAg/view?usp=sharing) – public access. - -The model was trained to detect 110 Qodana inspections. The whole -list of inspections can be found via the link [here](https://drive.google.com/file/d/1PVqjx7QEot1dIXyiYP_-dJnWGup2Ef7v/view?usp=sharing). - -Evaluation results are: - -Inspection | Description | F1-Score ---- | --- | --- -|No Errors | No errors from the [list](https://docs.google.com/spreadsheets/d/14BTj_lTTRrGlx-GPTcbMlc8zdt--WXLZHRnegKRrZYM/edit?usp=sharing) were detected by Qodana.| 0.73 | -| Syntax Error |Reports any irrelevant usages of java syntax.| 0.99| -| System Out Error | Reports any usages of System.out or System.err. | 0.99 | -| IO Resources | Reports any I/O resource which is not safely closed. | 0.97 | - -The rests of the inspections were not learnt by the model due to the class disbalance. -### Model that uses a line of program text as an input: -- [`train_dataset`](https://drive.google.com/file/d/1c-kJUV4NKuehCoLiIC3JWrJh3_NgTmvi/view?usp=sharing) – private access; -- [`evaluation_dataset`](https://drive.google.com/file/d/1AVN4Uj4omPEquC3EAL6XviFATkYKcY_2/view?usp=sharing) – private access; -- [`test_dataset`](https://drive.google.com/file/d/1J3gz3wS_l63SI0_OMym8x5pCj7-PCIgG/view?usp=sharing) – private access; -- [`model_weights`](https://drive.google.com/file/d/1fc32-5XyUeOpZ5AkRotqv_3cWksHjat_/view?usp=sharing) – public access. - -One sample in the dataset consists of one line of program in the context. The context is 2 lines of the same -program before and after the target line. When there are not enough lines before or after target, special -token `NOC` is added. - -The model was also trained to detect 110 inspections. The whole -list of inspections can be found via the link [here](https://drive.google.com/file/d/1PVqjx7QEot1dIXyiYP_-dJnWGup2Ef7v/view?usp=sharing). - -Evaluation results are: - -Inspection | Description | F1-score ---- | --- | --- -|No Errors | No errors from the [list](https://docs.google.com/spreadsheets/d/14BTj_lTTRrGlx-GPTcbMlc8zdt--WXLZHRnegKRrZYM/edit?usp=sharing) were detected by Qodana.| 0.99 | -| Syntax Error |Reports any irrelevant usages of java syntax.| 0.23| -| System Out Error | Reports any usages of System.out or System.err.| 0.30 | -| IO Resources | Reports any I/O resource which is not safely closed | 0.23 | - -The rests of the inspections were not learnt by the model due to the class disbalance. - -To use any of the model follow [`fine-tuning`](https://huggingface.co/transformers/training.html) tutorial from HuggingFace. Unarchive `model weights` zip and use absolute path to the root folder instead of built-in name of pretrained model. - -For example: - - RobertaForSequenceClassification.from_pretrained(, - num_labels=) diff --git a/src/python/evaluation/qodana/imitation_model/__init__.py b/src/python/evaluation/qodana/imitation_model/__init__.py deleted file mode 100644 index 0c4d7f8e..00000000 --- a/src/python/evaluation/qodana/imitation_model/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from src.python import MAIN_FOLDER - -MODEL_FOLDER = MAIN_FOLDER.parent / 'python/imitation_model' diff --git a/src/python/evaluation/qodana/imitation_model/common/__init__.py b/src/python/evaluation/qodana/imitation_model/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/qodana/imitation_model/common/evaluation_config.py b/src/python/evaluation/qodana/imitation_model/common/evaluation_config.py deleted file mode 100644 index e91bf687..00000000 --- a/src/python/evaluation/qodana/imitation_model/common/evaluation_config.py +++ /dev/null @@ -1,47 +0,0 @@ -import argparse - -from src.python.evaluation.qodana.imitation_model.common.util import ModelCommonArgument -from src.python.review.common.file_system import Extension - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('test_dataset_path', - type=str, - help='Path to the dataset received by either' - f' src.python.evaluation.qodana.fragment_to_inspections_list{Extension.PY.value}' - 'or src.python.evaluation.qodana.fragment_to_inspections_list_line_by_line' - f'{Extension.PY.value}script.') - - parser.add_argument('model_weights_directory_path', - type=str, - help='Path to the directory where trained imitation_model weights are stored.') - - parser.add_argument('-o', '--output_directory_path', - default=None, - type=str, - help='Path to the directory where labeled dataset will be saved. Default is the parent folder' - 'of test_dataset_path.') - - parser.add_argument('-sf', '--save_f1_score', - default=None, - action="store_true", - help=f'If enabled report with f1 scores by class will be saved to the {Extension.CSV.value}' - ' File will be saved to the labeled dataset parent directory. Default is False.') - - parser.add_argument(ModelCommonArgument.CONTEXT_LENGTH.value.short_name, - ModelCommonArgument.CONTEXT_LENGTH.value.long_name, - type=int, - default=40, - help=ModelCommonArgument.CONTEXT_LENGTH.value.description) - - parser.add_argument(ModelCommonArgument.BATCH_SIZE.value.short_name, - ModelCommonArgument.BATCH_SIZE.value.long_name, - type=int, - default=8, - help=ModelCommonArgument.BATCH_SIZE.value.description) - - parser.add_argument(ModelCommonArgument.THRESHOLD.value.short_name, - ModelCommonArgument.THRESHOLD.value.long_name, - type=float, - default=0.5, - help=ModelCommonArgument.THRESHOLD.value.description) diff --git a/src/python/evaluation/qodana/imitation_model/common/metric.py b/src/python/evaluation/qodana/imitation_model/common/metric.py deleted file mode 100644 index dce80a94..00000000 --- a/src/python/evaluation/qodana/imitation_model/common/metric.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging.config -from typing import Optional - -import torch -from sklearn.metrics import multilabel_confusion_matrix -from src.python.evaluation.qodana.imitation_model.common.util import MeasurerArgument - -logger = logging.getLogger(__name__) - - -class Measurer: - def __init__(self, threshold: float): - self.threshold = threshold - - def get_f1_score(self, predictions: torch.tensor, targets: torch.tensor) -> Optional[float]: - confusion_matrix = multilabel_confusion_matrix(targets, predictions) - false_positives = sum(score[0][1] for score in confusion_matrix) - false_negatives = sum(score[1][0] for score in confusion_matrix) - true_positives = sum(score[1][1] for score in confusion_matrix) - try: - f1_score = true_positives / (true_positives + 1 / 2 * (false_positives + false_negatives)) - return f1_score - except ZeroDivisionError: - logger.error("No values of the class present in the dataset.") - # return None to make it clear after printing what classes are missing in the datasets - return None - - def compute_metric(self, evaluation_predictions: torch.tensor) -> dict: - logits, targets = evaluation_predictions - prediction_probabilities = torch.from_numpy(logits).sigmoid() - predictions = torch.where(prediction_probabilities > self.threshold, 1, 0) - return {MeasurerArgument.F1_SCORE.value: self.get_f1_score(predictions, torch.tensor(targets))} - - def f1_score_by_classes(self, predictions: torch.tensor, targets: torch.tensor) -> dict: - unique_classes = range(len(targets[0])) - f1_scores_by_classes = {} - for unique_class in unique_classes: - class_mask = torch.where(targets[:, unique_class] == 1) - f1_scores_by_classes[str(unique_class)] = self.get_f1_score(predictions[class_mask[0], unique_class], - targets[class_mask[0], unique_class]) - return f1_scores_by_classes diff --git a/src/python/evaluation/qodana/imitation_model/common/train_config.py b/src/python/evaluation/qodana/imitation_model/common/train_config.py deleted file mode 100644 index ba2d93fa..00000000 --- a/src/python/evaluation/qodana/imitation_model/common/train_config.py +++ /dev/null @@ -1,118 +0,0 @@ -import argparse - -import torch -from src.python.evaluation.qodana.imitation_model.common.util import ( - DatasetColumnArgument, - ModelCommonArgument, - SeedArgument, -) -from transformers import Trainer, TrainingArguments - - -class MultilabelTrainer(Trainer): - """ By default RobertaForSequence classification does not support - multi-label classification. - - Target and logits tensors should be represented as torch.FloatTensor of shape (1,). - https://huggingface.co/transformers/model_doc/roberta.html#transformers.RobertaForSequenceClassification - - To fine-tune the model for the multi-label classification task we can simply modify the trainer by - changing its loss function. https://huggingface.co/transformers/main_classes/trainer.html - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def compute_loss(self, model, inputs, return_outputs=False): - labels = inputs.pop(DatasetColumnArgument.LABELS.value) - outputs = model(**inputs) - logits = outputs.logits - loss_bce = torch.nn.BCEWithLogitsLoss() - loss = loss_bce(logits.view(-1, self.model.config.num_labels), - labels.float().view(-1, self.model.config.num_labels)) - - return (loss, outputs) if return_outputs else loss - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('train_dataset_path', - type=str, - help='Path to the train dataset.') - - parser.add_argument('val_dataset_path', - type=str, - help='Path to the dataset received by either') - - parser.add_argument('-wp', '--trained_weights_directory_path', - default=None, - type=str, - help='Path to the directory where to save imitation_model weights. Default is the directory' - 'where train dataset is.') - - parser.add_argument(ModelCommonArgument.CONTEXT_LENGTH.value.short_name, - ModelCommonArgument.CONTEXT_LENGTH.value.long_name, - type=int, - default=40, - help=ModelCommonArgument.CONTEXT_LENGTH.value.description) - - parser.add_argument(ModelCommonArgument.BATCH_SIZE.value.short_name, - ModelCommonArgument.BATCH_SIZE.value.long_name, - type=int, - default=16, - help=ModelCommonArgument.BATCH_SIZE.value.description) - - parser.add_argument(ModelCommonArgument.THRESHOLD.value.short_name, - ModelCommonArgument.THRESHOLD.value.long_name, - type=float, - default=0.5, - help=ModelCommonArgument.THRESHOLD.value.description) - - parser.add_argument('-lr', '--learning_rate', - type=int, - default=2e-5, - help='Learning rate.') - - parser.add_argument('-wd', '--weight_decay', - type=int, - default=0.01, - help='Wight decay parameter for optimizer.') - - parser.add_argument('-e', '--epoch', - type=int, - default=1, - help='Number of epochs to train imitation_model.') - - parser.add_argument('-ws', '--warm_up_steps', - type=int, - default=300, - help='Number of steps used for a linear warmup, default is 300.') - - parser.add_argument('-sl', '--save_limit', - type=int, - default=1, - help='Total amount of checkpoints limit. Default is 1.') - - -class TrainingArgs: - def __init__(self, args): - self.args = args - - def get_training_args(self, val_steps_to_be_made): - return TrainingArguments(num_train_epochs=self.args.epoch, - per_device_train_batch_size=self.args.batch_size, - per_device_eval_batch_size=self.args.batch_size, - learning_rate=self.args.learning_rate, - warmup_steps=self.args.warm_up_steps, - weight_decay=self.args.weight_decay, - save_total_limit=self.args.save_limit, - output_dir=self.args.trained_weights_directory_path, - overwrite_output_dir=True, - load_best_model_at_end=True, - greater_is_better=True, - save_steps=val_steps_to_be_made, - eval_steps=val_steps_to_be_made, - logging_steps=val_steps_to_be_made, - evaluation_strategy=DatasetColumnArgument.STEPS.value, - logging_strategy=DatasetColumnArgument.STEPS.value, - seed=SeedArgument.SEED.value, - report_to=[DatasetColumnArgument.WANDB.value]) diff --git a/src/python/evaluation/qodana/imitation_model/common/util.py b/src/python/evaluation/qodana/imitation_model/common/util.py deleted file mode 100644 index da0d29e5..00000000 --- a/src/python/evaluation/qodana/imitation_model/common/util.py +++ /dev/null @@ -1,46 +0,0 @@ -from enum import Enum, unique - -from src.python.common.tool_arguments import ArgumentsInfo - - -@unique -class DatasetColumnArgument(Enum): - ID = 'id' - IN_ID = 'inspection_id' - INSPECTIONS = 'inspections' - INPUT_IDS = 'input_ids' - LABELS = 'labels' - DATASET_PATH = 'dataset_path' - STEPS = 'steps' - WEIGHTS = 'weights' - WANDB = 'wandb' - - -@unique -class SeedArgument(Enum): - SEED = 42 - - -@unique -class CustomTokens(Enum): - NOC = '[NOC]' # no context token to add when there are no lines for the context - - -@unique -class ModelCommonArgument(Enum): - THRESHOLD = ArgumentsInfo('-th', '--threshold', - 'If the probability of inspection on code sample is greater than threshold,' - 'inspection id will be assigned to the sample. ' - 'Default is 0.5.') - - CONTEXT_LENGTH = ArgumentsInfo('-cl', '--context_length', - 'Sequence length of 1 sample after tokenization, default is 40.') - - BATCH_SIZE = ArgumentsInfo('-bs', '--batch_size', - 'Batch size – default values are 16 for training and 8 for evaluation mode.') - - -@unique -class MeasurerArgument(Enum): - F1_SCORE = 'f1_score' - F1_SCORES_BY_CLS = 'f1_scores_by_class' diff --git a/src/python/evaluation/qodana/imitation_model/dataset/__init__.py b/src/python/evaluation/qodana/imitation_model/dataset/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/qodana/imitation_model/dataset/dataset.py b/src/python/evaluation/qodana/imitation_model/dataset/dataset.py deleted file mode 100644 index 088ce548..00000000 --- a/src/python/evaluation/qodana/imitation_model/dataset/dataset.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging - -import pandas as pd -import torch -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.qodana.imitation_model.common.util import DatasetColumnArgument -from torch.utils.data import Dataset -from transformers import RobertaTokenizer - -logger = logging.getLogger(__name__) - - -class QodanaDataset(Dataset): - """ MarkingArgument.ID.value is a an id of the solution that corresponds to the line - MarkingArgument.INSPECTIONS.value is a is a target column name in dataset - ColumnName.CODE.value is an observation column name in dataset where lines of code are stored - """ - - def __init__(self, data_path: str, context_length: int): - super().__init__() - df = pd.read_csv(data_path) - tokenizer = RobertaTokenizer.from_pretrained('roberta-base') - code = list(map(str, df[ColumnName.CODE.value])) - self.target = torch.tensor(df.iloc[:, 1:].astype(float).values) - self.code_encoded = tokenizer( - code, padding=True, truncation=True, max_length=context_length, return_tensors="pt", - )[DatasetColumnArgument.INPUT_IDS.value] - - def __getitem__(self, idx): - return {DatasetColumnArgument.INPUT_IDS.value: self.code_encoded[idx], - DatasetColumnArgument.LABELS.value: self.target[idx]} - - def __len__(self): - return len(self.target) diff --git a/src/python/evaluation/qodana/imitation_model/evaluation.py b/src/python/evaluation/qodana/imitation_model/evaluation.py deleted file mode 100644 index 9eac3ec2..00000000 --- a/src/python/evaluation/qodana/imitation_model/evaluation.py +++ /dev/null @@ -1,75 +0,0 @@ -import argparse -import sys -from pathlib import Path - -import numpy as np -import pandas as pd -import torch -import transformers -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.qodana.imitation_model.common.evaluation_config import configure_arguments -from src.python.evaluation.qodana.imitation_model.common.metric import Measurer -from src.python.evaluation.qodana.imitation_model.common.util import DatasetColumnArgument, MeasurerArgument -from src.python.evaluation.qodana.imitation_model.dataset.dataset import QodanaDataset -from src.python.review.common.file_system import Extension -from torch.utils.data import DataLoader -from transformers import RobertaForSequenceClassification - - -def get_predictions(eval_dataloader: torch.utils.data.DataLoader, - model: transformers.RobertaForSequenceClassification, - predictions: np.ndarray, - num_labels: int, - device: torch.device, - args: argparse.ArgumentParser) -> pd.DataFrame: - start_index = 0 - for batch in eval_dataloader: - with torch.no_grad(): - logits = model(input_ids=batch[DatasetColumnArgument.INPUT_IDS.value].to(device)).logits - logits = logits.sigmoid().detach().cpu().numpy() - predictions[start_index:start_index + args.batch_size, :num_labels] = (logits > args.threshold).astype(int) - start_index += args.batch_size - return pd.DataFrame(predictions, columns=range(num_labels), dtype=int) - - -def save_f1_scores(output_directory_path: Path, f1_score_by_class_dict: dict) -> None: - f1_score_report_file_name = f'{MeasurerArgument.F1_SCORES_BY_CLS.value}{Extension.CSV.value}' - f1_score_report_path = Path(output_directory_path).parent / f1_score_report_file_name - f1_score_report_df = pd.DataFrame({MeasurerArgument.F1_SCORE.value: f1_score_by_class_dict.values(), - 'inspection_id': range(len(f1_score_by_class_dict.values()))}) - write_dataframe_to_csv(f1_score_report_path, f1_score_report_df) - - -def main(): - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - if args.output_directory_path is None: - args.output_directory_path = Path(args.test_dataset_path).parent / f'predictions{Extension.CSV.value}' - - test_dataset = QodanaDataset(args.test_dataset_path, args.context_length) - num_labels = test_dataset[0][DatasetColumnArgument.LABELS.value].shape[0] - eval_dataloader = DataLoader(test_dataset, batch_size=args.batch_size) - predictions = np.zeros([len(test_dataset), num_labels], dtype=object) - - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") - model = RobertaForSequenceClassification.from_pretrained(args.model_weights_directory_path, - num_labels=num_labels).to(device) - model.eval() - - predictions = get_predictions(eval_dataloader, model, predictions, num_labels, device, args) - true_labels = torch.tensor(pd.read_csv(args.test_dataset_path).iloc[:, 1:].to_numpy()) - metric = Measurer(args.threshold) - f1_score_by_class_dict = metric.f1_score_by_classes(torch.tensor(predictions.to_numpy()), true_labels) - - print(f"{MeasurerArgument.F1_SCORE.value}:" - f"{metric.get_f1_score(torch.tensor(predictions.to_numpy()), true_labels)}", - f"\n{MeasurerArgument.F1_SCORES_BY_CLS.value}: {f1_score_by_class_dict}") - - write_dataframe_to_csv(args.output_directory_path, predictions) - if args.save_f1_score: - save_f1_scores(args.output_directory_path, f1_score_by_class_dict) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/qodana/imitation_model/preprocessing/README.md b/src/python/evaluation/qodana/imitation_model/preprocessing/README.md deleted file mode 100644 index 76e2b9d0..00000000 --- a/src/python/evaluation/qodana/imitation_model/preprocessing/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Data preprocessing - -This module transforms filtered and labeled dataset into the files that can be used as input -files for [train](src/python/evaluation/qodana/imitation_model/train.py) and -[evaluation](src/python/evaluation/qodana/imitation_model/evaluation.py) scripts. - -### Step 1 - -Run [fragment_to_inspections_list.py](https://github.com/hyperskill/hyperstyle/blob/roberta-model/src/python/evaluation/qodana/fragment_to_inspections_list.py) -script to get `numbered_ids.csv` file in case of working with code-blocks or alternatively run -[fragment_to_inspections_list_line_by_line.py](https://github.com/hyperskill/hyperstyle/blob/roberta-model/src/python/evaluation/qodana/fragment_to_inspections_list_line_by_line.py) -script to get `numbered_ids_line_by_line.csv` file. - -[Detailed instructions](https://github.com/hyperskill/hyperstyle/tree/roberta-model/src/python/evaluation/qodana) -on how to run following scripts. - -### Step 2 - -Run [encode_data.py](https://github.com/hyperskill/hyperstyle/blob/roberta-model/src/python/model/preprocessing/encode_data.py) with the -following arguments: - -Required arguments: - -`dataset_path` — path to `numbered_ids_line_by_line.csv` file or `numbered_ids.csv` file. - -Optional arguments: - -Argument | Description ---- | --- -|**‑o**, **‑‑output_file_path**| Path to the directory where output file will be created. If not set, output file will be saved in the parent directory of `dataset_path`.| -|**‑ohe**, **‑‑one_hot_encoding**| If `True` target column will be represented as one-hot-encoded vector. The length of each vector is equal to the unique number of classes in dataset. Default is `True`.| -|**‑c**, **‑‑add_context**| Should be used only when `dataset_path` is a path to `numbered_ids_line_by_line.csv`. If set to `True` each single line will be substituted by a piece of code – the context created from several lines. Default is `False`.| -|**‑n**, **‑‑n_lines_to_add**| A number of lines to append to the target line before and after it. A line is appended only if it matches the same solution. If there are not enough lines in the solution, special token will be appended instead. Default is 2.| - - -#### Script functionality overview: -- creates `one-hot-encoding` vectors matches each samples each sample in the dataset **(default)**. -- substitutes `NaN` values in the dataset by `\n` symbol **(default)**. -- transform lines of code into the `context` from several lines of code **(optional)**. - -### Step 3 - -Run [`split_dataset.py`](https://github.com/hyperskill/hyperstyle/blob/roberta-model/src/python/model/preprocessing/split_dataset.py) -with the following arguments: - -Required arguments: - -`dataset_path` — path to `encoded_dataset.csv` file obtained by running [encode_data.py](https://github.com/hyperskill/hyperstyle/blob/roberta-model/src/python/model/preprocessing/encode_data.py) script. - -Optional arguments: - -Argument | Description ---- | --- -|**‑o**, **‑‑output_directory_path**| Path to the directory where folders for train, test and validation datasets with the corresponding files will be created. If not set, folders will be created in the parent directory of `dataset_path`.| -|**‑ts**, **‑‑test_size**| Proportion of test dataset. Available values: 0 < n < 1. Default is 0.2.| -|**‑vs**, **‑‑val_size**| Proportion of validation dataset that will be taken from train dataset. Available values are: 0 < n < 1. Default is 0.3.| -|**‑sh**, **‑‑shuffle**| If `True` data will be shuffled before split. Default is `True`.| diff --git a/src/python/evaluation/qodana/imitation_model/preprocessing/__init__.py b/src/python/evaluation/qodana/imitation_model/preprocessing/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/qodana/imitation_model/preprocessing/encode_data.py b/src/python/evaluation/qodana/imitation_model/preprocessing/encode_data.py deleted file mode 100644 index 8b57888a..00000000 --- a/src/python/evaluation/qodana/imitation_model/preprocessing/encode_data.py +++ /dev/null @@ -1,162 +0,0 @@ -import argparse -import logging -import sys -from itertools import chain -from pathlib import Path -from typing import List - -import numpy as np -import pandas as pd -from sklearn.preprocessing import MultiLabelBinarizer -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.qodana.imitation_model.common.util import CustomTokens, DatasetColumnArgument -from src.python.review.common.file_system import Extension - - -logger = logging.getLogger(__name__) -sys.path.append('') -sys.path.append('../../../../..') - - -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('dataset_path', - type=lambda value: Path(value).absolute(), - help='Path to the dataset with the values to be encoded. ') - - parser.add_argument('-o', '--output_file_path', - help='Output file path. If not set, file will be saved to ' - 'the input file parent directory.', - type=str, - default='input_file_directory') - - parser.add_argument('-c', '--add_context', - help='Use for the datasets with code lines only, if set to True, ' - 'n lines before and n lines after target line will be added to each sample.' - ' Default is False.', - action='store_true') - - parser.add_argument('-n', '--n_lines_to_add', - help='Use only if add_context is enabled. Allows to add n-lines from the same piece of code, ' - 'before and after each line in the dataset. If there are no lines before or after a line' - 'from the same code-sample, special token will be added. Default is 2.', - default=2, - type=int) - - parser.add_argument('-ohe', '--one_hot_encoding', - help='If True, target column will be represented as one-hot-encoded vector. ' - 'The length of each vector is equal to the unique number of classes. ' - 'Default is True.', - action='store_false') - - -def __one_hot_encoding(df: pd.DataFrame) -> pd.DataFrame: - """ Transforms strings in 'inspections' column, - denoting inspection ids into n columns - with binary values: - - 1 x n_rows -> n_unique_classes x n_rows - - Where n_unique_classes is equal to the number - of unique inspections in the dataset. - - Example: - inspections -> 1, 2, 3 - '1, 2' 1 1 0 - '3' 0 0 1 - """ - target = df[DatasetColumnArgument.INSPECTIONS.value].to_numpy().astype(str) - target_list_int = [np.unique(tuple(map(int, label.split(',')))) for label in target] - try: - mlb = MultiLabelBinarizer() - encoded_target = mlb.fit_transform(target_list_int) - assert len(list(set(chain.from_iterable(target_list_int)))) == encoded_target.shape[1] - encoded_target = pd.DataFrame(data=encoded_target, columns=range(encoded_target.shape[1])) - return encoded_target - except AssertionError as e: - logger.error('encoded_target.shape[1] should be equal to number of classes') - raise e - - -class Context: - """ To each line of code add context from the same solution: - 'n_lines_before' line 'n_lines_after'. - If there are no lines before or / and after a piece of code, - special tokens are added. - """ - def __init__(self, df: pd.DataFrame, n_lines: int): - self.indices = df[DatasetColumnArgument.ID.value].to_numpy() - self.lines = df[ColumnName.CODE.value] - self.n_lines: int = n_lines - self.df = df - - def add_context_to_lines(self) -> pd.DataFrame: - lines_with_context = [] - for current_line_index, current_line in enumerate(self.lines): - context = self.add_context_before(current_line_index, current_line) - context = self.add_context_after(context, current_line_index) - lines_with_context.append(context[0]) - lines_with_context = pd.Series(lines_with_context) - self.df[ColumnName.CODE.value] = lines_with_context - return self.df - - def add_context_before(self, current_line_index: int, current_line: str) -> List[str]: - """ Add n_lines lines before the target line from the same piece of code, - If there are less than n lines above the target line will add - a special token. - """ - context = [''] - for n_line_index in range(current_line_index - self.n_lines, self.n_lines): - if n_line_index >= len(self.lines): - return context - if n_line_index == 0 or self.indices[n_line_index] != self.indices[current_line_index]: - context = [context[0] + CustomTokens.NOC.value] - else: - context = [context[0] + self.lines.iloc[n_line_index]] - if n_line_index != self.n_lines - 1: - context = [context[0] + '\n'] - context = [context[0] + current_line] - return context - - def add_context_after(self, context: List, current_line_index: int) -> List[str]: - """ Add n_lines lines after the target line from the same piece of code, - If there are less than n lines after the target line will add - a special token. - """ - for n_line_index in range(current_line_index + 1, self.n_lines + current_line_index + 1): - if n_line_index >= len(self.lines) or self.indices[n_line_index] != self.indices[current_line_index]: - context = [context[0] + CustomTokens.NOC.value] - else: - context = [context[0] + self.lines.iloc[n_line_index]] - if n_line_index != self.n_lines - 1: - context = [context[0] + '\n'] - return context - - -def main() -> None: - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - - dataset_path = args.dataset_path - output_file_path = args.output_file_path - - if output_file_path == 'input_file_directory': - output_file_path = Path(dataset_path).parent / f'encoded_dataset{Extension.CSV.value}' - - # nan -> \n (empty rows) - df = pd.read_csv(dataset_path) - df[ColumnName.CODE.value].fillna('\n', inplace=True) - - if args.one_hot_encoding: - target = __one_hot_encoding(df) - df = pd.concat([df[[ColumnName.ID.value, ColumnName.CODE.value]], target], axis=1) - - if args.add_context: - df = Context(df, args.n_lines_to_add).add_context_to_lines() - - write_dataframe_to_csv(output_file_path, df) - - -if __name__ == '__main__': - main() diff --git a/src/python/evaluation/qodana/imitation_model/preprocessing/split_dataset.py b/src/python/evaluation/qodana/imitation_model/preprocessing/split_dataset.py deleted file mode 100644 index 41a4319b..00000000 --- a/src/python/evaluation/qodana/imitation_model/preprocessing/split_dataset.py +++ /dev/null @@ -1,77 +0,0 @@ -import argparse -import os -from pathlib import Path - -import pandas as pd -from sklearn.model_selection import train_test_split -from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.qodana.imitation_model.common.util import SeedArgument -from src.python.review.common.file_system import Extension - - -def configure_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument('dataset_path', - type=str, - help=f'Path to the dataset received by either' - f' src.python.evaluation.qodana.fragment_to_inspections_list{Extension.PY.value}' - f'or src.python.evaluation.qodana.fragment_to_inspections_list_line_by_line' - f'{Extension.PY.value}script.') - - parser.add_argument('-d', '--output_directory_path', - type=str, - default=None, - help='Path to the directory where folders for train, test and validation datasets will be ' - 'created. If not set directories will be created in the parent directory of dataset_path') - - parser.add_argument('-ts', '--test_size', - type=int, - default=0.2, - help='Rate of test size from the whole dataset. Default is 0.2') - - parser.add_argument('-vs', '--val_size', - type=int, - default=0.3, - help='Rate of validation dataset from the train dataset. Default is 0.3 ') - - parser.add_argument('-sh', '--shuffle', - type=bool, - default=True, - help='If true, data will be shuffled before splitting. Default is True.') - - return parser - - -def split_dataset(dataset_path: str, output_directory_path: str, val_size: float, test_size: float, shuffle: bool): - df = pd.read_csv(dataset_path) - target = df.iloc[:, 2:] - code_bank = df[ColumnName.CODE.value] - - code_train, code_test, target_train, target_test = train_test_split(code_bank, - target, - test_size=test_size, - random_state=SeedArgument.SEED.value, - shuffle=shuffle) - - code_train, code_val, target_train, target_val = train_test_split(code_train, - target_train, - test_size=val_size, - random_state=SeedArgument.SEED.value, - shuffle=shuffle) - if output_directory_path is None: - output_directory_path = Path(dataset_path).parent - - for holdout in [("train", code_train, target_train), - ("val", code_val, target_val), - ("test", code_test, target_test)]: - df = pd.concat([holdout[1], holdout[2]], axis=1) - os.makedirs(os.path.join(output_directory_path, holdout[0]), exist_ok=True) - write_dataframe_to_csv(Path(output_directory_path) / holdout[0] / f'{holdout[0]}{Extension.CSV.value}', df) - - -if __name__ == "__main__": - parser = configure_parser() - args = parser.parse_args() - - split_dataset(args.dataset_path, args.output_directory_path, args.val_size, args.test_size, args.shuffle) diff --git a/src/python/evaluation/qodana/imitation_model/train.py b/src/python/evaluation/qodana/imitation_model/train.py deleted file mode 100644 index fc458d95..00000000 --- a/src/python/evaluation/qodana/imitation_model/train.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import os -import sys -from pathlib import Path - -import torch -from src.python.evaluation.qodana.imitation_model.common.metric import Measurer -from src.python.evaluation.qodana.imitation_model.common.train_config import ( - configure_arguments, MultilabelTrainer, TrainingArgs, -) -from src.python.evaluation.qodana.imitation_model.common.util import DatasetColumnArgument -from src.python.evaluation.qodana.imitation_model.dataset.dataset import QodanaDataset -from transformers import RobertaForSequenceClassification - - -def main(): - parser = argparse.ArgumentParser() - configure_arguments(parser) - args = parser.parse_args() - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") - train_dataset = QodanaDataset(args.train_dataset_path, args.context_length) - val_dataset = QodanaDataset(args.val_dataset_path, args.context_length) - train_steps_to_be_made = len(train_dataset) // args.batch_size - val_steps_to_be_made = train_steps_to_be_made // 5 - print(f'Steps to be made: {train_steps_to_be_made}, validate each {val_steps_to_be_made}th step.') - - num_labels = train_dataset[0][DatasetColumnArgument.LABELS.value].shape[0] - model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels).to(device) - - metrics = Measurer(args.threshold) - if args.trained_weights_directory_path is None: - args.trained_weights_directory_path = Path(args.train_dataset_path).parent / DatasetColumnArgument.WEIGHTS.value - os.makedirs(args.trained_weights_directory_path, exist_ok=True) - - train_args = TrainingArgs(args) - - trainer = MultilabelTrainer(model=model, - args=train_args.get_training_args(val_steps_to_be_made), - train_dataset=train_dataset, - eval_dataset=val_dataset, - compute_metrics=metrics.compute_metric) - trainer.train() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/python/evaluation/qodana/util/__init__.py b/src/python/evaluation/qodana/util/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/python/evaluation/qodana/util/issue_types.py b/src/python/evaluation/qodana/util/issue_types.py deleted file mode 100644 index aa495e43..00000000 --- a/src/python/evaluation/qodana/util/issue_types.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Dict - -from src.python.review.inspectors.issue import IssueType - -QODANA_CLASS_NAME_TO_ISSUE_TYPE: Dict[str, IssueType] = { -} diff --git a/src/python/evaluation/qodana/util/models.py b/src/python/evaluation/qodana/util/models.py deleted file mode 100644 index 08ce4c9f..00000000 --- a/src/python/evaluation/qodana/util/models.py +++ /dev/null @@ -1,66 +0,0 @@ -import json -from dataclasses import dataclass -from enum import Enum, unique -from typing import List - - -@dataclass(frozen=True) -class QodanaIssue: - fragment_id: int - line: int - offset: int - length: int - highlighted_element: str - description: str - problem_id: str - - def to_json(self) -> str: - issue = { - QodanaJsonField.FRAGMENT_ID.value: self.fragment_id, - QodanaJsonField.LINE.value: self.line, - QodanaJsonField.OFFSET.value: self.offset, - QodanaJsonField.LENGTH.value: self.length, - QodanaJsonField.HIGHLIGHTED_ELEMENT.value: self.highlighted_element, - QodanaJsonField.DESCRIPTION.value: self.description, - QodanaJsonField.PROBLEM_ID.value: self.problem_id, - } - return json.dumps(issue) - - @classmethod - def from_json(cls, str_json: str) -> 'QodanaIssue': - issue = json.loads(str_json) - return QodanaIssue( - fragment_id=issue[QodanaJsonField.FRAGMENT_ID.value], - line=issue[QodanaJsonField.LINE.value], - offset=issue[QodanaJsonField.OFFSET.value], - length=issue[QodanaJsonField.LENGTH.value], - highlighted_element=issue[QodanaJsonField.HIGHLIGHTED_ELEMENT.value], - description=issue[QodanaJsonField.DESCRIPTION.value], - problem_id=issue[QodanaJsonField.PROBLEM_ID.value], - ) - - @classmethod - def parse_list_issues_from_json(cls, str_json: str) -> List['QodanaIssue']: - return list(map(lambda i: QodanaIssue.from_json(i), json.loads(str_json)[QodanaJsonField.ISSUES.value])) - - -@unique -class QodanaColumnName(Enum): - INSPECTIONS = 'inspections' - ID = 'id' - INSPECTION_ID = 'inspection_id' - COUNT_ALL = 'count_all' - COUNT_UNIQUE = 'count_unique' - - -@unique -class QodanaJsonField(Enum): - FRAGMENT_ID = 'fragment_id' - LINE = 'line' - OFFSET = 'offset' - LENGTH = 'length' - HIGHLIGHTED_ELEMENT = 'highlighted_element' - DESCRIPTION = 'description' - PROBLEM_ID = 'problem_id' - - ISSUES = 'issues' diff --git a/src/python/evaluation/qodana/util/util.py b/src/python/evaluation/qodana/util/util.py deleted file mode 100644 index 3766b09d..00000000 --- a/src/python/evaluation/qodana/util/util.py +++ /dev/null @@ -1,51 +0,0 @@ -import argparse -import json -from pathlib import Path -from typing import Dict, List - -import pandas as pd -from src.python.common.tool_arguments import RunToolArgument -from src.python.evaluation.qodana.util.models import QodanaColumnName, QodanaIssue, QodanaJsonField - - -def to_json(issues: List[QodanaIssue]) -> str: - issues_json = { - QodanaJsonField.ISSUES.value: list(map(lambda i: i.to_json(), issues)), - } - return json.dumps(issues_json) - - -# Get a dictionary: Qodana inspection_id -> inspection_id from csv file with two columns: id, inspection_id -def get_inspections_dict(inspections_path: str) -> Dict[str, int]: - inspections_df = pd.read_csv(inspections_path) - inspections_dict = inspections_df.set_index(QodanaColumnName.INSPECTION_ID.value).T.to_dict('list') - for qodana_id, id_list in inspections_dict.items(): - inspections_dict[qodana_id] = id_list[0] - return inspections_dict - - -def replace_inspections_on_its_ids(issues_list: List[QodanaIssue], inspections_dict: Dict[str, int], - to_remove_duplicates: bool) -> str: - if len(issues_list) == 0: - inspections = '0' - else: - problem_id_list = list(map(lambda i: inspections_dict[i.problem_id], issues_list)) - if to_remove_duplicates: - problem_id_list = list(set(problem_id_list)) - problem_id_list.sort() - inspections = ','.join(str(p) for p in problem_id_list) - return inspections - - -def configure_model_converter_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument(RunToolArgument.QODANA_SOLUTIONS_FILE_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.QODANA_SOLUTIONS_FILE_PATH.value.description) - - parser.add_argument(RunToolArgument.QODANA_INSPECTIONS_PATH.value.long_name, - type=lambda value: Path(value).absolute(), - help=RunToolArgument.QODANA_INSPECTIONS_PATH.value.description) - - parser.add_argument(RunToolArgument.QODANA_DUPLICATES.value.long_name, - help=RunToolArgument.QODANA_DUPLICATES.value.description, - action='store_true') diff --git a/src/python/review/quality/penalty.py b/src/python/review/quality/penalty.py index ff4a6e13..e8cfce2f 100644 --- a/src/python/review/quality/penalty.py +++ b/src/python/review/quality/penalty.py @@ -6,6 +6,12 @@ from src.python.review.inspectors.issue import BaseIssue, IssueType from src.python.review.quality.model import QualityType + +@dataclass(frozen=True, eq=True) +class PenaltyIssue(BaseIssue): + influence_on_penalty: int + + # TODO: need testing ISSUE_TYPE_TO_PENALTY_COEFFICIENT = { IssueType.COHESION: 0.6, diff --git a/src/python/review/reviewers/utils/print_review.py b/src/python/review/reviewers/utils/print_review.py index f7aa62eb..1467c45c 100644 --- a/src/python/review/reviewers/utils/print_review.py +++ b/src/python/review/reviewers/utils/print_review.py @@ -4,12 +4,12 @@ from pathlib import Path from typing import Any, Dict, List, Union -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue from src.python.review.application_config import ApplicationConfig from src.python.review.common.file_system import get_file_line from src.python.review.inspectors.inspector_type import InspectorType from src.python.review.inspectors.issue import BaseIssue, IssueDifficulty, IssueType from src.python.review.quality.model import QualityType +from src.python.review.quality.penalty import PenaltyIssue from src.python.review.reviewers.review_result import FileReviewResult, GeneralReviewResult, ReviewResult diff --git a/test/python/common/__init__.py b/test/python/common/__init__.py deleted file mode 100644 index 689b1893..00000000 --- a/test/python/common/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from test.python import TEST_DATA_FOLDER - -CURRENT_TEST_DATA_FOLDER = TEST_DATA_FOLDER / 'common' - -FILE_SYSTEM_DATA_FOLDER = CURRENT_TEST_DATA_FOLDER / 'file_system' diff --git a/test/python/common/file_system/__init__.py b/test/python/common/file_system/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/python/common/file_system/test_subprocess.py b/test/python/common/file_system/test_subprocess.py deleted file mode 100644 index c3a60ffd..00000000 --- a/test/python/common/file_system/test_subprocess.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -from pathlib import Path -from test.python.common import FILE_SYSTEM_DATA_FOLDER -from test.python.evaluation.testing_config import get_testing_arguments -from typing import Optional - -import pytest -from src.python.evaluation.evaluation_config import EvaluationConfig -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import create_file, get_content_from_file -from src.python.review.common.subprocess_runner import run_in_subprocess - -INPUT_DATA = [ - ('in_1.java', LanguageVersion.JAVA_11), - ('in_2.py', LanguageVersion.PYTHON_3), -] - - -def inspect_code(config: EvaluationConfig, file: str, language: LanguageVersion, history: Optional[str] = None) -> str: - command = config.build_command(file, language.value, history) - return run_in_subprocess(command) - - -@pytest.mark.parametrize(('test_file', 'language'), INPUT_DATA) -def test_synthetic_files(test_file: str, language: LanguageVersion): - input_file = FILE_SYSTEM_DATA_FOLDER / test_file - test_args = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) - config = EvaluationConfig(test_args) - - expected_output = inspect_code(config, input_file, language) - - input_code = get_content_from_file(Path(input_file)) - actual_file = next(create_file(FILE_SYSTEM_DATA_FOLDER / f'actual_file{language.extension_by_language().value}', - input_code)) - - actual_output = inspect_code(config, actual_file, language) - os.remove(actual_file) - - assert actual_output == expected_output diff --git a/test/python/common_util.py b/test/python/common_util.py deleted file mode 100644 index 4823bec8..00000000 --- a/test/python/common_util.py +++ /dev/null @@ -1,20 +0,0 @@ -from pathlib import Path -from typing import List, Tuple - -import pandas as pd -from src.python.review.common.file_system import ( - Extension, get_all_file_system_items, match_condition, pair_in_and_out_files, -) - - -def get_in_and_out_list(root: Path, - in_ext: Extension = Extension.CSV, - out_ext: Extension = Extension.CSV) -> List[Tuple[Path, Path]]: - in_files = get_all_file_system_items(root, match_condition(rf'in_\d+{in_ext.value}')) - out_files = get_all_file_system_items(root, match_condition(rf'out_\d+{out_ext.value}')) - return pair_in_and_out_files(in_files, out_files) - - -def equal_df(expected_df: pd.DataFrame, actual_df: pd.DataFrame) -> bool: - return expected_df.reset_index(drop=True).equals( - actual_df.reset_index(drop=True)) or (expected_df.empty and actual_df.empty) diff --git a/test/python/evaluation/__init__.py b/test/python/evaluation/__init__.py deleted file mode 100644 index 293fdcae..00000000 --- a/test/python/evaluation/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from test.python import TEST_DATA_FOLDER - -from src.python import MAIN_FOLDER - -CURRENT_TEST_DATA_FOLDER = TEST_DATA_FOLDER / 'evaluation' - -XLSX_DATA_FOLDER = CURRENT_TEST_DATA_FOLDER / 'xlsx_files' - -TARGET_XLSX_DATA_FOLDER = CURRENT_TEST_DATA_FOLDER / 'xlsx_target_files' - -RESULTS_DIR_PATH = MAIN_FOLDER.parent / 'evaluation/results' - -EVALUATION_COMMON_DIR_PATH = CURRENT_TEST_DATA_FOLDER / 'common' - -PANDAS_UTIL_DIR_PATH = EVALUATION_COMMON_DIR_PATH / 'pandas_util' - -INSPECTORS_DIR_PATH = EVALUATION_COMMON_DIR_PATH / 'inspectors' diff --git a/test/python/evaluation/common/__init__.py b/test/python/evaluation/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/python/evaluation/common/pandas_util/__init__.py b/test/python/evaluation/common/pandas_util/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/python/evaluation/common/pandas_util/test_drop_duplicates.py b/test/python/evaluation/common/pandas_util/test_drop_duplicates.py deleted file mode 100644 index acd47445..00000000 --- a/test/python/evaluation/common/pandas_util/test_drop_duplicates.py +++ /dev/null @@ -1,18 +0,0 @@ -from pathlib import Path -from test.python.common_util import equal_df, get_in_and_out_list -from test.python.evaluation import PANDAS_UTIL_DIR_PATH - -import pytest -from src.python.evaluation.common.pandas_util import drop_duplicates, get_solutions_df_by_file_path - -RESOURCES_PATH = PANDAS_UTIL_DIR_PATH / 'drop_duplicates' - -IN_AND_OUT_FILES = get_in_and_out_list(RESOURCES_PATH) - - -@pytest.mark.parametrize(('in_file', 'out_file'), IN_AND_OUT_FILES) -def test(in_file: Path, out_file: Path): - in_df = get_solutions_df_by_file_path(in_file) - out_df = get_solutions_df_by_file_path(out_file) - filtered_df = drop_duplicates(in_df) - assert equal_df(out_df, filtered_df) diff --git a/test/python/evaluation/common/pandas_util/test_filter_by_language.py b/test/python/evaluation/common/pandas_util/test_filter_by_language.py deleted file mode 100644 index 25af150d..00000000 --- a/test/python/evaluation/common/pandas_util/test_filter_by_language.py +++ /dev/null @@ -1,29 +0,0 @@ -from pathlib import Path -from test.python.common_util import equal_df, get_in_and_out_list -from test.python.evaluation import PANDAS_UTIL_DIR_PATH - -import pytest -from src.python.evaluation.common.pandas_util import filter_df_by_language, get_solutions_df_by_file_path -from src.python.review.application_config import LanguageVersion -from src.python.review.common.file_system import get_name_from_path - -RESOURCES_PATH = PANDAS_UTIL_DIR_PATH / 'filter_by_language' - - -IN_FILE_TO_LANGUAGES = { - 'in_1.csv': set(LanguageVersion), - 'in_2.csv': set(), - 'in_3.csv': [LanguageVersion.PYTHON_3], - 'in_4.csv': [LanguageVersion.PYTHON_3, LanguageVersion.PYTHON_3], - 'in_5.csv': [LanguageVersion.PYTHON_3, LanguageVersion.JAVA_11], -} - -IN_AND_OUT_FILES = get_in_and_out_list(RESOURCES_PATH) - - -@pytest.mark.parametrize(('in_file', 'out_file'), IN_AND_OUT_FILES) -def test(in_file: Path, out_file: Path): - in_df = get_solutions_df_by_file_path(in_file) - out_df = get_solutions_df_by_file_path(out_file) - filtered_df = filter_df_by_language(in_df, IN_FILE_TO_LANGUAGES[get_name_from_path(str(in_file))]) - assert equal_df(out_df, filtered_df) diff --git a/test/python/evaluation/inspectors/__init__.py b/test/python/evaluation/inspectors/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/python/evaluation/inspectors/diffs_between_df/__init__.py b/test/python/evaluation/inspectors/diffs_between_df/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/python/evaluation/inspectors/diffs_between_df/test_diifs_between_df.py b/test/python/evaluation/inspectors/diffs_between_df/test_diifs_between_df.py deleted file mode 100644 index 72571842..00000000 --- a/test/python/evaluation/inspectors/diffs_between_df/test_diifs_between_df.py +++ /dev/null @@ -1,98 +0,0 @@ -from pathlib import Path -from test.python.evaluation import INSPECTORS_DIR_PATH - -import pytest -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.common.util import ColumnName -from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.evaluation.inspectors.diffs_between_df import find_diffs -from src.python.review.inspectors.inspector_type import InspectorType -from src.python.review.inspectors.issue import IssueDifficulty, IssueType - -RESOURCES_PATH = INSPECTORS_DIR_PATH / 'diffs_between_df' - -EMPTY_DIFFS = { - ColumnName.GRADE.value: [], - ColumnName.DECREASED_GRADE.value: [], - ColumnName.USER.value: 0, - ColumnName.TRACEBACK.value: {}, - ColumnName.PENALTY.value: {}, -} - -INCORRECT_GRADE_DIFFS = { - ColumnName.GRADE.value: [1, 2], - ColumnName.DECREASED_GRADE.value: [], - ColumnName.USER.value: 0, - ColumnName.TRACEBACK.value: {}, - ColumnName.PENALTY.value: {}, -} - -ISSUES = { - PenaltyIssue( - origin_class='C0305', - description='Trailing newlines', - line_no=15, - column_no=1, - type=IssueType('CODE_STYLE'), - - file_path=Path(), - inspector_type=InspectorType.UNDEFINED, - influence_on_penalty=0, - difficulty=IssueDifficulty.EASY, - ), PenaltyIssue( - origin_class='E211', - description='whitespace before \'(\'', - line_no=1, - column_no=6, - type=IssueType('CODE_STYLE'), - - file_path=Path(), - inspector_type=InspectorType.UNDEFINED, - influence_on_penalty=0, - difficulty=IssueDifficulty.EASY, - ), -} - -ISSUES_DIFFS = { - ColumnName.GRADE.value: [], - ColumnName.DECREASED_GRADE.value: [], - ColumnName.USER.value: 0, - ColumnName.TRACEBACK.value: { - 1: ISSUES, - }, - ColumnName.PENALTY.value: {}, -} - -MIXED_DIFFS = { - ColumnName.GRADE.value: [2, 3], - ColumnName.DECREASED_GRADE.value: [], - ColumnName.USER.value: 0, - ColumnName.TRACEBACK.value: { - 1: ISSUES, - }, - ColumnName.PENALTY.value: {}, -} - -DECREASED_GRADE = { - ColumnName.GRADE.value: [], - ColumnName.DECREASED_GRADE.value: [2, 3], - ColumnName.USER.value: 0, - ColumnName.TRACEBACK.value: {}, - ColumnName.PENALTY.value: {}, -} - -TEST_DATA = [ - ('old_1.csv', 'new_1.csv', EMPTY_DIFFS), - ('old_2.csv', 'new_2.csv', INCORRECT_GRADE_DIFFS), - ('old_3.csv', 'new_3.csv', ISSUES_DIFFS), - ('old_4.csv', 'new_4.csv', MIXED_DIFFS), - ('old_5.csv', 'new_5.csv', DECREASED_GRADE), -] - - -@pytest.mark.parametrize(('old_file', 'new_file', 'diffs'), TEST_DATA) -def test(old_file: Path, new_file: Path, diffs: dict): - old_df = get_solutions_df_by_file_path(RESOURCES_PATH / old_file) - new_df = get_solutions_df_by_file_path(RESOURCES_PATH / new_file) - actual_diffs = find_diffs(old_df, new_df) - assert actual_diffs == diffs diff --git a/test/python/evaluation/issues_statistics/__init__.py b/test/python/evaluation/issues_statistics/__init__.py deleted file mode 100644 index 604e3d61..00000000 --- a/test/python/evaluation/issues_statistics/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from test.python.evaluation import CURRENT_TEST_DATA_FOLDER - -ISSUES_STATISTICS_TEST_DATA_FOLDER = CURRENT_TEST_DATA_FOLDER / 'issues_statistics' - -GET_RAW_ISSUES_DATA_FOLDER = ISSUES_STATISTICS_TEST_DATA_FOLDER / 'get_raw_issues' - -GET_RAW_ISSUES_TEST_FILES_FOLDER = GET_RAW_ISSUES_DATA_FOLDER / 'test_files' - -GET_RAW_ISSUES_TARGET_FILES_FOLDER = GET_RAW_ISSUES_DATA_FOLDER / 'target_files' - -GET_RAW_ISSUES_STATISTICS_DATA_FOLDER = ISSUES_STATISTICS_TEST_DATA_FOLDER / 'get_raw_issues_statistics' - -GET_RAW_ISSUES_STATISTICS_TEST_FILES_FOLDER = GET_RAW_ISSUES_STATISTICS_DATA_FOLDER / 'test_files' - -GET_RAW_ISSUES_STATISTICS_TARGET_FILES_FOLDER = GET_RAW_ISSUES_STATISTICS_DATA_FOLDER / 'target_files' diff --git a/test/python/evaluation/issues_statistics/test_get_raw_issues.py b/test/python/evaluation/issues_statistics/test_get_raw_issues.py deleted file mode 100644 index 976701ba..00000000 --- a/test/python/evaluation/issues_statistics/test_get_raw_issues.py +++ /dev/null @@ -1,343 +0,0 @@ -from pathlib import Path -from test.python.common_util import equal_df -from test.python.evaluation.issues_statistics import ( - GET_RAW_ISSUES_TARGET_FILES_FOLDER, GET_RAW_ISSUES_TEST_FILES_FOLDER, -) -from typing import List, Optional - -import pandas as pd -import pytest -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.issues_statistics.get_raw_issues import _filter_issues, _get_output_path, inspect_solutions -from src.python.review.inspectors.inspector_type import InspectorType -from src.python.review.inspectors.issue import ( - BaseIssue, - CodeIssue, - IssueDifficulty, - IssueType, - LineLenIssue, - MaintainabilityLackIssue, -) - -ORIGINAL_DF_NAME = 'original_df' -ORIGINAL_DF_CSV = f'{ORIGINAL_DF_NAME}.csv' -ORIGINAL_DF_XLSX = f'{ORIGINAL_DF_NAME}.xlsx' - -ORIGINAL_DF_WITH_RAW_ISSUES_CSV = f'{ORIGINAL_DF_NAME}_with_raw_issues.csv' -ORIGINAL_DF_WITH_RAW_ISSUES_XLSX = f'{ORIGINAL_DF_NAME}_with_raw_issues.xlsx' - -NEW_DF_NAME = 'new_df' - -GET_OUTPUT_PATH_TEST_DATA = [ - (Path(ORIGINAL_DF_CSV), None, Path(ORIGINAL_DF_WITH_RAW_ISSUES_CSV)), - (Path(ORIGINAL_DF_XLSX), None, Path(ORIGINAL_DF_WITH_RAW_ISSUES_XLSX)), - (Path(ORIGINAL_DF_CSV), Path(f'{NEW_DF_NAME}.csv'), Path(f'{NEW_DF_NAME}.csv')), - (Path(ORIGINAL_DF_CSV), Path(f'{NEW_DF_NAME}.xlsx'), Path(f'{NEW_DF_NAME}.xlsx')), - (Path(ORIGINAL_DF_XLSX), Path(f'{NEW_DF_NAME}.csv'), Path(f'{NEW_DF_NAME}.csv')), - (Path(ORIGINAL_DF_XLSX), Path(f'{NEW_DF_NAME}.xlsx'), Path(f'{NEW_DF_NAME}.xlsx')), - (Path(ORIGINAL_DF_CSV), Path(NEW_DF_NAME), Path(ORIGINAL_DF_WITH_RAW_ISSUES_CSV)), - (Path(ORIGINAL_DF_XLSX), Path(NEW_DF_NAME), Path(ORIGINAL_DF_WITH_RAW_ISSUES_XLSX)), - (Path(ORIGINAL_DF_CSV), Path(f'{NEW_DF_NAME}/'), Path(ORIGINAL_DF_WITH_RAW_ISSUES_CSV)), - (Path(ORIGINAL_DF_XLSX), Path(f'{NEW_DF_NAME}/'), Path(ORIGINAL_DF_WITH_RAW_ISSUES_XLSX)), - (Path(ORIGINAL_DF_CSV), Path(f'{NEW_DF_NAME}.unknown'), Path(ORIGINAL_DF_WITH_RAW_ISSUES_CSV)), - (Path(ORIGINAL_DF_XLSX), Path(f'{NEW_DF_NAME}.unknown'), Path(ORIGINAL_DF_WITH_RAW_ISSUES_XLSX)), -] - - -@pytest.mark.parametrize(('solutions_file_path', 'output_path', 'expected_output_path'), GET_OUTPUT_PATH_TEST_DATA) -def test_get_output_path(solutions_file_path: Path, output_path: Optional[Path], expected_output_path: Path): - actual_output_path = _get_output_path(solutions_file_path, output_path) - assert actual_output_path == expected_output_path - - -ISSUES_FOR_FILTERING = [ - CodeIssue( - origin_class="MissingSwitchDefaultCheck", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=13, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.HARD, - ), - CodeIssue( - origin_class="SwitchStmtsShouldHaveDefault", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=1, - inspector_type=InspectorType.PMD, - difficulty=IssueDifficulty.HARD, - ), - CodeIssue( - origin_class="MagicNumberCheck", - type=IssueType.INFO, - description="Some description", - file_path=Path(""), - line_no=303, - column_no=25, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.EASY, - ), - MaintainabilityLackIssue( - origin_class="SomeMaintainabilityCheck", - type=IssueType.MAINTAINABILITY, - description="Some description", - file_path=Path(""), - line_no=574, - column_no=50, - inspector_type=InspectorType.CHECKSTYLE, - maintainability_lack=0, - difficulty=IssueDifficulty.HARD, - ), - LineLenIssue( - origin_class="SomeLineLenCheck", - type=IssueType.LINE_LEN, - description="Some description", - file_path=Path(""), - line_no=139, - column_no=24, - inspector_type=InspectorType.CHECKSTYLE, - line_len=10, - difficulty=IssueDifficulty.EASY, - ), -] - -ISSUES_WITHOUT_DUPLICATES = [ - CodeIssue( - origin_class="MissingSwitchDefaultCheck", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=13, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.HARD, - ), - CodeIssue( - origin_class="MagicNumberCheck", - type=IssueType.INFO, - description="Some description", - file_path=Path(""), - line_no=303, - column_no=25, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.EASY, - ), - MaintainabilityLackIssue( - origin_class="SomeMaintainabilityCheck", - type=IssueType.MAINTAINABILITY, - description="Some description", - file_path=Path(""), - line_no=574, - column_no=50, - inspector_type=InspectorType.CHECKSTYLE, - maintainability_lack=0, - difficulty=IssueDifficulty.HARD, - ), - LineLenIssue( - origin_class="SomeLineLenCheck", - type=IssueType.LINE_LEN, - description="Some description", - file_path=Path(""), - line_no=139, - column_no=24, - inspector_type=InspectorType.CHECKSTYLE, - line_len=10, - difficulty=IssueDifficulty.EASY, - ), -] - -ISSUES_WITHOUT_ZERO_MEASURE_ISSUES = [ - CodeIssue( - origin_class="MissingSwitchDefaultCheck", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=13, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.HARD, - ), - CodeIssue( - origin_class="SwitchStmtsShouldHaveDefault", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=1, - inspector_type=InspectorType.PMD, - difficulty=IssueDifficulty.HARD, - ), - CodeIssue( - origin_class="MagicNumberCheck", - type=IssueType.INFO, - description="Some description", - file_path=Path(""), - line_no=303, - column_no=25, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.EASY, - ), - LineLenIssue( - origin_class="SomeLineLenCheck", - type=IssueType.LINE_LEN, - description="Some description", - file_path=Path(""), - line_no=139, - column_no=24, - inspector_type=InspectorType.CHECKSTYLE, - line_len=10, - difficulty=IssueDifficulty.EASY, - ), -] - -ISSUES_WITHOUT_INFO_CATEGORY = [ - CodeIssue( - origin_class="MissingSwitchDefaultCheck", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=13, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.HARD, - ), - CodeIssue( - origin_class="SwitchStmtsShouldHaveDefault", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=1, - inspector_type=InspectorType.PMD, - difficulty=IssueDifficulty.HARD, - ), - MaintainabilityLackIssue( - origin_class="SomeMaintainabilityCheck", - type=IssueType.MAINTAINABILITY, - description="Some description", - file_path=Path(""), - line_no=574, - column_no=50, - inspector_type=InspectorType.CHECKSTYLE, - maintainability_lack=0, - difficulty=IssueDifficulty.HARD, - ), - LineLenIssue( - origin_class="SomeLineLenCheck", - type=IssueType.LINE_LEN, - description="Some description", - file_path=Path(""), - line_no=139, - column_no=24, - inspector_type=InspectorType.CHECKSTYLE, - line_len=10, - difficulty=IssueDifficulty.EASY, - ), -] - -FILTERED_ISSUES = [ - CodeIssue( - origin_class="MissingSwitchDefaultCheck", - type=IssueType.ERROR_PRONE, - description="Some description", - file_path=Path(""), - line_no=112, - column_no=13, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.HARD, - ), - LineLenIssue( - origin_class="SomeLineLenCheck", - type=IssueType.LINE_LEN, - description="Some description", - file_path=Path(""), - line_no=139, - column_no=24, - inspector_type=InspectorType.CHECKSTYLE, - line_len=10, - difficulty=IssueDifficulty.EASY, - ), -] - -FILTER_ISSUES_TEST_DATA = [ - ( - ISSUES_FOR_FILTERING, - True, # allow_duplicates - True, # allow_zero_measure_issues - True, # allow_info_issues - ISSUES_FOR_FILTERING, - ), - ( - ISSUES_FOR_FILTERING, - False, # allow_duplicates - True, # allow_zero_measure_issues - True, # allow_info_issues - ISSUES_WITHOUT_DUPLICATES, - ), - ( - ISSUES_FOR_FILTERING, - True, # allow_duplicates - False, # allow_zero_measure_issues - True, # allow_info_issues - ISSUES_WITHOUT_ZERO_MEASURE_ISSUES, - ), - ( - ISSUES_FOR_FILTERING, - True, # allow_duplicates - True, # allow_zero_measure_issues - False, # allow_info_issues - ISSUES_WITHOUT_INFO_CATEGORY, - ), - ( - ISSUES_FOR_FILTERING, - False, # allow_duplicates - False, # allow_zero_measure_issues - False, # allow_info_issues - FILTERED_ISSUES, - ), -] - - -@pytest.mark.parametrize( - ('issues', 'allow_duplicates', 'allow_zero_measure_issues', 'allow_info_issues', 'expected_issues'), - FILTER_ISSUES_TEST_DATA, -) -def test_filter_issues( - issues: List[BaseIssue], - allow_duplicates: bool, - allow_zero_measure_issues: bool, - allow_info_issues: bool, - expected_issues: List[BaseIssue], -): - assert _filter_issues(issues, allow_duplicates, allow_zero_measure_issues, allow_info_issues) == expected_issues - - -TEST_CORRECT_OUTPUT_DATA = [ - ('test_fragment_per_language.csv', 'target_fragment_per_language.csv'), - ('test_incorrect_language.csv', 'target_incorrect_language.csv'), - ('test_incorrect_code.csv', 'target_incorrect_code.csv'), - ('test_rows_with_null.csv', 'target_rows_with_null.csv'), -] - - -@pytest.mark.parametrize(('test_file', 'target_file'), TEST_CORRECT_OUTPUT_DATA) -def test_correct_output(test_file: str, target_file: str): - solutions_file_path = Path(GET_RAW_ISSUES_TEST_FILES_FOLDER / test_file) - solutions = get_solutions_df_by_file_path(solutions_file_path) - - test_dataframe = inspect_solutions( - solutions, - solutions_file_path, - allow_duplicates=False, - allow_info_issues=False, - allow_zero_measure_issues=False, - to_save_path=False, - ) - - target_dataframe = pd.read_csv(GET_RAW_ISSUES_TARGET_FILES_FOLDER / target_file) - - assert equal_df(target_dataframe, test_dataframe) diff --git a/test/python/evaluation/issues_statistics/test_get_raw_issues_statistics.py b/test/python/evaluation/issues_statistics/test_get_raw_issues_statistics.py deleted file mode 100644 index 24c10a67..00000000 --- a/test/python/evaluation/issues_statistics/test_get_raw_issues_statistics.py +++ /dev/null @@ -1,112 +0,0 @@ -from pathlib import Path -from test.python.common_util import equal_df -from test.python.evaluation.issues_statistics import ( - GET_RAW_ISSUES_STATISTICS_TARGET_FILES_FOLDER, - GET_RAW_ISSUES_STATISTICS_TEST_FILES_FOLDER, -) -from typing import Optional - -import pandas as pd -import pytest -from src.python.evaluation.common.pandas_util import get_solutions_df_by_file_path -from src.python.evaluation.issues_statistics.get_raw_issues_statistics import ( - _convert_language_code_to_language, - _get_output_folder, - DEFAULT_OUTPUT_FOLDER_NAME, - inspect_raw_issues, -) -from src.python.review.common.language import Language - -DF_PARENT_FOLDER_NAME = 'parent_folder' -DF_NAME = 'input_df' -DF_PATH = Path(DF_PARENT_FOLDER_NAME) / DF_NAME -DEFAULT_OUTPUT_PATH = Path(DF_PARENT_FOLDER_NAME) / DEFAULT_OUTPUT_FOLDER_NAME - -NEW_FOLDER = 'new_folder' - -GET_OUTPUT_FOLDER_PATH_TEST_DATA = [ - (DF_PATH, None, DEFAULT_OUTPUT_PATH), - (DF_PATH, Path(NEW_FOLDER), Path(NEW_FOLDER)), -] - - -@pytest.mark.parametrize( - ('solutions_file_path', 'output_folder', 'expected_output_folder'), - GET_OUTPUT_FOLDER_PATH_TEST_DATA, -) -def test_get_output_folder(solutions_file_path: Path, output_folder: Optional[Path], expected_output_folder: Path): - actual_output_folder = _get_output_folder(solutions_file_path, output_folder) - assert actual_output_folder == expected_output_folder - - -CONVERT_LANGUAGE_CODE_TO_LANGUAGE_TEST_DATA = [ - ('java7', 'JAVA'), - ('java8', 'JAVA'), - ('java9', 'JAVA'), - ('java11', 'JAVA'), - ('java15', 'JAVA'), - ('python3', 'PYTHON'), - ('kotlin', 'KOTLIN'), - ('javascript', 'JAVASCRIPT'), - ('some_weird_lang', 'some_weird_lang'), -] - - -@pytest.mark.parametrize(('language_code', 'expected_language'), CONVERT_LANGUAGE_CODE_TO_LANGUAGE_TEST_DATA) -def test_convert_language_code_to_language(language_code: str, expected_language: str): - actual_language = _convert_language_code_to_language(fragment_id='0', language_code=language_code) - assert actual_language == expected_language - - -INSPECT_SOLUTIONS_TEST_DATA = [ - ( - 'test_df_with_null.csv', - 'target_df_with_null_python.csv', - Language.PYTHON.value, - ), - ( - 'test_df_with_null.csv', - 'target_df_with_null_unknown.csv', - '', - ), - ( - 'test_df_with_empty_raw_issues.csv', - 'target_df_with_empty_raw_issues.csv', - Language.KOTLIN.value, - ), - ( - 'test_df_with_incorrect_language.csv', - 'target_df_with_incorrect_language.csv', - 'some_weird_lang', - ), - ( - 'test_df_single_lang.csv', - 'target_df_single_lang.csv', - Language.JAVA.value, - ), - ( - 'test_df_multi_lang.csv', - 'target_df_multi_lang_java.csv', - Language.JAVA.value, - ), - ( - 'test_df_multi_lang.csv', - 'target_df_multi_lang_js.csv', - Language.JS.value, - ), - ( - 'test_df_multi_lang.csv', - 'target_df_multi_lang_python.csv', - Language.PYTHON.value, - ), -] - - -@pytest.mark.parametrize(('test_file', 'target_file', 'lang'), INSPECT_SOLUTIONS_TEST_DATA) -def test_inspect_solutions(test_file: str, target_file: str, lang: str): - test_df = get_solutions_df_by_file_path(GET_RAW_ISSUES_STATISTICS_TEST_FILES_FOLDER / test_file) - stats = inspect_raw_issues(test_df) - - freq_stats = pd.read_csv(GET_RAW_ISSUES_STATISTICS_TARGET_FILES_FOLDER / target_file) - - assert equal_df(stats[lang], freq_stats) diff --git a/test/python/evaluation/issues_statistics/test_raw_issue_encoding_decoding.py b/test/python/evaluation/issues_statistics/test_raw_issue_encoding_decoding.py deleted file mode 100644 index 79abcbba..00000000 --- a/test/python/evaluation/issues_statistics/test_raw_issue_encoding_decoding.py +++ /dev/null @@ -1,228 +0,0 @@ -import json -import textwrap -from pathlib import Path - -import pytest -from src.python.evaluation.issues_statistics.common.raw_issue_encoder_decoder import RawIssueDecoder, RawIssueEncoder -from src.python.review.inspectors.inspector_type import InspectorType -from src.python.review.inspectors.issue import ( - BaseIssue, - BoolExprLenIssue, - CodeIssue, - CohesionIssue, - CyclomaticComplexityIssue, - FuncLenIssue, - IssueDifficulty, - IssueType, - LineLenIssue, - MaintainabilityLackIssue, -) - -FILE_PATH = 'some_file.py' -DESCRIPTION = 'Some description' - -ISSUE_AND_JSON_ISSUE = [ - ( - CodeIssue( - origin_class='SomeCodeIssueClass', - type=IssueType.CODE_STYLE, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=656, - column_no=42, - inspector_type=InspectorType.CHECKSTYLE, - difficulty=IssueDifficulty.EASY, - ), - f""" - {{ - "origin_class": "SomeCodeIssueClass", - "type": "CODE_STYLE", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 656, - "column_no": 42, - "inspector_type": "CHECKSTYLE", - "difficulty": "EASY" - }} - """, - ), - ( - BoolExprLenIssue( - origin_class='SomeBoolExprLenIssueClass', - type=IssueType.BOOL_EXPR_LEN, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=983, - column_no=428, - inspector_type=InspectorType.DETEKT, - bool_expr_len=975, - difficulty=IssueDifficulty.EASY, - ), - f""" - {{ - "origin_class": "SomeBoolExprLenIssueClass", - "type": "BOOL_EXPR_LEN", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 983, - "column_no": 428, - "inspector_type": "DETEKT", - "difficulty": "EASY", - "measure": 975 - }} - """, - ), - ( - FuncLenIssue( - origin_class='SomeFuncLenIssueClass', - type=IssueType.FUNC_LEN, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=790, - column_no=487, - inspector_type=InspectorType.ESLINT, - func_len=909, - difficulty=IssueDifficulty.EASY, - ), - f""" - {{ - "origin_class": "SomeFuncLenIssueClass", - "type": "FUNC_LEN", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 790, - "column_no": 487, - "inspector_type": "ESLINT", - "difficulty": "EASY", - "measure": 909 - }} - """, - ), - ( - LineLenIssue( - origin_class='SomeLineLenIssueClass', - type=IssueType.LINE_LEN, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=154, - column_no=383, - inspector_type=InspectorType.PMD, - line_len=383, - difficulty=IssueDifficulty.EASY, - ), - f""" - {{ - "origin_class": "SomeLineLenIssueClass", - "type": "LINE_LEN", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 154, - "column_no": 383, - "inspector_type": "PMD", - "difficulty": "EASY", - "measure": 383 - }} - """, - ), - ( - CyclomaticComplexityIssue( - origin_class='SomeCyclomaticComplexityIssueClass', - type=IssueType.CYCLOMATIC_COMPLEXITY, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=670, - column_no=78, - inspector_type=InspectorType.INTELLIJ, - cc_value=229, - difficulty=IssueDifficulty.HARD, - ), - f""" - {{ - "origin_class": "SomeCyclomaticComplexityIssueClass", - "type": "CYCLOMATIC_COMPLEXITY", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 670, - "column_no": 78, - "inspector_type": "INTELLIJ", - "difficulty": "HARD", - "measure": 229 - }} - """, - ), - ( - CohesionIssue( - origin_class='SomeCohesionIssueClass', - type=IssueType.COHESION, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=997, - column_no=386, - inspector_type=InspectorType.PYLINT, - cohesion_lack=564, - difficulty=IssueDifficulty.HARD, - ), - f""" - {{ - "origin_class": "SomeCohesionIssueClass", - "type": "COHESION", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 997, - "column_no": 386, - "inspector_type": "PYLINT", - "difficulty": "HARD", - "measure": 564 - }} - """, - ), - ( - MaintainabilityLackIssue( - origin_class='SomeMaintainabilityLackIssueClass', - type=IssueType.MAINTAINABILITY, - description=DESCRIPTION, - file_path=Path(FILE_PATH), - line_no=830, - column_no=542, - inspector_type=InspectorType.RADON, - maintainability_lack=431, - difficulty=IssueDifficulty.HARD, - ), - f""" - {{ - "origin_class": "SomeMaintainabilityLackIssueClass", - "type": "MAINTAINABILITY", - "description": "{DESCRIPTION}", - "file_path": "{FILE_PATH}", - "line_no": 830, - "column_no": 542, - "inspector_type": "RADON", - "difficulty": "HARD", - "measure": 431 - }} - """, - ), -] - - -@pytest.mark.parametrize(('issue', 'expected_json'), ISSUE_AND_JSON_ISSUE) -def test_encode_issue(issue: BaseIssue, expected_json: str): - assert json.dumps(issue, cls=RawIssueEncoder, indent=4) == textwrap.dedent(expected_json).strip() - - -@pytest.mark.parametrize(('expected_issue', 'json_issue'), ISSUE_AND_JSON_ISSUE) -def test_decode_issue(json_issue: str, expected_issue: BaseIssue): - assert json.loads(json_issue, cls=RawIssueDecoder) == expected_issue - - -@pytest.mark.parametrize(('issue', 'json_issue'), ISSUE_AND_JSON_ISSUE) -def test_encode_decode(issue: BaseIssue, json_issue: str): - assert json.loads(json.dumps(issue, cls=RawIssueEncoder), cls=RawIssueDecoder) == issue - - -@pytest.mark.parametrize(('issue', 'json_issue'), ISSUE_AND_JSON_ISSUE) -def test_decode_encode(issue: BaseIssue, json_issue: str): - assert ( - json.dumps(json.loads(json_issue, cls=RawIssueDecoder), cls=RawIssueEncoder, indent=4) - == textwrap.dedent(json_issue).strip() - ) diff --git a/test/python/evaluation/test_data_path.py b/test/python/evaluation/test_data_path.py deleted file mode 100644 index bae12c11..00000000 --- a/test/python/evaluation/test_data_path.py +++ /dev/null @@ -1,15 +0,0 @@ -from test.python.evaluation import XLSX_DATA_FOLDER -from test.python.evaluation.testing_config import get_testing_arguments - -import pytest -from src.python.evaluation.evaluation_config import EvaluationConfig -from src.python.evaluation.evaluation_run_tool import get_solutions_df, inspect_solutions_df - - -def test_incorrect_data_path(): - with pytest.raises(FileNotFoundError): - testing_arguments_dict = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) - testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / 'do_not_exist.xlsx' - config = EvaluationConfig(testing_arguments_dict) - lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - assert inspect_solutions_df(config, lang_code_dataframe) diff --git a/test/python/evaluation/test_output_results.py b/test/python/evaluation/test_output_results.py deleted file mode 100644 index 44508688..00000000 --- a/test/python/evaluation/test_output_results.py +++ /dev/null @@ -1,34 +0,0 @@ -from test.python.common_util import equal_df -from test.python.evaluation import TARGET_XLSX_DATA_FOLDER, XLSX_DATA_FOLDER -from test.python.evaluation.testing_config import get_testing_arguments - -import pandas as pd -import pytest -from src.python.evaluation.evaluation_config import EvaluationConfig -from src.python.evaluation.evaluation_run_tool import get_solutions_df, inspect_solutions_df - -FILE_NAMES = [ - ('test_sorted_order.xlsx', 'target_sorted_order.xlsx', False), - ('test_sorted_order.xlsx', 'target_sorted_order.xlsx', True), - ('test_unsorted_order.xlsx', 'target_unsorted_order.xlsx', False), - ('test_unsorted_order.xlsx', 'target_unsorted_order.xlsx', True), -] - - -@pytest.mark.parametrize(('test_file', 'target_file', 'output_type'), FILE_NAMES) -def test_correct_output(test_file: str, target_file: str, output_type: bool): - - testing_arguments_dict = get_testing_arguments(to_add_tool_path=True) - testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / test_file - testing_arguments_dict.traceback = output_type - - config = EvaluationConfig(testing_arguments_dict) - lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - test_dataframe = inspect_solutions_df(config, lang_code_dataframe) - - sheet_name = 'grades' - if output_type: - sheet_name = 'traceback' - target_dataframe = pd.read_excel(TARGET_XLSX_DATA_FOLDER / target_file, sheet_name=sheet_name) - - assert equal_df(target_dataframe, test_dataframe) diff --git a/test/python/evaluation/test_tool_path.py b/test/python/evaluation/test_tool_path.py deleted file mode 100644 index 8ee4cd07..00000000 --- a/test/python/evaluation/test_tool_path.py +++ /dev/null @@ -1,28 +0,0 @@ -from test.python.evaluation import XLSX_DATA_FOLDER -from test.python.evaluation.testing_config import get_testing_arguments - -import pytest -from src.python import MAIN_FOLDER -from src.python.evaluation.evaluation_config import EvaluationConfig -from src.python.evaluation.evaluation_run_tool import get_solutions_df, inspect_solutions_df - - -def test_correct_tool_path(): - try: - testing_arguments_dict = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) - testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / 'test_unsorted_order.xlsx' - config = EvaluationConfig(testing_arguments_dict) - lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - inspect_solutions_df(config, lang_code_dataframe) - except Exception: - pytest.fail("Unexpected error") - - -def test_incorrect_tool_path(): - with pytest.raises(Exception): - testing_arguments_dict = get_testing_arguments(to_add_traceback=True) - testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / 'test_unsorted_order.xlsx' - testing_arguments_dict.tool_path = MAIN_FOLDER.parent / 'review/incorrect_path.py' - config = EvaluationConfig(testing_arguments_dict) - lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - assert inspect_solutions_df(config, lang_code_dataframe) diff --git a/test/python/evaluation/test_xlsx_file_structure.py b/test/python/evaluation/test_xlsx_file_structure.py deleted file mode 100644 index f043772d..00000000 --- a/test/python/evaluation/test_xlsx_file_structure.py +++ /dev/null @@ -1,23 +0,0 @@ -from test.python.evaluation import XLSX_DATA_FOLDER -from test.python.evaluation.testing_config import get_testing_arguments - -import pytest -from src.python.evaluation.evaluation_config import EvaluationConfig -from src.python.evaluation.evaluation_run_tool import get_solutions_df, inspect_solutions_df - -FILE_NAMES = [ - 'test_wrong_column_name.xlsx', - 'test_java_no_version.xlsx', - 'test_empty_lang_cell.xlsx', - 'test_empty_table.xlsx', -] - - -@pytest.mark.parametrize('file_name', FILE_NAMES) -def test_wrong_column(file_name: str): - with pytest.raises(KeyError): - testing_arguments_dict = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) - testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / file_name - config = EvaluationConfig(testing_arguments_dict) - lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - assert inspect_solutions_df(config, lang_code_dataframe) diff --git a/test/python/evaluation/testing_config.py b/test/python/evaluation/testing_config.py deleted file mode 100644 index 4927e2ce..00000000 --- a/test/python/evaluation/testing_config.py +++ /dev/null @@ -1,25 +0,0 @@ -from argparse import Namespace - -from src.python import MAIN_FOLDER -from src.python.evaluation.common.util import EvaluationArgument -from src.python.review.reviewers.perform_review import OutputFormat - - -def get_testing_arguments(to_add_traceback=None, to_add_tool_path=None, to_add_history=None) -> Namespace: - testing_arguments = Namespace(format=OutputFormat.JSON.value, - output_file_name=EvaluationArgument.RESULT_FILE_NAME_XLSX.value, - output_folder_path=None, - with_history=False) - if to_add_traceback: - testing_arguments.traceback = True - - if to_add_tool_path: - testing_arguments.tool_path = MAIN_FOLDER.parent / 'review/run_tool.py' - - if to_add_history: - testing_arguments.with_history = True - - testing_arguments.solutions_file_path = None - testing_arguments.to_drop_nan = False - - return testing_arguments diff --git a/test/python/quality/test_penalty.py b/test/python/quality/test_penalty.py index a596531b..70717302 100644 --- a/test/python/quality/test_penalty.py +++ b/test/python/quality/test_penalty.py @@ -10,7 +10,7 @@ CURRENT_ISSUES = [ BaseIssue( - file_path=Path("."), + file_path=Path(""), line_no=1, column_no=1, description="Possibly misspelt word", @@ -20,7 +20,7 @@ difficulty=IssueDifficulty.MEDIUM, ), BaseIssue( - file_path=Path("."), + file_path=Path(""), line_no=10, column_no=5, description="Lambda may not be necessary", diff --git a/test/resources/common/file_system/in_1.java b/test/resources/common/file_system/in_1.java deleted file mode 100644 index 1e4a378b..00000000 --- a/test/resources/common/file_system/in_1.java +++ /dev/null @@ -1,8 +0,0 @@ -public class Main { - public static void main(String[] args) { - - int variable = 123456; - - System.out.println(variable); - } -} \ No newline at end of file diff --git a/test/resources/common/file_system/in_2.py b/test/resources/common/file_system/in_2.py deleted file mode 100644 index aa1637a2..00000000 --- a/test/resources/common/file_system/in_2.py +++ /dev/null @@ -1,13 +0,0 @@ -a = int(input()) -b = int(input()) -c = int(input()) -d = int(input()) - -if a > b: - print('a > b') - -if a > b and a > b: - print('a > b again') - -if a > b and a < d: - print('b < a < d') \ No newline at end of file