diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c08b6e9f..d4385934 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,7 @@ jobs: pip install flake8 pytest pip install -r requirements.txt pip install -r requirements-test.txt + pip install -r requirements-evaluation.txt - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -38,6 +39,14 @@ jobs: java-version: '11' - name: Check java version run: java -version + - name: Test with pytest run: | pytest + - name: Upload pytest test results + uses: actions/upload-artifact@v2 + with: + name: pytest-results-${{ matrix.python-version }} + path: test + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} diff --git a/README.md b/README.md index 43370839..67ce8430 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ Simply clone the repository and run the following commands: 1. `pip install -r requirements.txt` 2. `pip install -r requirements-test.txt` for tests +3. `pip install -r requirements-evaluation.txt` for evaluation ## Usage diff --git a/requirements-evaluation.txt b/requirements-evaluation.txt new file mode 100644 index 00000000..11910373 --- /dev/null +++ b/requirements-evaluation.txt @@ -0,0 +1,2 @@ +openpyxl==3.0.7 +pandas==1.2.3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d5a07c54..c61d633a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ radon==4.5.0 # extra libraries and frameworks django==3.2 requests==2.25.1 +argparse==1.4.0 diff --git a/src/python/common/__init__.py b/src/python/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/python/common/tool_arguments.py b/src/python/common/tool_arguments.py new file mode 100644 index 00000000..9fe4181b --- /dev/null +++ b/src/python/common/tool_arguments.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass +from enum import Enum, unique +from typing import List, Optional + +from src.python.review.application_config import LanguageVersion +from src.python.review.inspectors.inspector_type import InspectorType + + +@unique +class VerbosityLevel(Enum): + """ + Same meaning as the logging level. Should be used in command-line args. + """ + DEBUG = '3' + INFO = '2' + ERROR = '1' + DISABLE = '0' + + @classmethod + def values(cls) -> List[str]: + return [member.value for member in VerbosityLevel.__members__.values()] + + +@dataclass(frozen=True) +class ArgumentsInfo: + short_name: Optional[str] + long_name: str + description: str + + +@unique +class RunToolArgument(Enum): + VERBOSITY = ArgumentsInfo('-v', '--verbosity', + 'Choose logging level: ' + f'{VerbosityLevel.ERROR.value} - ERROR; ' + f'{VerbosityLevel.INFO.value} - INFO; ' + f'{VerbosityLevel.DEBUG.value} - DEBUG; ' + f'{VerbosityLevel.DISABLE.value} - disable logging; ' + 'default is 0') + + inspectors = [inspector.lower() for inspector in InspectorType.available_values()] + disabled_inspectors_example = f'-d {inspectors[0].lower()},{inspectors[1].lower()}' + + DISABLE = ArgumentsInfo('-d', '--disable', + 'Disable inspectors. ' + f'Available values: {", ".join(inspectors)}. ' + f'Example: {disabled_inspectors_example}') + + DUPLICATES = ArgumentsInfo(None, '--allow-duplicates', + 'Allow duplicate issues found by different linters. ' + 'By default, duplicates are skipped.') + + LANG_VERSION = ArgumentsInfo(None, '--language-version', + 'Specify the language version for JAVA inspectors.' + 'Available values are: ' + f'{LanguageVersion.PYTHON_3.value}, {LanguageVersion.JAVA_8.value}, ' + f'{LanguageVersion.JAVA_11.value}, {LanguageVersion.KOTLIN.value}.') + + CPU = ArgumentsInfo(None, '--n-cpu', + 'Specify number of cpu that can be used to run inspectors') + + PATH = ArgumentsInfo(None, 'path', 'Path to file or directory to inspect.') + + FORMAT = ArgumentsInfo('-f', '--format', + 'The output format. Default is JSON.') + + START_LINE = ArgumentsInfo('-s', '--start-line', + 'The first line to be analyzed. It starts from 1.') + + END_LINE = ArgumentsInfo('-e', '--end-line', 'The end line to be analyzed or None.') + + NEW_FORMAT = ArgumentsInfo(None, '--new-format', + 'The argument determines whether the tool ' + 'should use the new format') diff --git a/src/python/evaluation/README.md b/src/python/evaluation/README.md new file mode 100644 index 00000000..67e1d45e --- /dev/null +++ b/src/python/evaluation/README.md @@ -0,0 +1,31 @@ +# Hyperstyle evaluation + +This tool allows running the `Hyperstyle` tool on an xlsx table to get code quality for all code fragments. Please, note that your input file should consist of at least 2 obligatory columns to run xlsx-tool on its code fragments: + +- `code` +- `lang` + +Possible values for column `lang` are: `python3`, `kotlin`, `java8`, `java11`. + +Output file is a new `xlsx` file with 3 columns: +- `code` +- `lang` +- `grade` +Grade assessment is conducted by [`run_tool.py`](https://github.com/hyperskill/hyperstyle/blob/main/README.md) with default arguments. Avaliable values for column `grade` are: BAD, MODERATE, GOOD, EXCELLENT. It is also possible add fourth column: `traceback` to get full inspectors feedback on each code fragment. More details on enabling traceback column in **Optional Arguments** table. + +## Usage + +Run the [xlsx_run_tool.py](xlsx_run_tool.py) with the arguments from command line. + +Required arguments: + +`xlsx_file_path` — path to xlsx-file with code samples to inspect. + +Optional arguments: +Argument | Description +--- | --- +|**‑f**, **‑‑format**| The output format. Available values: `json`, `text`. The default value is `json` . Use this argument when `traceback` is enabled, otherwise it will not be used.| +|**‑tp**, **‑‑tool_path**| Path to run-tool. Default is `src/python/review/run_tool.py` .| +|**‑tr**, **‑‑traceback**| To include a column with errors traceback into an output file. Default is `False`.| +|**‑ofp**, **‑‑output_folder_path**| An explicit folder path to store file with results. Default is a parent directory of a folder with xlsx-file sent for inspection. | +|**‑ofn**, **‑‑output_file_name**| A name of an output file where evaluation results will be stored. Default is `results.xlsx`.| diff --git a/src/python/evaluation/__init__.py b/src/python/evaluation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/python/evaluation/common/__init__.py b/src/python/evaluation/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/python/evaluation/common/util.py b/src/python/evaluation/common/util.py new file mode 100644 index 00000000..c306d3b7 --- /dev/null +++ b/src/python/evaluation/common/util.py @@ -0,0 +1,34 @@ +from enum import Enum, unique + +from src.python.review.application_config import LanguageVersion +from src.python.review.common.file_system import Extension + + +@unique +class ColumnName(Enum): + CODE = "code" + LANG = "lang" + LANGUAGE = "language" + GRADE = "grade" + + +@unique +class EvaluationArgument(Enum): + TRACEBACK = "traceback" + RESULT_FILE_NAME = "results" + RESULT_FILE_NAME_EXT = f"{RESULT_FILE_NAME}{Extension.XLSX.value}" + + +script_structure_rule = ("Please, make sure your XLSX-file matches following script standards: \n" + "1. Your XLSX-file should have 2 obligatory columns named:" + f"'{ColumnName.CODE.value}' & '{ColumnName.LANG.value}'. \n" + f"'{ColumnName.CODE.value}' column -- relates to the code-sample. \n" + f"'{ColumnName.LANG.value}' column -- relates to the language of a " + "particular code-sample. \n" + "2. Your code samples should belong to the one of the supported languages. \n" + "Supported languages are: Java, Kotlin, Python. \n" + f"3. Check that '{ColumnName.LANG.value}' column cells are filled with " + "acceptable language-names: \n" + f"Acceptable language-names are: {LanguageVersion.PYTHON_3.value}, " + f"{LanguageVersion.JAVA_8.value} ," + f"{LanguageVersion.JAVA_11.value} and {LanguageVersion.KOTLIN.value}.") diff --git a/src/python/evaluation/common/xlsx_util.py b/src/python/evaluation/common/xlsx_util.py new file mode 100644 index 00000000..032a5ce6 --- /dev/null +++ b/src/python/evaluation/common/xlsx_util.py @@ -0,0 +1,42 @@ +import logging.config +from pathlib import Path +from typing import Union + +import pandas as pd +from openpyxl import load_workbook, Workbook +from src.python.evaluation.evaluation_config import EvaluationConfig + +logger = logging.getLogger(__name__) + + +def remove_sheet(workbook_path: Union[str, Path], sheet_name: str, to_raise_error: bool = False) -> None: + try: + workbook = load_workbook(workbook_path) + workbook.remove(workbook[sheet_name]) + workbook.save(workbook_path) + + except KeyError as e: + message = f'Sheet with specified name: {sheet_name} does not exist.' + if to_raise_error: + logger.exception(message) + raise e + else: + logger.info(message) + + +def create_and_get_workbook_path(config: EvaluationConfig) -> Path: + workbook = Workbook() + workbook_path = config.get_output_file_path() + workbook.save(workbook_path) + return workbook_path + + +def write_dataframe_to_xlsx_sheet(xlsx_file_path: Union[str, Path], df: pd.DataFrame, sheet_name: str, + mode: str = 'a', to_write_row_names: bool = False) -> None: + """ + mode: str Available values are {'w', 'a'}. File mode to use (write or append). + to_write_row_names: bool Write row names. + """ + + with pd.ExcelWriter(xlsx_file_path, mode=mode) as writer: + df.to_excel(writer, sheet_name=sheet_name, index=to_write_row_names) diff --git a/src/python/evaluation/evaluation_config.py b/src/python/evaluation/evaluation_config.py new file mode 100644 index 00000000..5cee71dc --- /dev/null +++ b/src/python/evaluation/evaluation_config.py @@ -0,0 +1,43 @@ +import logging.config +from argparse import Namespace +from pathlib import Path +from typing import List, Union + +from src.python.common.tool_arguments import RunToolArgument +from src.python.evaluation.common.util import EvaluationArgument +from src.python.review.application_config import LanguageVersion +from src.python.review.common.file_system import create_directory + +logger = logging.getLogger(__name__) + + +class EvaluationConfig: + def __init__(self, args: Namespace): + self.tool_path: Union[str, Path] = args.tool_path + self.output_format: str = args.format + self.xlsx_file_path: Union[str, Path] = args.xlsx_file_path + self.traceback: bool = args.traceback + self.output_folder_path: Union[str, Path] = args.output_folder_path + self.output_file_name: str = args.output_file_name + + def build_command(self, inspected_file_path: Union[str, Path], lang: str) -> List[str]: + command = [LanguageVersion.PYTHON_3.value, + self.tool_path, + inspected_file_path, + RunToolArgument.FORMAT.value.short_name, self.output_format] + + if lang == LanguageVersion.JAVA_8.value or lang == LanguageVersion.JAVA_11.value: + command.extend([RunToolArgument.LANG_VERSION.value.long_name, lang]) + return command + + def get_output_file_path(self) -> Path: + if self.output_folder_path is None: + try: + self.output_folder_path = ( + Path(self.xlsx_file_path).parent.parent / EvaluationArgument.RESULT_FILE_NAME.value + ) + create_directory(self.output_folder_path) + except FileNotFoundError as e: + logger.error('XLSX-file with the specified name does not exists.') + raise e + return Path(self.output_folder_path) / self.output_file_name diff --git a/src/python/evaluation/xlsx_run_tool.py b/src/python/evaluation/xlsx_run_tool.py new file mode 100644 index 00000000..7235b041 --- /dev/null +++ b/src/python/evaluation/xlsx_run_tool.py @@ -0,0 +1,169 @@ +import argparse +import logging.config +import os +import re +import sys +import traceback +from pathlib import Path +from typing import Type + +sys.path.append('') +sys.path.append('../../..') + +import pandas as pd +from src.python.common.tool_arguments import RunToolArgument +from src.python.evaluation.common.util import ColumnName, EvaluationArgument, script_structure_rule +from src.python.evaluation.common.xlsx_util import ( + create_and_get_workbook_path, + remove_sheet, + write_dataframe_to_xlsx_sheet, +) +from src.python.evaluation.evaluation_config import EvaluationConfig +from src.python.review.application_config import LanguageVersion +from src.python.review.common.file_system import create_file, new_temp_dir +from src.python.review.common.subprocess_runner import run_in_subprocess +from src.python.review.reviewers.perform_review import OutputFormat + +logger = logging.getLogger(__name__) + + +def configure_arguments(parser: argparse.ArgumentParser, run_tool_arguments: Type[RunToolArgument]) -> None: + parser.add_argument('xlsx_file_path', + type=lambda value: Path(value).absolute(), + help='Local XLSX-file path. ' + 'Your XLSX-file must include column-names: ' + f'"{ColumnName.CODE.value}" and ' + f'"{ColumnName.LANG.value}". Acceptable values for ' + f'"{ColumnName.LANG.value}" column are: ' + f'{LanguageVersion.PYTHON_3.value}, {LanguageVersion.JAVA_8.value}, ' + f'{LanguageVersion.JAVA_11.value}, {LanguageVersion.KOTLIN.value}.') + + parser.add_argument('-tp', '--tool-path', + default=Path('src/python/review/run_tool.py').absolute(), + type=lambda value: Path(value).absolute(), + help='Path to script to run on files.') + + parser.add_argument('-tr', '--traceback', + help='If True, column with the full inspector feedback will be added ' + 'to the output file with results.', + action='store_true') + + parser.add_argument('-ofp', '--output-folder-path', + help='An absolute path to the folder where file with evaluation results' + 'will be stored.' + 'Default is the path to a directory, where is the folder with xlsx_file.', + # if None default path will be specified based on xlsx_file_path. + default=None, + type=str) + + parser.add_argument('-ofn', '--output-file-name', + help='Filename for that will be created to store inspection results.' + f'Default is "{EvaluationArgument.RESULT_FILE_NAME_EXT.value}"', + default=f'{EvaluationArgument.RESULT_FILE_NAME_EXT.value}', + type=str) + + parser.add_argument(run_tool_arguments.FORMAT.value.short_name, + run_tool_arguments.FORMAT.value.long_name, + default=OutputFormat.JSON.value, + choices=OutputFormat.values(), + type=str, + help=f'{run_tool_arguments.FORMAT.value.description}' + f'Use this argument when {EvaluationArgument.TRACEBACK.value} argument' + 'is enabled argument will not be used otherwise.') + + +def create_dataframe(config: EvaluationConfig) -> pd.DataFrame: + report = pd.DataFrame( + { + ColumnName.LANGUAGE.value: [], + ColumnName.CODE.value: [], + ColumnName.GRADE.value: [], + }, + ) + + if config.traceback: + report[EvaluationArgument.TRACEBACK.value] = [] + + try: + lang_code_dataframe = pd.read_excel(config.xlsx_file_path) + + except FileNotFoundError as e: + logger.error('XLSX-file with the specified name does not exists.') + raise e + + try: + for lang, code in zip(lang_code_dataframe[ColumnName.LANG.value], + lang_code_dataframe[ColumnName.CODE.value]): + + with new_temp_dir() as create_temp_dir: + temp_dir_path = create_temp_dir + lang_extension = LanguageVersion.language_by_extension(lang) + temp_file_path = os.path.join(temp_dir_path, ('file' + lang_extension)) + temp_file_path = next(create_file(temp_file_path, code)) + + try: + assert os.path.exists(temp_file_path) + except AssertionError as e: + logger.exception('Path does not exist.') + raise e + + command = config.build_command(temp_file_path, lang) + results = run_in_subprocess(command) + os.remove(temp_file_path) + temp_dir_path.rmdir() + # this regular expression matches final tool grade: EXCELLENT, GOOD, MODERATE or BAD + grades = re.match(r'^.*{"code":\s"([A-Z]+)"', results).group(1) + output_row_values = [lang, code, grades] + column_indices = [ColumnName.LANGUAGE.value, + ColumnName.CODE.value, + ColumnName.GRADE.value] + + if config.traceback: + output_row_values.append(results) + column_indices.append(EvaluationArgument.TRACEBACK.value) + + new_file_report_row = pd.Series(data=output_row_values, index=column_indices) + report = report.append(new_file_report_row, ignore_index=True) + + return report + + except KeyError as e: + logger.error(script_structure_rule) + raise e + + except Exception as e: + traceback.print_exc() + logger.exception('An unexpected error.') + raise e + + +def main() -> int: + parser = argparse.ArgumentParser() + configure_arguments(parser, RunToolArgument) + + try: + args = parser.parse_args() + config = EvaluationConfig(args) + workbook_path = create_and_get_workbook_path(config) + results = create_dataframe(config) + write_dataframe_to_xlsx_sheet(workbook_path, results, 'inspection_results') + # remove empty sheet that was initially created with the workbook + remove_sheet(workbook_path, 'Sheet') + return 0 + + except FileNotFoundError: + logger.error('XLSX-file with the specified name does not exists.') + return 2 + + except KeyError: + logger.error(script_structure_rule) + return 2 + + except Exception: + traceback.print_exc() + logger.exception('An unexpected error.') + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/python/review/application_config.py b/src/python/review/application_config.py index 41a1c296..8d9a56f5 100644 --- a/src/python/review/application_config.py +++ b/src/python/review/application_config.py @@ -2,6 +2,7 @@ from enum import Enum, unique from typing import List, Optional, Set +from src.python.review.common.file_system import Extension from src.python.review.inspectors.inspector_type import InspectorType @@ -22,7 +23,22 @@ class LanguageVersion(Enum): JAVA_8 = 'java8' JAVA_9 = 'java9' JAVA_11 = 'java11' + PYTHON_3 = 'python3' + KOTLIN = 'kotlin' @classmethod def values(cls) -> List[str]: return [member.value for member in cls.__members__.values()] + + @classmethod + def language_to_extension_dict(cls) -> dict: + return {cls.PYTHON_3.value: Extension.PY.value, + cls.JAVA_7.value: Extension.JAVA.value, + cls.JAVA_8.value: Extension.JAVA.value, + cls.JAVA_9.value: Extension.JAVA.value, + cls.JAVA_11.value: Extension.JAVA.value, + cls.KOTLIN.value: Extension.KT.value} + + @classmethod + def language_by_extension(cls, lang: str) -> str: + return cls.language_to_extension_dict()[lang] diff --git a/src/python/review/common/file_system.py b/src/python/review/common/file_system.py index 764a2d5e..3ab06061 100644 --- a/src/python/review/common/file_system.py +++ b/src/python/review/common/file_system.py @@ -29,6 +29,7 @@ class Extension(Enum): KT = '.kt' JS = '.js' KTS = '.kts' + XLSX = '.xlsx' ItemCondition = Callable[[str], bool] @@ -66,8 +67,9 @@ def create_file(file_path: Union[str, Path], content: str): file_path = Path(file_path) create_directory(os.path.dirname(file_path)) - with open(file_path, 'w') as f: - f.write(content) + with open(file_path, 'w+') as f: + f.writelines(content) + yield Path(file_path) def create_directory(directory: str) -> None: diff --git a/src/python/review/run_tool.py b/src/python/review/run_tool.py index 0d96f6e8..bdfbb41f 100644 --- a/src/python/review/run_tool.py +++ b/src/python/review/run_tool.py @@ -1,15 +1,17 @@ import argparse +import enum import logging.config import os import sys import traceback -from enum import Enum, unique from pathlib import Path -from typing import List, Set +from typing import Set + sys.path.append('') sys.path.append('../../..') +from src.python.common.tool_arguments import RunToolArgument, VerbosityLevel from src.python.review.application_config import ApplicationConfig, LanguageVersion from src.python.review.inspectors.inspector_type import InspectorType from src.python.review.logging_config import logging_config @@ -23,21 +25,6 @@ logger = logging.getLogger(__name__) -@unique -class VerbosityLevel(Enum): - """ - Same meaning as the logging level. Should be used in command-line args. - """ - DEBUG = '3' - INFO = '2' - ERROR = '1' - DISABLE = '0' - - @classmethod - def values(cls) -> List[str]: - return [member.value for _, member in VerbosityLevel.__members__.items()] - - def parse_disabled_inspectors(value: str) -> Set[InspectorType]: passed_names = value.upper().split(',') allowed_names = {inspector.value for inspector in InspectorType} @@ -55,70 +42,66 @@ def positive_int(value: str) -> int: return value_int -def configure_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('-v', '--verbosity', - help='Choose logging level: ' - f'{VerbosityLevel.ERROR.value} - ERROR; ' - f'{VerbosityLevel.INFO.value} - INFO; ' - f'{VerbosityLevel.DEBUG.value} - DEBUG; ' - f'{VerbosityLevel.DISABLE.value} - disable logging; ' - 'default is 0', +def configure_arguments(parser: argparse.ArgumentParser, tool_arguments: enum.EnumMeta) -> None: + parser.add_argument(tool_arguments.VERBOSITY.value.short_name, + tool_arguments.VERBOSITY.value.long_name, + help=tool_arguments.VERBOSITY.value.description, default=VerbosityLevel.DISABLE.value, choices=VerbosityLevel.values(), type=str) # Usage example: -d Flake8,Intelli - inspectors = [inspector.lower() for inspector in InspectorType.available_values()] - example = f'-d {inspectors[0].lower()},{inspectors[1].lower()}' - - parser.add_argument('-d', '--disable', - help='Disable inspectors. ' - f'Allowed values: {", ".join(inspectors)}. ' - f'Example: {example}', + parser.add_argument(tool_arguments.DISABLE.value.short_name, + tool_arguments.DISABLE.value.long_name, + help=tool_arguments.DISABLE.value.description, type=parse_disabled_inspectors, default=set()) - parser.add_argument('--allow-duplicates', action='store_true', - help='Allow duplicate issues found by different linters. ' - 'By default, duplicates are skipped.') + parser.add_argument(tool_arguments.DUPLICATES.value.long_name, + action='store_true', + help=tool_arguments.DUPLICATES.value.description) # TODO: deprecated argument: language_version. Delete after several releases. - parser.add_argument('--language_version', '--language-version', - help='Specify the language version for JAVA inspectors.', + parser.add_argument('--language_version', + tool_arguments.LANG_VERSION.value.long_name, + help=tool_arguments.LANG_VERSION.value.description, default=None, choices=LanguageVersion.values(), type=str) # TODO: deprecated argument: --n_cpu. Delete after several releases. - parser.add_argument('--n_cpu', '--n-cpu', - help='Specify number of cpu that can be used to run inspectors', + parser.add_argument('--n_cpu', + tool_arguments.CPU.value.long_name, + help=tool_arguments.CPU.value.description, default=1, type=positive_int) - parser.add_argument('path', + parser.add_argument(tool_arguments.PATH.value.long_name, type=lambda value: Path(value).absolute(), - help='Path to file or directory to inspect.') + help=tool_arguments.PATH.value.description) - parser.add_argument('-f', '--format', + parser.add_argument(tool_arguments.FORMAT.value.short_name, + tool_arguments.FORMAT.value.long_name, default=OutputFormat.JSON.value, choices=OutputFormat.values(), type=str, - help='The output format. Default is JSON.') + help=tool_arguments.FORMAT.value.description) - parser.add_argument('-s', '--start-line', + parser.add_argument(tool_arguments.START_LINE.value.short_name, + tool_arguments.START_LINE.value.long_name, default=1, type=positive_int, - help='The first line to be analyzed. It starts from 1.') + help=tool_arguments.START_LINE.value.description) - parser.add_argument('-e', '--end-line', + parser.add_argument(tool_arguments.END_LINE.value.short_name, + tool_arguments.END_LINE.value.long_name, default=None, type=positive_int, - help='The end line to be analyzed or None.') + help=tool_arguments.END_LINE.value.description) - parser.add_argument('--new-format', + parser.add_argument(tool_arguments.NEW_FORMAT.value.long_name, action='store_true', - help='The argument determines whether the tool ' - 'should use the new format') + help=tool_arguments.NEW_FORMAT.value.description) def configure_logging(verbosity: VerbosityLevel) -> None: @@ -136,7 +119,7 @@ def configure_logging(verbosity: VerbosityLevel) -> None: def main() -> int: parser = argparse.ArgumentParser() - configure_arguments(parser) + configure_arguments(parser, RunToolArgument) try: args = parser.parse_args() diff --git a/test/python/evaluation/__init__.py b/test/python/evaluation/__init__.py new file mode 100644 index 00000000..31b1b86f --- /dev/null +++ b/test/python/evaluation/__init__.py @@ -0,0 +1,11 @@ +from test.python import TEST_DATA_FOLDER + +from src.python import MAIN_FOLDER + +CURRENT_TEST_DATA_FOLDER = TEST_DATA_FOLDER / 'evaluation' + +XLSX_DATA_FOLDER = CURRENT_TEST_DATA_FOLDER / 'xlsx_files' + +TARGET_XLSX_DATA_FOLDER = CURRENT_TEST_DATA_FOLDER / 'xlsx_target_files' + +RESULTS_DIR_PATH = MAIN_FOLDER.parent / 'evaluation/results' diff --git a/test/python/evaluation/test_data_path.py b/test/python/evaluation/test_data_path.py new file mode 100644 index 00000000..0d8e3502 --- /dev/null +++ b/test/python/evaluation/test_data_path.py @@ -0,0 +1,14 @@ +from test.python.evaluation import XLSX_DATA_FOLDER +from test.python.evaluation.testing_config import get_testing_arguments + +import pytest +from src.python.evaluation.evaluation_config import EvaluationConfig +from src.python.evaluation.xlsx_run_tool import create_dataframe + + +def test_incorrect_data_path(): + with pytest.raises(FileNotFoundError): + testing_arguments_dict = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) + testing_arguments_dict.xlsx_file_path = XLSX_DATA_FOLDER / 'do_not_exist.xlsx' + config = EvaluationConfig(testing_arguments_dict) + assert create_dataframe(config) diff --git a/test/python/evaluation/test_output_results.py b/test/python/evaluation/test_output_results.py new file mode 100644 index 00000000..519652e5 --- /dev/null +++ b/test/python/evaluation/test_output_results.py @@ -0,0 +1,32 @@ +from test.python.evaluation import TARGET_XLSX_DATA_FOLDER, XLSX_DATA_FOLDER +from test.python.evaluation.testing_config import get_testing_arguments + +import pandas as pd +import pytest +from src.python.evaluation.evaluation_config import EvaluationConfig +from src.python.evaluation.xlsx_run_tool import create_dataframe + +FILE_NAMES = [ + ('test_sorted_order.xlsx', 'target_sorted_order.xlsx', False), + ('test_sorted_order.xlsx', 'target_sorted_order.xlsx', True), + ('test_unsorted_order.xlsx', 'target_unsorted_order.xlsx', False), + ('test_unsorted_order.xlsx', 'target_unsorted_order.xlsx', True), +] + + +@pytest.mark.parametrize(('test_file', 'target_file', 'output_type'), FILE_NAMES) +def test_correct_output(test_file: str, target_file: str, output_type: bool): + + testing_arguments_dict = get_testing_arguments(to_add_tool_path=True) + testing_arguments_dict.xlsx_file_path = XLSX_DATA_FOLDER / test_file + testing_arguments_dict.traceback = output_type + + config = EvaluationConfig(testing_arguments_dict) + test_dataframe = create_dataframe(config) + + sheet_name = 'grades' + if output_type: + sheet_name = 'traceback' + target_dataframe = pd.read_excel(TARGET_XLSX_DATA_FOLDER / target_file, sheet_name=sheet_name) + + assert test_dataframe.reset_index(drop=True).equals(target_dataframe.reset_index(drop=True)) diff --git a/test/python/evaluation/test_tool_path.py b/test/python/evaluation/test_tool_path.py new file mode 100644 index 00000000..0581caad --- /dev/null +++ b/test/python/evaluation/test_tool_path.py @@ -0,0 +1,26 @@ +from test.python.evaluation import XLSX_DATA_FOLDER +from test.python.evaluation.testing_config import get_testing_arguments + +import pytest +from src.python import MAIN_FOLDER +from src.python.evaluation.evaluation_config import EvaluationConfig +from src.python.evaluation.xlsx_run_tool import create_dataframe + + +def test_correct_tool_path(): + try: + testing_arguments_dict = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) + testing_arguments_dict.xlsx_file_path = XLSX_DATA_FOLDER / 'test_unsorted_order.xlsx' + config = EvaluationConfig(testing_arguments_dict) + create_dataframe(config) + except Exception: + pytest.fail("Unexpected error") + + +def test_incorrect_tool_path(): + with pytest.raises(Exception): + testing_arguments_dict = get_testing_arguments(to_add_traceback=True) + testing_arguments_dict.xlsx_file_path = XLSX_DATA_FOLDER / 'test_unsorted_order.xlsx' + testing_arguments_dict.tool_path = MAIN_FOLDER.parent / 'review/incorrect_path.py' + config = EvaluationConfig(testing_arguments_dict) + assert create_dataframe(config) diff --git a/test/python/evaluation/test_xlsx_file_structure.py b/test/python/evaluation/test_xlsx_file_structure.py new file mode 100644 index 00000000..9965992e --- /dev/null +++ b/test/python/evaluation/test_xlsx_file_structure.py @@ -0,0 +1,23 @@ +from test.python.evaluation import XLSX_DATA_FOLDER +from test.python.evaluation.testing_config import get_testing_arguments + +import pytest +from src.python.evaluation.evaluation_config import EvaluationConfig +from src.python.evaluation.xlsx_run_tool import create_dataframe + + +FILE_NAMES = [ + 'test_wrong_column_name.xlsx', + 'test_java_no_version.xlsx', + 'test_empty_lang_cell.xlsx', + 'test_empty_table.xlsx', +] + + +@pytest.mark.parametrize('file_name', FILE_NAMES) +def test_wrong_column(file_name: str): + with pytest.raises(KeyError): + testing_arguments_dict = get_testing_arguments(to_add_traceback=True, to_add_tool_path=True) + testing_arguments_dict.xlsx_file_path = XLSX_DATA_FOLDER / file_name + config = EvaluationConfig(testing_arguments_dict) + assert create_dataframe(config) diff --git a/test/python/evaluation/testing_config.py b/test/python/evaluation/testing_config.py new file mode 100644 index 00000000..70341635 --- /dev/null +++ b/test/python/evaluation/testing_config.py @@ -0,0 +1,18 @@ +from argparse import Namespace + +from src.python import MAIN_FOLDER +from src.python.evaluation.common.util import EvaluationArgument +from src.python.review.reviewers.perform_review import OutputFormat + + +def get_testing_arguments(to_add_traceback=None, to_add_tool_path=None) -> Namespace: + testing_arguments = Namespace(format=OutputFormat.JSON.value, + output_file_name=EvaluationArgument.RESULT_FILE_NAME_EXT.value, + output_folder_path=None) + if to_add_traceback: + testing_arguments.traceback = True + + if to_add_tool_path: + testing_arguments.tool_path = MAIN_FOLDER.parent / 'review/run_tool.py' + + return testing_arguments diff --git a/test/resources/evaluation/xlsx_files/__init__.py b/test/resources/evaluation/xlsx_files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/resources/evaluation/xlsx_files/test_empty_lang_cell.xlsx b/test/resources/evaluation/xlsx_files/test_empty_lang_cell.xlsx new file mode 100644 index 00000000..91cdada0 Binary files /dev/null and b/test/resources/evaluation/xlsx_files/test_empty_lang_cell.xlsx differ diff --git a/test/resources/evaluation/xlsx_files/test_empty_table.xlsx b/test/resources/evaluation/xlsx_files/test_empty_table.xlsx new file mode 100644 index 00000000..486b83a1 Binary files /dev/null and b/test/resources/evaluation/xlsx_files/test_empty_table.xlsx differ diff --git a/test/resources/evaluation/xlsx_files/test_java_no_version.xlsx b/test/resources/evaluation/xlsx_files/test_java_no_version.xlsx new file mode 100644 index 00000000..d473b2cf Binary files /dev/null and b/test/resources/evaluation/xlsx_files/test_java_no_version.xlsx differ diff --git a/test/resources/evaluation/xlsx_files/test_sorted_order.xlsx b/test/resources/evaluation/xlsx_files/test_sorted_order.xlsx new file mode 100644 index 00000000..bfdca3b2 Binary files /dev/null and b/test/resources/evaluation/xlsx_files/test_sorted_order.xlsx differ diff --git a/test/resources/evaluation/xlsx_files/test_unsorted_order.xlsx b/test/resources/evaluation/xlsx_files/test_unsorted_order.xlsx new file mode 100644 index 00000000..75bc9783 Binary files /dev/null and b/test/resources/evaluation/xlsx_files/test_unsorted_order.xlsx differ diff --git a/test/resources/evaluation/xlsx_files/test_wrong_column_name.xlsx b/test/resources/evaluation/xlsx_files/test_wrong_column_name.xlsx new file mode 100644 index 00000000..d1fc1415 Binary files /dev/null and b/test/resources/evaluation/xlsx_files/test_wrong_column_name.xlsx differ diff --git a/test/resources/evaluation/xlsx_target_files/__init__.py b/test/resources/evaluation/xlsx_target_files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/resources/evaluation/xlsx_target_files/target_sorted_order.xlsx b/test/resources/evaluation/xlsx_target_files/target_sorted_order.xlsx new file mode 100644 index 00000000..8cbc432f Binary files /dev/null and b/test/resources/evaluation/xlsx_target_files/target_sorted_order.xlsx differ diff --git a/test/resources/evaluation/xlsx_target_files/target_unsorted_order.xlsx b/test/resources/evaluation/xlsx_target_files/target_unsorted_order.xlsx new file mode 100644 index 00000000..25ed6146 Binary files /dev/null and b/test/resources/evaluation/xlsx_target_files/target_unsorted_order.xlsx differ diff --git a/whitelist.txt b/whitelist.txt index a36bef18..c60ec292 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -73,6 +73,16 @@ KTS nl splitext dirname +hyperstyle +XLSX +Eval +eval +openpyxl +dataframe +writelines +rmdir +df +unique # Springlint issues cbo dit @@ -80,3 +90,4 @@ lcom noc nom wmc +util \ No newline at end of file