diff --git a/src/python/evaluation/README.md b/src/python/evaluation/README.md index f8fafbe0..b98e2deb 100644 --- a/src/python/evaluation/README.md +++ b/src/python/evaluation/README.md @@ -33,3 +33,4 @@ Argument | Description |**‑‑traceback**| To include a column with errors traceback into an output file. Default is `False`.| |**‑ofp**, **‑‑output‑folder‑path**| An explicit folder path to store file with results. Default is a parent directory of a folder with xlsx-file or csv-file sent for inspection. | |**‑ofn**, **‑‑output‑file‑name**| A name of an output file where evaluation results will be stored. Default is `results.xlsx` or `results.csv`.| +|**‑‑to‑drop‑nan**| If True, empty code fragments will be deleted from df. Default is `False`.| diff --git a/src/python/evaluation/evaluation_config.py b/src/python/evaluation/evaluation_config.py index b894d529..06672696 100644 --- a/src/python/evaluation/evaluation_config.py +++ b/src/python/evaluation/evaluation_config.py @@ -26,6 +26,7 @@ def __init__(self, args: Namespace): self.output_folder_path: Union[str, Path] = args.output_folder_path self.extension: Extension = get_restricted_extension(self.solutions_file_path, [Extension.XLSX, Extension.CSV]) self.__init_output_file_name(args.output_file_name) + self.to_drop_nan: bool = args.to_drop_nan def __init_output_file_name(self, output_file_name: Optional[str]): if output_file_name is None: diff --git a/src/python/evaluation/evaluation_run_tool.py b/src/python/evaluation/evaluation_run_tool.py index bff335ed..6304bf4f 100644 --- a/src/python/evaluation/evaluation_run_tool.py +++ b/src/python/evaluation/evaluation_run_tool.py @@ -70,6 +70,10 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None: 'must contain the history of previous errors.', action='store_true') + parser.add_argument('--to-drop-nan', + help='If True, empty code fragments will be deleted from df', + action='store_true') + def get_language_version(lang_key: str) -> LanguageVersion: try: @@ -80,7 +84,8 @@ def get_language_version(lang_key: str) -> LanguageVersion: raise KeyError(e) -def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str], config: EvaluationConfig) -> str: +def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str], + config: EvaluationConfig) -> Optional[str]: print(f'current id: {fragment_id}') # Tool does not work correctly with tmp files from module on macOS # thus we create a real file in the file system @@ -107,6 +112,8 @@ def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataF if config.traceback: report[ColumnName.TRACEBACK.value] = [] try: + if config.to_drop_nan: + lang_code_dataframe = lang_code_dataframe.dropna() lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply( lambda row: __inspect_row(row[ColumnName.LANG.value], row[ColumnName.CODE.value], diff --git a/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py b/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py index 046c2606..0de99fc6 100644 --- a/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py +++ b/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py @@ -97,8 +97,7 @@ def print_stat(language: Language, stat: IssuesStat) -> None: print(f'Collected statistics for {language.value.lower()} language:') for issue_type, freq in stat.items(): print(f'{issue_type}: {freq} times;') - print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories. ' - f'Most likely it is {IssueType.INFO} category') + print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories.') def __parse_language(language: str) -> Language: diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/README.md b/src/python/evaluation/paper_evaluation/user_dynamics/README.md index 9e46df11..7d004bb1 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/README.md +++ b/src/python/evaluation/paper_evaluation/user_dynamics/README.md @@ -10,20 +10,22 @@ Required arguments: `solutions_file_path` — path to csv-file with code samples. -Optional arguments: -Argument | Description ---- | --- -|**‑fb**, **‑‑freq-boundary**| The boundary of solutions count for one student to analyze. The default value is 100.| -|**‑n**, **‑‑n**| Top n popular issues in solutions. The default value is 100. | +In the result a file with students issues dynamics will be created. +We have three categories of dynamics: +- all (count of all code quality issues expect INFO issues) +- formatting (count of formatting code quality issues from CODE_STYLE category) +- other (all issues minus formatting issues) -In the result a file with students issues dynamics will be created. -Also, the top of issues for all students will be printed into the terminal. This statistics has key of issue and frequency for all students. +Each type of dynamics will be saved into a separated folder with csv files for each student. +Each csv file has only two columns: fragment id and issues count. -An example of issues dynamics: +An example of the csv file: ```text -user,traceback -0,"0,0,0,0,0,0,0,0,0,0,0,1,0,0,3,0,0,0,0,2,0,4,0,6,3,0,3,0,0,0,1,1,0,0,0,1,0,0,0,2,0,0,0,0,0,0,4,0,0,0,1,6,0,1,0,1,3,0,0,1,1,0,0,0,0,0,3,6,1,0,0,0,0,0,0,0,4,1,0,0,1,0,8,0,2,8,0,0,0,0,1,1,1,1,3,7,23,0,9" -1,"0,0,0,3,0,0,2,1,0,0,0,0,4,1,0,0,1,1,0,0,0,0,0,6,0,1,1,0,8,1,2,1,1,0,0,1,0,4,10,1,1,1,3,0,1,0,0,0,1,0,0,0,0,0,0,2,0,3,0,0,2,2,3,2,0,0,0,1,0,1,1,0,0,1,0,4,6,2,0,0,1,0,0,0,0,2,0,0,0,2,1,2,1,0,1,7,1,0,1,1,0,1,0" -``` -Each number in the traceback column is the count of issues in one solution. -The numbers of issues sorted by timestamps. \ No newline at end of file +issue_count,time +2,0 +20,1 +16,2 +15,3 +5,4 +5,5 +``` \ No newline at end of file diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py index 7d9bbec5..6c4ab4ad 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py +++ b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py @@ -1,19 +1,18 @@ import argparse import sys -from collections import Counter from pathlib import Path -from typing import Dict, List +from typing import List +import numpy as np import pandas as pd from src.python.common.tool_arguments import RunToolArgument from src.python.evaluation.common.csv_util import write_dataframe_to_csv from src.python.evaluation.common.pandas_util import ( - drop_duplicates, filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger, + filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger, ) from src.python.evaluation.common.util import ColumnName from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.evaluation.paper_evaluation.comparison_with_other_tools.tutor_statistics import sort_freq_dict -from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import UserStatistics +from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension from src.python.review.inspectors.issue import IssueType @@ -23,58 +22,45 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None: type=lambda value: Path(value).absolute(), help=RunToolArgument.SOLUTIONS_FILE_PATH.value) - parser.add_argument('-fb', '--freq-boundary', - help='The boundary of solutions count for one student to analyze', - type=int, - default=100) - - parser.add_argument('-n', '--n', - help='Top n popular issues in solutions', - type=int, - default=10) - - -def __get_top_freq_issues(issues: List[List[PenaltyIssue]], n: int) -> Dict[str, int]: - all_issues = list(map(lambda i: i.origin_class, [item for sublist in issues for item in sublist])) - return dict(Counter(all_issues).most_common(n)) - - -# Get statistics only for users that have >= freq_boundary solutions in solutions_df -# Statistics for each student has: -# - - list of list of issues, but without INFO issues -# - - for each key of issue from has frequency. -# Contains only top_n issues -def __get_user_statistics(solutions_df: pd.DataFrame, freq_boundary: int = 100, - top_n: int = 10) -> List[UserStatistics]: - stat = [] - counts = solutions_df[ColumnName.USER.value].value_counts() - solutions_df = solutions_df[solutions_df[ColumnName.USER.value].isin(counts[counts > freq_boundary].index)] - for user in solutions_df[ColumnName.USER.value].unique(): - user_df = filter_df_by_single_value(solutions_df, - ColumnName.USER.value, user).sort_values(ColumnName.TIME.value) - user_df = drop_duplicates(user_df) - traceback = list(map(lambda t: get_issues_from_json(t), - list(user_df[ColumnName.TRACEBACK.value]))) - # Filter info category - traceback = list(filter(lambda issues_list: filter(lambda i: i.type != IssueType.INFO, issues_list), traceback)) - top_issues = __get_top_freq_issues(traceback, top_n) - stat.append(UserStatistics(traceback, top_issues)) - return stat + +ALL_ISSUES_COUNT = DynamicsColumn.ALL_ISSUES_COUNT.value +FORMATTING_ISSUES_COUNT = DynamicsColumn.FORMATTING_ISSUES_COUNT.value +OTHER_ISSUES_COUNT = DynamicsColumn.OTHER_ISSUES_COUNT.value -def __get_student_dynamics(stats: List[UserStatistics]) -> pd.DataFrame: - dynamics = map(lambda s: s.get_traceback_dynamics(), stats) - dynamics_dict = {i: ','.join(map(lambda d: str(d), dyn)) for (i, dyn) in enumerate(dynamics)} - return pd.DataFrame(dynamics_dict.items(), columns=[ColumnName.USER.value, ColumnName.TRACEBACK.value]) +def __get_all_issues(traceback: str) -> List[PenaltyIssue]: + return list(filter(lambda i: i.type != IssueType.INFO, get_issues_from_json(traceback))) -def __get_total_top(stats: List[UserStatistics]) -> Dict[str, int]: - total_top_n = {} - for d in map(lambda s: s.top_issues, stats): - for k, v in d.items(): - total_top_n.setdefault(k, 0) - total_top_n[k] += v - return sort_freq_dict(total_top_n) +def __get_formatting_issues(traceback: str) -> List[PenaltyIssue]: + return list(filter(lambda i: i.type == IssueType.CODE_STYLE, __get_all_issues(traceback))) + + +def __write_dynamics(output_path: Path, user_fragments: pd.DataFrame, index: int) -> None: + output_path.mkdir(parents=True, exist_ok=True) + user_fragments.columns = [DynamicsColumn.ISSUE_COUNT.value] + user_fragments[ColumnName.TIME.value] = np.arange(len(user_fragments)) + write_dataframe_to_csv(output_path / f'user_{index}{Extension.CSV.value}', user_fragments) + + +def __get_users_statistics(solutions_df: pd.DataFrame, output_path: Path) -> None: + users = solutions_df[ColumnName.USER.value].unique() + for index, user in enumerate(users): + user_df = filter_df_by_single_value(solutions_df, + ColumnName.USER.value, user).sort_values(ColumnName.TIME.value) + user_df[ALL_ISSUES_COUNT] = user_df.apply(lambda row: + len(__get_all_issues( + row[ColumnName.TRACEBACK.value])), + axis=1) + user_df[FORMATTING_ISSUES_COUNT] = user_df.apply(lambda row: + len(__get_formatting_issues( + row[ColumnName.TRACEBACK.value])), + axis=1) + user_df[OTHER_ISSUES_COUNT] = user_df[ALL_ISSUES_COUNT] - user_df[FORMATTING_ISSUES_COUNT] + + __write_dynamics(output_path / 'all', user_df[[ALL_ISSUES_COUNT]], index) + __write_dynamics(output_path / 'formatting', user_df[[FORMATTING_ISSUES_COUNT]], index) + __write_dynamics(output_path / 'other', user_df[[OTHER_ISSUES_COUNT]], index) def main() -> int: @@ -86,16 +72,10 @@ def main() -> int: solutions_file_path = args.solutions_file_path extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) solutions_df = get_solutions_df(extension, solutions_file_path) - solutions_df = filter_df_by_single_value(solutions_df, ColumnName.IS_PUBLIC.value, 'YES') - stats = __get_user_statistics(solutions_df, freq_boundary=args.freq_boundary, top_n=args.n) - dynamics = __get_student_dynamics(stats) - output_path = get_parent_folder(Path(solutions_file_path)) / f'student_issues_dynamics{Extension.CSV.value}' - write_dataframe_to_csv(output_path, dynamics) - print(f'The students dynamics was saved here: {output_path}') - total_top = __get_total_top(stats) - print('Total top issues:') - for i, (key, freq) in enumerate(total_top.items()): - print(f'{i}. {key} was found {freq} times') + + output_path = get_parent_folder(Path(solutions_file_path)) / 'dynamics' + output_path.mkdir(parents=True, exist_ok=True) + __get_users_statistics(solutions_df, output_path) return 0 except FileNotFoundError: diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py new file mode 100644 index 00000000..d70745f2 --- /dev/null +++ b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py @@ -0,0 +1,60 @@ +import argparse +import sys +from pathlib import Path +from statistics import median + +import pandas as pd +import plotly.graph_objects as go +from src.python.evaluation.common.pandas_util import logger +from src.python.evaluation.common.util import ColumnName +from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn +from src.python.review.common.file_system import Extension, extension_file_condition, get_all_file_system_items + + +def configure_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument('dynamics_folder_path', + type=lambda value: Path(value).absolute(), + help='Add description here') + + +def main() -> int: + parser = argparse.ArgumentParser() + configure_arguments(parser) + + try: + args = parser.parse_args() + dynamics_folder_path = args.dynamics_folder_path + dynamics_paths = get_all_file_system_items(dynamics_folder_path, extension_file_condition(Extension.CSV)) + dynamic_fig = go.Figure() + + medians = [] + for i, dynamic in enumerate(dynamics_paths): + dynamic_df = pd.read_csv(dynamic) + + dynamic_df = dynamic_df.head(100) + dynamic_fig.add_trace(go.Scatter( + x=dynamic_df[ColumnName.TIME.value], + y=dynamic_df[DynamicsColumn.ISSUE_COUNT.value], + name=f'user {i}', + )) + medians.append(median(dynamic_df[DynamicsColumn.ISSUE_COUNT.value])) + + dynamic_fig.update_layout(title='Code quality issues dynamics for Python', + xaxis_title='Submission number', + yaxis_title='Code quality issues count') + dynamic_fig.show() + + medians = go.Figure(data=go.Scatter(x=list(range(len(medians))), y=medians)) + medians.update_layout(title='Median values for code quality issues dynamics for Python', + xaxis_title='Student number', + yaxis_title='Median of code quality issues count') + medians.show() + return 0 + + except Exception: + logger.exception('An unexpected error.') + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py new file mode 100644 index 00000000..149016d7 --- /dev/null +++ b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py @@ -0,0 +1,72 @@ +import argparse +import sys +import uuid +from pathlib import Path +from typing import List + +import pandas as pd +from src.python.common.tool_arguments import RunToolArgument +from src.python.evaluation.common.csv_util import write_dataframe_to_csv +from src.python.evaluation.common.pandas_util import get_solutions_df, logger +from src.python.evaluation.common.util import ColumnName +from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension + + +''' +This scripts allows unpacking solutions to the solutions dataframe. +The initial dataframe has only several obligatory columns user_id,times,codes, +where is an array with times separated by ; symbol and + is an array with code fragments separated by ₣ symbol. +The and arrays have to has the same length. +The resulting dataset will have several: columns user_id,time,code, +where each row contains obly one time and one code fragment +''' + + +def configure_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, + type=lambda value: Path(value).absolute(), + help='Path to the compressed solutions') + + +def __parse_time_and_solutions(times_str: str, solutions_str: str) -> pd.DataFrame: + times = times_str.split(',') + solutions = solutions_str.split('₣') + time_to_solution = dict(zip(times, solutions)) + user_df = pd.DataFrame(time_to_solution.items(), columns=[ColumnName.TIME.value, ColumnName.CODE.value]) + user_df[ColumnName.USER.value] = uuid.uuid4() + return user_df + + +def __add_user_df(user_df_list: List[pd.DataFrame], user_df: pd.DataFrame): + user_df_list.append(user_df) + + +def main() -> int: + parser = argparse.ArgumentParser() + configure_arguments(parser) + + try: + args = parser.parse_args() + solutions_file_path = args.solutions_file_path + extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) + solutions_df = get_solutions_df(extension, solutions_file_path) + user_df_list = [] + solutions_df.apply(lambda row: __add_user_df(user_df_list, + __parse_time_and_solutions(row['times'], row['codes'])), axis=1) + unpacked_solutions = pd.concat(user_df_list) + output_path = get_parent_folder(Path(solutions_file_path)) / f'unpacked_solutions{Extension.CSV.value}' + write_dataframe_to_csv(output_path, unpacked_solutions) + return 0 + + except FileNotFoundError: + logger.error('CSV-file with the specified name does not exists.') + return 2 + + except Exception: + logger.exception('An unexpected error.') + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py b/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py index 0423a70b..7b1d4091 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py +++ b/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py @@ -1,9 +1,19 @@ from dataclasses import dataclass +from enum import Enum, unique from typing import Dict, List from src.python.evaluation.inspectors.common.statistics import PenaltyIssue +@unique +class DynamicsColumn(Enum): + ALL_ISSUES_COUNT = 'all_issues_count' + FORMATTING_ISSUES_COUNT = 'formatting_issues_count' + OTHER_ISSUES_COUNT = 'other_issues_count' + + ISSUE_COUNT = 'issue_count' + + @dataclass class UserStatistics: traceback: List[List[PenaltyIssue]] diff --git a/test/python/evaluation/testing_config.py b/test/python/evaluation/testing_config.py index f4bb9203..4927e2ce 100644 --- a/test/python/evaluation/testing_config.py +++ b/test/python/evaluation/testing_config.py @@ -20,5 +20,6 @@ def get_testing_arguments(to_add_traceback=None, to_add_tool_path=None, to_add_h testing_arguments.with_history = True testing_arguments.solutions_file_path = None + testing_arguments.to_drop_nan = False return testing_arguments diff --git a/whitelist.txt b/whitelist.txt index 67d97c06..cb7461dd 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -1,4 +1,6 @@ +Histogram2d abstractmethod +arange astype atclause bce @@ -188,3 +190,4 @@ writelines xaxis xlsx xpath +yaxis