From ff7732a53d4807405332f4cd2621a4e9ad989ce4 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Thu, 29 Jul 2021 11:10:24 +0300 Subject: [PATCH 01/10] Add scripts for surveys statistics gathering --- .../tutor_statistics.py | 4 +- .../paper_evaluation/survey_handler/README.md | 73 +++++++++++++++++++ .../survey_handler/__init__.py | 0 .../survey_handler/survey_statistics.py | 64 ++++++++++++++++ .../survey_statistics_gathering.py | 43 +++++++++++ 5 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 src/python/evaluation/paper_evaluation/survey_handler/README.md create mode 100644 src/python/evaluation/paper_evaluation/survey_handler/__init__.py create mode 100644 src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py create mode 100644 src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py diff --git a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py index 4e7bf2e6..083d6311 100644 --- a/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py +++ b/src/python/evaluation/paper_evaluation/comparison_with_other_tools/tutor_statistics.py @@ -38,7 +38,7 @@ def __init__(self, solutions_df: pd.DataFrame, to_drop_duplicates: bool = False) task_df[ComparisonColumnName.TUTOR_ERROR.value].dropna().values)) for cell_errors in errors_list: for error in cell_errors: - self.error_to_freq[error] += 1 + self.error_to_freq[error.strip()] += 1 self.task_to_error_freq[task] += 1 self.fragments_with_error += 1 self.task_to_freq = sort_freq_dict(self.task_to_freq) @@ -91,7 +91,7 @@ def __init__(self, solutions_df: pd.DataFrame, to_drop_duplicates: bool = False) def __parse_issues(issues_str: str) -> List[str]: if pd.isna(issues_str) or issues_str == ERROR_CONST: return [] - return issues_str.split(';') + return list(map(lambda i: i.strip(), issues_str.split(';'))) @staticmethod def __add_issues(issues_dict: Dict[str, int], issues: List[str]) -> None: diff --git a/src/python/evaluation/paper_evaluation/survey_handler/README.md b/src/python/evaluation/paper_evaluation/survey_handler/README.md new file mode 100644 index 00000000..4afe932e --- /dev/null +++ b/src/python/evaluation/paper_evaluation/survey_handler/README.md @@ -0,0 +1,73 @@ +# Surveys handlers + +These scripts allow handling surveys results for the SIGCSE paper. +We have two surveys (for Python and for Java) where participants should choose a fragments +that has better formatting. +Each question in the surveys have randomly orders for fragments. +The left fragment can have good formatting, but at the same time, it can have bad formatting. +To handle these cases we created JSON configs with this information and another one with the results. +These scripts allow processing these config files. + +## Usage + +Run the [survey_statistics_gathering.py](survey_statistics_gathering.py) with the arguments from command line. + +Required arguments: + +`questions_json_path` — path to the JSON with labelled questions; +`results_json_path` — path to the JSON with survey results. + +An example of `questions_json` file: +```json +{ + "questions": [ + { + "number": 1, + "left_fragment": "before_formatting", + "right_fragment": "after_formatting" + }, + { + "number": 2, + "left_fragment": "after_formatting", + "right_fragment": "before_formatting" + } + ] +} +``` + +An example of `results_json` file: + +```json +{ + "questions": [ + { + "number": 1, + "left_fragment": 0, + "right_fragment": 11, + "both": 0 + }, + { + "number": 2, + "left_fragment": 10, + "right_fragment": 0, + "both": 1 + } + ] +} +``` + +An example of the statistics: +```text +total participants=11 +------before----after----any---- +1. 0 11 0 +2. 1 10 0 +3. 0 11 0 +4. 0 11 0 +5. 0 11 0 +6. 1 10 0 +7. 0 11 0 +8. 1 8 2 +9. 0 11 0 +10. 0 8 3 +``` diff --git a/src/python/evaluation/paper_evaluation/survey_handler/__init__.py b/src/python/evaluation/paper_evaluation/survey_handler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py new file mode 100644 index 00000000..5474437c --- /dev/null +++ b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass +from enum import unique, Enum +from typing import List, Dict, Union, Any + + +@dataclass +class Question: + with_formatting_count: int = 0 + without_formatting_count: int = 0 + any_formatting_count: int = 0 + + def get_total(self): + return self.with_formatting_count + self.without_formatting_count + self.any_formatting_count + + +@unique +class SurveyJsonField(Enum): + NUMBER = 'number' + LEFT_FRAGMENT = 'left_fragment' + RIGHT_FRAGMENT = 'right_fragment' + + BEFORE_FORMATTING = 'before_formatting' + BOTH = 'both' + + QUESTIONS = 'questions' + + +@dataclass +class SurveyStatistics: + questions: List[Question] + + def __init__(self, questions_json: List[Dict[str, Any]], results_json: List[Dict[str, int]]): + self.questions = [] + for result_json in results_json: + question_number = result_json[SurveyJsonField.NUMBER.value] + question = self.__find_json_question(questions_json, question_number) + if question[SurveyJsonField.LEFT_FRAGMENT.value] == SurveyJsonField.BEFORE_FORMATTING.value: + without_formatting_count = result_json[SurveyJsonField.LEFT_FRAGMENT.value] + with_formatting_count = result_json[SurveyJsonField.RIGHT_FRAGMENT.value] + else: + without_formatting_count = result_json[SurveyJsonField.RIGHT_FRAGMENT.value] + with_formatting_count = result_json[SurveyJsonField.LEFT_FRAGMENT.value] + any_formatting_count = result_json[SurveyJsonField.BOTH.value] + self.questions.append(Question(with_formatting_count, without_formatting_count, any_formatting_count)) + + @staticmethod + def __find_json_question(questions_json: List[Dict[str, Any]], question_number: int) -> Dict[str, Any]: + for question in questions_json: + if question[SurveyJsonField.NUMBER.value] == question_number: + return question + raise ValueError(f'Did not find question {question_number}') + + def print_stat(self): + if len(self.questions) == 0: + print('No questions found') + return + print(f'total participants={self.questions[0].get_total()}') + print('------before----after----any----') + for index, question in enumerate(self.questions): + print(f'{index + 1}.\t\t{question.without_formatting_count}\t\t{question.with_formatting_count}\t\t ' + f'{question.any_formatting_count}') + + + diff --git a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py new file mode 100644 index 00000000..2c2e4e6f --- /dev/null +++ b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py @@ -0,0 +1,43 @@ +import argparse +import json +import sys +from pathlib import Path + +from src.python.evaluation.evaluation_run_tool import logger +from src.python.evaluation.paper_evaluation.survey_handler.survey_statistics import SurveyStatistics, SurveyJsonField +from src.python.review.common.file_system import get_content_from_file + + +def configure_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument('questions_json_path', + type=lambda value: Path(value).absolute(), + help='Path to the JSON with labelled questions') + + parser.add_argument('results_json_path', + type=lambda value: Path(value).absolute(), + help='Path to the JSON with survey results') + + +def main() -> int: + parser = argparse.ArgumentParser() + configure_arguments(parser) + + try: + args = parser.parse_args() + questions_json = json.loads(get_content_from_file(args.questions_json_path)) + results_json = json.loads(get_content_from_file(args.results_json_path)) + stat = SurveyStatistics(questions_json[SurveyJsonField.QUESTIONS.value], results_json[SurveyJsonField.QUESTIONS.value]) + stat.print_stat() + return 0 + + except FileNotFoundError: + logger.error('JSON file did not found') + return 2 + + except Exception: + logger.exception('An unexpected error.') + return 2 + + +if __name__ == '__main__': + sys.exit(main()) From 00282d3cdfed3e7685f735a8394c016ebbc670bd Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Thu, 29 Jul 2021 11:26:55 +0300 Subject: [PATCH 02/10] Fix flake8 --- .../paper_evaluation/survey_handler/survey_statistics.py | 7 ++----- .../survey_handler/survey_statistics_gathering.py | 7 +++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py index 5474437c..8cfc898b 100644 --- a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py +++ b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from enum import unique, Enum -from typing import List, Dict, Union, Any +from enum import Enum, unique +from typing import Any, Dict, List @dataclass @@ -59,6 +59,3 @@ def print_stat(self): for index, question in enumerate(self.questions): print(f'{index + 1}.\t\t{question.without_formatting_count}\t\t{question.with_formatting_count}\t\t ' f'{question.any_formatting_count}') - - - diff --git a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py index 2c2e4e6f..82b8b7d3 100644 --- a/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py +++ b/src/python/evaluation/paper_evaluation/survey_handler/survey_statistics_gathering.py @@ -4,7 +4,7 @@ from pathlib import Path from src.python.evaluation.evaluation_run_tool import logger -from src.python.evaluation.paper_evaluation.survey_handler.survey_statistics import SurveyStatistics, SurveyJsonField +from src.python.evaluation.paper_evaluation.survey_handler.survey_statistics import SurveyJsonField, SurveyStatistics from src.python.review.common.file_system import get_content_from_file @@ -26,7 +26,10 @@ def main() -> int: args = parser.parse_args() questions_json = json.loads(get_content_from_file(args.questions_json_path)) results_json = json.loads(get_content_from_file(args.results_json_path)) - stat = SurveyStatistics(questions_json[SurveyJsonField.QUESTIONS.value], results_json[SurveyJsonField.QUESTIONS.value]) + stat = SurveyStatistics( + questions_json[SurveyJsonField.QUESTIONS.value], + results_json[SurveyJsonField.QUESTIONS.value], + ) stat.print_stat() return 0 From 08419ac4d2f62b16833361d0f551ba91da520ea3 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Tue, 3 Aug 2021 13:56:03 +0300 Subject: [PATCH 03/10] Add dynamics handlers --- src/python/evaluation/evaluation_run_tool.py | 4 +- .../inspectors_stat/statistics_gathering.py | 3 +- .../paper_evaluation/user_dynamics/README.md | 30 ++--- .../user_dynamics/dynamics_gathering.py | 106 +++++++----------- .../user_dynamics/dynamics_visualization.py | 60 ++++++++++ .../user_dynamics/unpack_solutions.py | 72 ++++++++++++ .../user_dynamics/user_statistics.py | 10 ++ 7 files changed, 205 insertions(+), 80 deletions(-) create mode 100644 src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py create mode 100644 src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py diff --git a/src/python/evaluation/evaluation_run_tool.py b/src/python/evaluation/evaluation_run_tool.py index bff335ed..f6a42aeb 100644 --- a/src/python/evaluation/evaluation_run_tool.py +++ b/src/python/evaluation/evaluation_run_tool.py @@ -80,7 +80,8 @@ def get_language_version(lang_key: str) -> LanguageVersion: raise KeyError(e) -def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str], config: EvaluationConfig) -> str: +def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str], + config: EvaluationConfig) -> Optional[str]: print(f'current id: {fragment_id}') # Tool does not work correctly with tmp files from module on macOS # thus we create a real file in the file system @@ -107,6 +108,7 @@ def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataF if config.traceback: report[ColumnName.TRACEBACK.value] = [] try: + lang_code_dataframe = lang_code_dataframe.dropna() lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply( lambda row: __inspect_row(row[ColumnName.LANG.value], row[ColumnName.CODE.value], diff --git a/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py b/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py index 046c2606..0de99fc6 100644 --- a/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py +++ b/src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py @@ -97,8 +97,7 @@ def print_stat(language: Language, stat: IssuesStat) -> None: print(f'Collected statistics for {language.value.lower()} language:') for issue_type, freq in stat.items(): print(f'{issue_type}: {freq} times;') - print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories. ' - f'Most likely it is {IssueType.INFO} category') + print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories.') def __parse_language(language: str) -> Language: diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/README.md b/src/python/evaluation/paper_evaluation/user_dynamics/README.md index 9e46df11..7d004bb1 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/README.md +++ b/src/python/evaluation/paper_evaluation/user_dynamics/README.md @@ -10,20 +10,22 @@ Required arguments: `solutions_file_path` — path to csv-file with code samples. -Optional arguments: -Argument | Description ---- | --- -|**‑fb**, **‑‑freq-boundary**| The boundary of solutions count for one student to analyze. The default value is 100.| -|**‑n**, **‑‑n**| Top n popular issues in solutions. The default value is 100. | +In the result a file with students issues dynamics will be created. +We have three categories of dynamics: +- all (count of all code quality issues expect INFO issues) +- formatting (count of formatting code quality issues from CODE_STYLE category) +- other (all issues minus formatting issues) -In the result a file with students issues dynamics will be created. -Also, the top of issues for all students will be printed into the terminal. This statistics has key of issue and frequency for all students. +Each type of dynamics will be saved into a separated folder with csv files for each student. +Each csv file has only two columns: fragment id and issues count. -An example of issues dynamics: +An example of the csv file: ```text -user,traceback -0,"0,0,0,0,0,0,0,0,0,0,0,1,0,0,3,0,0,0,0,2,0,4,0,6,3,0,3,0,0,0,1,1,0,0,0,1,0,0,0,2,0,0,0,0,0,0,4,0,0,0,1,6,0,1,0,1,3,0,0,1,1,0,0,0,0,0,3,6,1,0,0,0,0,0,0,0,4,1,0,0,1,0,8,0,2,8,0,0,0,0,1,1,1,1,3,7,23,0,9" -1,"0,0,0,3,0,0,2,1,0,0,0,0,4,1,0,0,1,1,0,0,0,0,0,6,0,1,1,0,8,1,2,1,1,0,0,1,0,4,10,1,1,1,3,0,1,0,0,0,1,0,0,0,0,0,0,2,0,3,0,0,2,2,3,2,0,0,0,1,0,1,1,0,0,1,0,4,6,2,0,0,1,0,0,0,0,2,0,0,0,2,1,2,1,0,1,7,1,0,1,1,0,1,0" -``` -Each number in the traceback column is the count of issues in one solution. -The numbers of issues sorted by timestamps. \ No newline at end of file +issue_count,time +2,0 +20,1 +16,2 +15,3 +5,4 +5,5 +``` \ No newline at end of file diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py index 7d9bbec5..023edb9c 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py +++ b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py @@ -1,19 +1,18 @@ import argparse import sys -from collections import Counter from pathlib import Path -from typing import Dict, List +from typing import List import pandas as pd +import numpy as np from src.python.common.tool_arguments import RunToolArgument from src.python.evaluation.common.csv_util import write_dataframe_to_csv from src.python.evaluation.common.pandas_util import ( - drop_duplicates, filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger, + filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger, ) from src.python.evaluation.common.util import ColumnName from src.python.evaluation.inspectors.common.statistics import PenaltyIssue -from src.python.evaluation.paper_evaluation.comparison_with_other_tools.tutor_statistics import sort_freq_dict -from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import UserStatistics +from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension from src.python.review.inspectors.issue import IssueType @@ -23,58 +22,45 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None: type=lambda value: Path(value).absolute(), help=RunToolArgument.SOLUTIONS_FILE_PATH.value) - parser.add_argument('-fb', '--freq-boundary', - help='The boundary of solutions count for one student to analyze', - type=int, - default=100) - - parser.add_argument('-n', '--n', - help='Top n popular issues in solutions', - type=int, - default=10) - - -def __get_top_freq_issues(issues: List[List[PenaltyIssue]], n: int) -> Dict[str, int]: - all_issues = list(map(lambda i: i.origin_class, [item for sublist in issues for item in sublist])) - return dict(Counter(all_issues).most_common(n)) - - -# Get statistics only for users that have >= freq_boundary solutions in solutions_df -# Statistics for each student has: -# - - list of list of issues, but without INFO issues -# - - for each key of issue from has frequency. -# Contains only top_n issues -def __get_user_statistics(solutions_df: pd.DataFrame, freq_boundary: int = 100, - top_n: int = 10) -> List[UserStatistics]: - stat = [] - counts = solutions_df[ColumnName.USER.value].value_counts() - solutions_df = solutions_df[solutions_df[ColumnName.USER.value].isin(counts[counts > freq_boundary].index)] - for user in solutions_df[ColumnName.USER.value].unique(): - user_df = filter_df_by_single_value(solutions_df, - ColumnName.USER.value, user).sort_values(ColumnName.TIME.value) - user_df = drop_duplicates(user_df) - traceback = list(map(lambda t: get_issues_from_json(t), - list(user_df[ColumnName.TRACEBACK.value]))) - # Filter info category - traceback = list(filter(lambda issues_list: filter(lambda i: i.type != IssueType.INFO, issues_list), traceback)) - top_issues = __get_top_freq_issues(traceback, top_n) - stat.append(UserStatistics(traceback, top_issues)) - return stat + +ALL_ISSUES_COUNT = DynamicsColumn.ALL_ISSUES_COUNT.value +FORMATTING_ISSUES_COUNT = DynamicsColumn.FORMATTING_ISSUES_COUNT.value +OTHER_ISSUES_COUNT = DynamicsColumn.OTHER_ISSUES_COUNT.value -def __get_student_dynamics(stats: List[UserStatistics]) -> pd.DataFrame: - dynamics = map(lambda s: s.get_traceback_dynamics(), stats) - dynamics_dict = {i: ','.join(map(lambda d: str(d), dyn)) for (i, dyn) in enumerate(dynamics)} - return pd.DataFrame(dynamics_dict.items(), columns=[ColumnName.USER.value, ColumnName.TRACEBACK.value]) +def __get_all_issues(traceback: str) -> List[PenaltyIssue]: + return list(filter(lambda i: i.type != IssueType.INFO, get_issues_from_json(traceback))) -def __get_total_top(stats: List[UserStatistics]) -> Dict[str, int]: - total_top_n = {} - for d in map(lambda s: s.top_issues, stats): - for k, v in d.items(): - total_top_n.setdefault(k, 0) - total_top_n[k] += v - return sort_freq_dict(total_top_n) +def __get_formatting_issues(traceback: str) -> List[PenaltyIssue]: + return list(filter(lambda i: i.type == IssueType.CODE_STYLE, __get_all_issues(traceback))) + + +def __write_dynamics(output_path: Path, user_fragments: pd.DataFrame, index: int) -> None: + output_path.mkdir(parents=True, exist_ok=True) + user_fragments.columns = [DynamicsColumn.ISSUE_COUNT.value] + user_fragments[ColumnName.TIME.value] = np.arange(len(user_fragments)) + write_dataframe_to_csv(output_path / f'user_{index}{Extension.CSV.value}', user_fragments) + + +def __get_users_statistics(solutions_df: pd.DataFrame, output_path: Path) -> None: + users = solutions_df[ColumnName.USER.value].unique() + for index, user in enumerate(users): + user_df = filter_df_by_single_value(solutions_df, + ColumnName.USER.value, user).sort_values(ColumnName.TIME.value) + user_df[ALL_ISSUES_COUNT] = user_df.apply(lambda row: + len(__get_all_issues( + row[ColumnName.TRACEBACK.value])), + axis=1) + user_df[FORMATTING_ISSUES_COUNT] = user_df.apply(lambda row: + len(__get_formatting_issues( + row[ColumnName.TRACEBACK.value])), + axis=1) + user_df[OTHER_ISSUES_COUNT] = user_df[ALL_ISSUES_COUNT] - user_df[FORMATTING_ISSUES_COUNT] + + __write_dynamics(output_path / 'all', user_df[[ALL_ISSUES_COUNT]], index) + __write_dynamics(output_path / 'formatting', user_df[[FORMATTING_ISSUES_COUNT]], index) + __write_dynamics(output_path / 'other', user_df[[OTHER_ISSUES_COUNT]], index) def main() -> int: @@ -86,16 +72,10 @@ def main() -> int: solutions_file_path = args.solutions_file_path extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) solutions_df = get_solutions_df(extension, solutions_file_path) - solutions_df = filter_df_by_single_value(solutions_df, ColumnName.IS_PUBLIC.value, 'YES') - stats = __get_user_statistics(solutions_df, freq_boundary=args.freq_boundary, top_n=args.n) - dynamics = __get_student_dynamics(stats) - output_path = get_parent_folder(Path(solutions_file_path)) / f'student_issues_dynamics{Extension.CSV.value}' - write_dataframe_to_csv(output_path, dynamics) - print(f'The students dynamics was saved here: {output_path}') - total_top = __get_total_top(stats) - print('Total top issues:') - for i, (key, freq) in enumerate(total_top.items()): - print(f'{i}. {key} was found {freq} times') + + output_path = get_parent_folder(Path(solutions_file_path)) / 'dynamics' + output_path.mkdir(parents=True, exist_ok=True) + __get_users_statistics(solutions_df, output_path) return 0 except FileNotFoundError: diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py new file mode 100644 index 00000000..d70745f2 --- /dev/null +++ b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py @@ -0,0 +1,60 @@ +import argparse +import sys +from pathlib import Path +from statistics import median + +import pandas as pd +import plotly.graph_objects as go +from src.python.evaluation.common.pandas_util import logger +from src.python.evaluation.common.util import ColumnName +from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn +from src.python.review.common.file_system import Extension, extension_file_condition, get_all_file_system_items + + +def configure_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument('dynamics_folder_path', + type=lambda value: Path(value).absolute(), + help='Add description here') + + +def main() -> int: + parser = argparse.ArgumentParser() + configure_arguments(parser) + + try: + args = parser.parse_args() + dynamics_folder_path = args.dynamics_folder_path + dynamics_paths = get_all_file_system_items(dynamics_folder_path, extension_file_condition(Extension.CSV)) + dynamic_fig = go.Figure() + + medians = [] + for i, dynamic in enumerate(dynamics_paths): + dynamic_df = pd.read_csv(dynamic) + + dynamic_df = dynamic_df.head(100) + dynamic_fig.add_trace(go.Scatter( + x=dynamic_df[ColumnName.TIME.value], + y=dynamic_df[DynamicsColumn.ISSUE_COUNT.value], + name=f'user {i}', + )) + medians.append(median(dynamic_df[DynamicsColumn.ISSUE_COUNT.value])) + + dynamic_fig.update_layout(title='Code quality issues dynamics for Python', + xaxis_title='Submission number', + yaxis_title='Code quality issues count') + dynamic_fig.show() + + medians = go.Figure(data=go.Scatter(x=list(range(len(medians))), y=medians)) + medians.update_layout(title='Median values for code quality issues dynamics for Python', + xaxis_title='Student number', + yaxis_title='Median of code quality issues count') + medians.show() + return 0 + + except Exception: + logger.exception('An unexpected error.') + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py new file mode 100644 index 00000000..481e20c6 --- /dev/null +++ b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py @@ -0,0 +1,72 @@ +import argparse +import sys +import uuid +from pathlib import Path +from typing import List + +import pandas as pd +from src.python.common.tool_arguments import RunToolArgument +from src.python.evaluation.common.csv_util import write_dataframe_to_csv +from src.python.evaluation.common.util import ColumnName +from src.python.evaluation.common.pandas_util import get_solutions_df, logger +from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension + + +''' +This scripts allows unpacking solutions to the solutions dataframe. +The initial dataframe has only several obligatory columns user_id,times,codes, +where is an array with times separated by ; symbol and + is an array with code fragments separated by ₣ symbol. +The and arrays have to has the same length. +The resulting dataset will have several: columns user_id,time,code, +where each row contains obly one time and one code fragment +''' + + +def configure_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name, + type=lambda value: Path(value).absolute(), + help='Path to the compressed solutions') + + +def __parse_time_and_solutions(times_str: str, solutions_str: str) -> pd.DataFrame: + times = times_str.split(',') + solutions = solutions_str.split('₣') + time_to_solution = dict(zip(times, solutions)) + user_df = pd.DataFrame(time_to_solution.items(), columns=[ColumnName.TIME.value, ColumnName.CODE.value]) + user_df[ColumnName.USER.value] = uuid.uuid4() + return user_df + + +def __add_user_df(user_df_list: List[pd.DataFrame], user_df: pd.DataFrame): + user_df_list.append(user_df) + + +def main() -> int: + parser = argparse.ArgumentParser() + configure_arguments(parser) + + try: + args = parser.parse_args() + solutions_file_path = args.solutions_file_path + extension = get_restricted_extension(solutions_file_path, [Extension.CSV]) + solutions_df = get_solutions_df(extension, solutions_file_path) + user_df_list = [] + solutions_df.apply(lambda row: __add_user_df(user_df_list, + __parse_time_and_solutions(row['times'], row['codes'])), axis=1) + unpacked_solutions = pd.concat(user_df_list) + output_path = get_parent_folder(Path(solutions_file_path)) / f'unpacked_solutions{Extension.CSV.value}' + write_dataframe_to_csv(output_path, unpacked_solutions) + return 0 + + except FileNotFoundError: + logger.error('CSV-file with the specified name does not exists.') + return 2 + + except Exception: + logger.exception('An unexpected error.') + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py b/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py index 0423a70b..7b1d4091 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py +++ b/src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py @@ -1,9 +1,19 @@ from dataclasses import dataclass +from enum import Enum, unique from typing import Dict, List from src.python.evaluation.inspectors.common.statistics import PenaltyIssue +@unique +class DynamicsColumn(Enum): + ALL_ISSUES_COUNT = 'all_issues_count' + FORMATTING_ISSUES_COUNT = 'formatting_issues_count' + OTHER_ISSUES_COUNT = 'other_issues_count' + + ISSUE_COUNT = 'issue_count' + + @dataclass class UserStatistics: traceback: List[List[PenaltyIssue]] From 7ef84e04c3f049b46316e679deebf6d75f321afd Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Tue, 3 Aug 2021 14:38:02 +0300 Subject: [PATCH 04/10] Fix flake8 --- .../paper_evaluation/user_dynamics/dynamics_gathering.py | 2 +- .../paper_evaluation/user_dynamics/unpack_solutions.py | 2 +- whitelist.txt | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py index 023edb9c..6c4ab4ad 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py +++ b/src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import List -import pandas as pd import numpy as np +import pandas as pd from src.python.common.tool_arguments import RunToolArgument from src.python.evaluation.common.csv_util import write_dataframe_to_csv from src.python.evaluation.common.pandas_util import ( diff --git a/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py index 481e20c6..149016d7 100644 --- a/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py +++ b/src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py @@ -7,8 +7,8 @@ import pandas as pd from src.python.common.tool_arguments import RunToolArgument from src.python.evaluation.common.csv_util import write_dataframe_to_csv -from src.python.evaluation.common.util import ColumnName from src.python.evaluation.common.pandas_util import get_solutions_df, logger +from src.python.evaluation.common.util import ColumnName from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension diff --git a/whitelist.txt b/whitelist.txt index 67d97c06..968cff9d 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -1,4 +1,5 @@ abstractmethod +arange astype atclause bce @@ -17,6 +18,7 @@ const consts copytree coroutines +ColumnName csv ctor cuda @@ -182,8 +184,10 @@ varargs wandb warmup webp +Histogram2d wmc wps +yaxis writelines xaxis xlsx From bacef64cd02452f8654f1fa1f17ccf375ac7fcb6 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Tue, 3 Aug 2021 14:57:34 +0300 Subject: [PATCH 05/10] Sort whitelist --- whitelist.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/whitelist.txt b/whitelist.txt index 968cff9d..ff594c39 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -1,3 +1,5 @@ +ColumnName +Histogram2d abstractmethod arange astype @@ -18,7 +20,6 @@ const consts copytree coroutines -ColumnName csv ctor cuda @@ -184,11 +185,10 @@ varargs wandb warmup webp -Histogram2d wmc wps -yaxis writelines xaxis xlsx xpath +yaxis From 3eee1badcdaed73cbdf116efa6f8ad7f03587dfb Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Tue, 3 Aug 2021 15:32:02 +0300 Subject: [PATCH 06/10] Fix evaluation run tool script --- src/python/evaluation/README.md | 1 + src/python/evaluation/evaluation_run_tool.py | 10 ++++++++-- test/python/evaluation/test_xlsx_file_structure.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/python/evaluation/README.md b/src/python/evaluation/README.md index f8fafbe0..9e0ff653 100644 --- a/src/python/evaluation/README.md +++ b/src/python/evaluation/README.md @@ -33,3 +33,4 @@ Argument | Description |**‑‑traceback**| To include a column with errors traceback into an output file. Default is `False`.| |**‑ofp**, **‑‑output‑folder‑path**| An explicit folder path to store file with results. Default is a parent directory of a folder with xlsx-file or csv-file sent for inspection. | |**‑ofn**, **‑‑output‑file‑name**| A name of an output file where evaluation results will be stored. Default is `results.xlsx` or `results.csv`.| +|**‑‑to_drop_nan**| If True, empty code fragments will be deleted from df. Default is `False`.| diff --git a/src/python/evaluation/evaluation_run_tool.py b/src/python/evaluation/evaluation_run_tool.py index f6a42aeb..737ce7c2 100644 --- a/src/python/evaluation/evaluation_run_tool.py +++ b/src/python/evaluation/evaluation_run_tool.py @@ -70,6 +70,10 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None: 'must contain the history of previous errors.', action='store_true') + parser.add_argument('--to_drop_nan', + help='If True, empty code fragments will be deleted from df', + action='store_true') + def get_language_version(lang_key: str) -> LanguageVersion: try: @@ -100,7 +104,8 @@ def __get_grade_from_traceback(traceback: str) -> str: # TODO: calculate grade after it -def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataFrame) -> pd.DataFrame: +def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataFrame, + to_drop_nan: bool = True) -> pd.DataFrame: report = pd.DataFrame(columns=lang_code_dataframe.columns) report[ColumnName.TRACEBACK.value] = [] @@ -108,7 +113,8 @@ def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataF if config.traceback: report[ColumnName.TRACEBACK.value] = [] try: - lang_code_dataframe = lang_code_dataframe.dropna() + if to_drop_nan: + lang_code_dataframe = lang_code_dataframe.dropna() lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply( lambda row: __inspect_row(row[ColumnName.LANG.value], row[ColumnName.CODE.value], diff --git a/test/python/evaluation/test_xlsx_file_structure.py b/test/python/evaluation/test_xlsx_file_structure.py index f043772d..6ea2af1e 100644 --- a/test/python/evaluation/test_xlsx_file_structure.py +++ b/test/python/evaluation/test_xlsx_file_structure.py @@ -20,4 +20,4 @@ def test_wrong_column(file_name: str): testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / file_name config = EvaluationConfig(testing_arguments_dict) lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - assert inspect_solutions_df(config, lang_code_dataframe) + assert inspect_solutions_df(config, lang_code_dataframe, to_drop_nan=False) From 6950f5e1863511be6a9e263aedeb319f523a5a70 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Tue, 3 Aug 2021 15:32:58 +0300 Subject: [PATCH 07/10] Use to_drop_nan arg --- src/python/evaluation/evaluation_config.py | 1 + src/python/evaluation/evaluation_run_tool.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/evaluation/evaluation_config.py b/src/python/evaluation/evaluation_config.py index b894d529..681f9e75 100644 --- a/src/python/evaluation/evaluation_config.py +++ b/src/python/evaluation/evaluation_config.py @@ -26,6 +26,7 @@ def __init__(self, args: Namespace): self.output_folder_path: Union[str, Path] = args.output_folder_path self.extension: Extension = get_restricted_extension(self.solutions_file_path, [Extension.XLSX, Extension.CSV]) self.__init_output_file_name(args.output_file_name) + self.to_drop_nan = args.to_drop_nan def __init_output_file_name(self, output_file_name: Optional[str]): if output_file_name is None: diff --git a/src/python/evaluation/evaluation_run_tool.py b/src/python/evaluation/evaluation_run_tool.py index 737ce7c2..453426f5 100644 --- a/src/python/evaluation/evaluation_run_tool.py +++ b/src/python/evaluation/evaluation_run_tool.py @@ -149,7 +149,7 @@ def main() -> int: args = parser.parse_args() config = EvaluationConfig(args) lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - results = inspect_solutions_df(config, lang_code_dataframe) + results = inspect_solutions_df(config, lang_code_dataframe, to_drop_nan=config.to_drop_nan) write_df_to_file(results, config.get_output_file_path(), config.extension) end = time.time() print(f'All time: {end - start}') From eec689915b0072c30234caaff66754910fb7aca5 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Tue, 3 Aug 2021 15:44:55 +0300 Subject: [PATCH 08/10] Fix tests --- src/python/evaluation/evaluation_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/python/evaluation/evaluation_config.py b/src/python/evaluation/evaluation_config.py index 681f9e75..a060e7ba 100644 --- a/src/python/evaluation/evaluation_config.py +++ b/src/python/evaluation/evaluation_config.py @@ -26,7 +26,10 @@ def __init__(self, args: Namespace): self.output_folder_path: Union[str, Path] = args.output_folder_path self.extension: Extension = get_restricted_extension(self.solutions_file_path, [Extension.XLSX, Extension.CSV]) self.__init_output_file_name(args.output_file_name) - self.to_drop_nan = args.to_drop_nan + try: + self.to_drop_nan: bool = args.to_drop_nan + except AttributeError: + self.to_drop_nan = False def __init_output_file_name(self, output_file_name: Optional[str]): if output_file_name is None: From 2edbacee9e9ecc138284c495a627953f3ecff582 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Mon, 9 Aug 2021 10:22:26 +0300 Subject: [PATCH 09/10] Fix PR#95 comments --- src/python/evaluation/README.md | 2 +- src/python/evaluation/evaluation_config.py | 5 +---- src/python/evaluation/evaluation_run_tool.py | 9 ++++----- test/python/evaluation/testing_config.py | 1 + whitelist.txt | 1 - 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/python/evaluation/README.md b/src/python/evaluation/README.md index 9e0ff653..b98e2deb 100644 --- a/src/python/evaluation/README.md +++ b/src/python/evaluation/README.md @@ -33,4 +33,4 @@ Argument | Description |**‑‑traceback**| To include a column with errors traceback into an output file. Default is `False`.| |**‑ofp**, **‑‑output‑folder‑path**| An explicit folder path to store file with results. Default is a parent directory of a folder with xlsx-file or csv-file sent for inspection. | |**‑ofn**, **‑‑output‑file‑name**| A name of an output file where evaluation results will be stored. Default is `results.xlsx` or `results.csv`.| -|**‑‑to_drop_nan**| If True, empty code fragments will be deleted from df. Default is `False`.| +|**‑‑to‑drop‑nan**| If True, empty code fragments will be deleted from df. Default is `False`.| diff --git a/src/python/evaluation/evaluation_config.py b/src/python/evaluation/evaluation_config.py index a060e7ba..06672696 100644 --- a/src/python/evaluation/evaluation_config.py +++ b/src/python/evaluation/evaluation_config.py @@ -26,10 +26,7 @@ def __init__(self, args: Namespace): self.output_folder_path: Union[str, Path] = args.output_folder_path self.extension: Extension = get_restricted_extension(self.solutions_file_path, [Extension.XLSX, Extension.CSV]) self.__init_output_file_name(args.output_file_name) - try: - self.to_drop_nan: bool = args.to_drop_nan - except AttributeError: - self.to_drop_nan = False + self.to_drop_nan: bool = args.to_drop_nan def __init_output_file_name(self, output_file_name: Optional[str]): if output_file_name is None: diff --git a/src/python/evaluation/evaluation_run_tool.py b/src/python/evaluation/evaluation_run_tool.py index 453426f5..6304bf4f 100644 --- a/src/python/evaluation/evaluation_run_tool.py +++ b/src/python/evaluation/evaluation_run_tool.py @@ -70,7 +70,7 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None: 'must contain the history of previous errors.', action='store_true') - parser.add_argument('--to_drop_nan', + parser.add_argument('--to-drop-nan', help='If True, empty code fragments will be deleted from df', action='store_true') @@ -104,8 +104,7 @@ def __get_grade_from_traceback(traceback: str) -> str: # TODO: calculate grade after it -def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataFrame, - to_drop_nan: bool = True) -> pd.DataFrame: +def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataFrame) -> pd.DataFrame: report = pd.DataFrame(columns=lang_code_dataframe.columns) report[ColumnName.TRACEBACK.value] = [] @@ -113,7 +112,7 @@ def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataF if config.traceback: report[ColumnName.TRACEBACK.value] = [] try: - if to_drop_nan: + if config.to_drop_nan: lang_code_dataframe = lang_code_dataframe.dropna() lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply( lambda row: __inspect_row(row[ColumnName.LANG.value], @@ -149,7 +148,7 @@ def main() -> int: args = parser.parse_args() config = EvaluationConfig(args) lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - results = inspect_solutions_df(config, lang_code_dataframe, to_drop_nan=config.to_drop_nan) + results = inspect_solutions_df(config, lang_code_dataframe) write_df_to_file(results, config.get_output_file_path(), config.extension) end = time.time() print(f'All time: {end - start}') diff --git a/test/python/evaluation/testing_config.py b/test/python/evaluation/testing_config.py index f4bb9203..58250860 100644 --- a/test/python/evaluation/testing_config.py +++ b/test/python/evaluation/testing_config.py @@ -20,5 +20,6 @@ def get_testing_arguments(to_add_traceback=None, to_add_tool_path=None, to_add_h testing_arguments.with_history = True testing_arguments.solutions_file_path = None + testing_arguments.to_drop_nan = True return testing_arguments diff --git a/whitelist.txt b/whitelist.txt index ff594c39..cb7461dd 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -1,4 +1,3 @@ -ColumnName Histogram2d abstractmethod arange From b7222ec9881a84318d566bd4101490265d4d24d4 Mon Sep 17 00:00:00 2001 From: "Anastasiia.Birillo" Date: Mon, 9 Aug 2021 10:34:52 +0300 Subject: [PATCH 10/10] Fix tests --- test/python/evaluation/test_xlsx_file_structure.py | 2 +- test/python/evaluation/testing_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/python/evaluation/test_xlsx_file_structure.py b/test/python/evaluation/test_xlsx_file_structure.py index 6ea2af1e..f043772d 100644 --- a/test/python/evaluation/test_xlsx_file_structure.py +++ b/test/python/evaluation/test_xlsx_file_structure.py @@ -20,4 +20,4 @@ def test_wrong_column(file_name: str): testing_arguments_dict.solutions_file_path = XLSX_DATA_FOLDER / file_name config = EvaluationConfig(testing_arguments_dict) lang_code_dataframe = get_solutions_df(config.extension, config.solutions_file_path) - assert inspect_solutions_df(config, lang_code_dataframe, to_drop_nan=False) + assert inspect_solutions_df(config, lang_code_dataframe) diff --git a/test/python/evaluation/testing_config.py b/test/python/evaluation/testing_config.py index 58250860..4927e2ce 100644 --- a/test/python/evaluation/testing_config.py +++ b/test/python/evaluation/testing_config.py @@ -20,6 +20,6 @@ def get_testing_arguments(to_add_traceback=None, to_add_tool_path=None, to_add_h testing_arguments.with_history = True testing_arguments.solutions_file_path = None - testing_arguments.to_drop_nan = True + testing_arguments.to_drop_nan = False return testing_arguments