User dynamics

src/python/evaluation/README.md

@@ -33,3 +33,4 @@ Argument | Description
 |**‑‑traceback**| To include a column with errors traceback into an output file. Default is `False`.|
 |**‑ofp**, **‑‑output‑folder‑path**| An explicit folder path to store file with results. Default is a parent directory of a folder with xlsx-file or csv-file sent for inspection. |
 |**‑ofn**, **‑‑output‑file‑name**| A name of an output file where evaluation results will be stored. Default is `results.xlsx` or `results.csv`.|
+|**‑‑to‑drop‑nan**| If True, empty code fragments will be deleted from df. Default is `False`.|

src/python/evaluation/evaluation_config.py

@@ -26,6 +26,7 @@ def __init__(self, args: Namespace):
         self.output_folder_path: Union[str, Path] = args.output_folder_path
         self.extension: Extension = get_restricted_extension(self.solutions_file_path, [Extension.XLSX, Extension.CSV])
         self.__init_output_file_name(args.output_file_name)
+        self.to_drop_nan: bool = args.to_drop_nan
 
     def __init_output_file_name(self, output_file_name: Optional[str]):
         if output_file_name is None:

src/python/evaluation/evaluation_run_tool.py

@@ -70,6 +70,10 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None:
                              'must contain the history of previous errors.',
                         action='store_true')
 
+    parser.add_argument('--to-drop-nan',
+                        help='If True, empty code fragments will be deleted from df',
+                        action='store_true')
+
 
 def get_language_version(lang_key: str) -> LanguageVersion:
     try:
@@ -80,7 +84,8 @@ def get_language_version(lang_key: str) -> LanguageVersion:
         raise KeyError(e)
 
 
-def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str], config: EvaluationConfig) -> str:
+def __inspect_row(lang: str, code: str, fragment_id: int, history: Optional[str],
+                  config: EvaluationConfig) -> Optional[str]:
     print(f'current id: {fragment_id}')
     # Tool does not work correctly with tmp files from <tempfile> module on macOS
     # thus we create a real file in the file system
@@ -107,6 +112,8 @@ def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataF
     if config.traceback:
         report[ColumnName.TRACEBACK.value] = []
     try:
+        if config.to_drop_nan:
+            lang_code_dataframe = lang_code_dataframe.dropna()
         lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply(
             lambda row: __inspect_row(row[ColumnName.LANG.value],
                                       row[ColumnName.CODE.value],

src/python/evaluation/inspectors/inspectors_stat/statistics_gathering.py

@@ -97,8 +97,7 @@ def print_stat(language: Language, stat: IssuesStat) -> None:
     print(f'Collected statistics for {language.value.lower()} language:')
     for issue_type, freq in stat.items():
         print(f'{issue_type}: {freq} times;')
-    print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories. '
-          f'Most likely it is {IssueType.INFO} category')
+    print(f'Note: {IssueType.UNDEFINED} means a category that is not categorized among the four main categories.')
 
 
 def __parse_language(language: str) -> Language:

src/python/evaluation/paper_evaluation/user_dynamics/README.md

@@ -10,20 +10,22 @@ Required arguments:
 
 `solutions_file_path` — path to csv-file with code samples.
 
-Optional arguments:
-Argument | Description
---- | ---
-|**‑fb**, **‑‑freq-boundary**| The boundary of solutions count for one student to analyze. The default value is 100.|
-|**‑n**, **‑‑n**| Top n popular issues in solutions. The default value is 100. |
+In the result a file with students issues dynamics will be created. 
+We have three categories of dynamics:
+- all (count of all code quality issues expect INFO issues)
+- formatting (count of formatting code quality issues from CODE_STYLE category)
+- other (all issues minus formatting issues)
 
-In the result a file with students issues dynamics will be created.
-Also, the top of issues for all students will be printed into the terminal. This statistics has key of issue and frequency for all students.
+Each type of dynamics will be saved into a separated folder with csv files for each student.
+Each csv file has only two columns: fragment id and issues count.
 
-An example of issues dynamics:
+An example of the csv file:
 ```text
-user,traceback
-0,"0,0,0,0,0,0,0,0,0,0,0,1,0,0,3,0,0,0,0,2,0,4,0,6,3,0,3,0,0,0,1,1,0,0,0,1,0,0,0,2,0,0,0,0,0,0,4,0,0,0,1,6,0,1,0,1,3,0,0,1,1,0,0,0,0,0,3,6,1,0,0,0,0,0,0,0,4,1,0,0,1,0,8,0,2,8,0,0,0,0,1,1,1,1,3,7,23,0,9"
-1,"0,0,0,3,0,0,2,1,0,0,0,0,4,1,0,0,1,1,0,0,0,0,0,6,0,1,1,0,8,1,2,1,1,0,0,1,0,4,10,1,1,1,3,0,1,0,0,0,1,0,0,0,0,0,0,2,0,3,0,0,2,2,3,2,0,0,0,1,0,1,1,0,0,1,0,4,6,2,0,0,1,0,0,0,0,2,0,0,0,2,1,2,1,0,1,7,1,0,1,1,0,1,0"
-```
-Each number in the traceback column is the count of issues in one solution. 
-The numbers of issues sorted by timestamps.
+issue_count,time
+2,0
+20,1
+16,2
+15,3
+5,4
+5,5
+```

src/python/evaluation/paper_evaluation/user_dynamics/dynamics_gathering.py

@@ -1,19 +1,18 @@
 import argparse
 import sys
-from collections import Counter
 from pathlib import Path
-from typing import Dict, List
+from typing import List
 
+import numpy as np
 import pandas as pd
 from src.python.common.tool_arguments import RunToolArgument
 from src.python.evaluation.common.csv_util import write_dataframe_to_csv
 from src.python.evaluation.common.pandas_util import (
-    drop_duplicates, filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger,
+    filter_df_by_single_value, get_issues_from_json, get_solutions_df, logger,
 )
 from src.python.evaluation.common.util import ColumnName
 from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
-from src.python.evaluation.paper_evaluation.comparison_with_other_tools.tutor_statistics import sort_freq_dict
-from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import UserStatistics
+from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn
 from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension
 from src.python.review.inspectors.issue import IssueType
 
@@ -23,58 +22,45 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None:
                         type=lambda value: Path(value).absolute(),
                         help=RunToolArgument.SOLUTIONS_FILE_PATH.value)
 
-    parser.add_argument('-fb', '--freq-boundary',
-                        help='The boundary of solutions count for one student to analyze',
-                        type=int,
-                        default=100)
-
-    parser.add_argument('-n', '--n',
-                        help='Top n popular issues in solutions',
-                        type=int,
-                        default=10)
-
-
-def __get_top_freq_issues(issues: List[List[PenaltyIssue]], n: int) -> Dict[str, int]:
-    all_issues = list(map(lambda i: i.origin_class, [item for sublist in issues for item in sublist]))
-    return dict(Counter(all_issues).most_common(n))
-
-
-# Get statistics only for users that have >= freq_boundary solutions in solutions_df
-# Statistics for each student has:
-#  - <traceback> - list of list of issues, but without INFO issues
-#  - <top_issues> - for each key of issue from <traceback> has frequency.
-#    Contains only top_n issues
-def __get_user_statistics(solutions_df: pd.DataFrame, freq_boundary: int = 100,
-                          top_n: int = 10) -> List[UserStatistics]:
-    stat = []
-    counts = solutions_df[ColumnName.USER.value].value_counts()
-    solutions_df = solutions_df[solutions_df[ColumnName.USER.value].isin(counts[counts > freq_boundary].index)]
-    for user in solutions_df[ColumnName.USER.value].unique():
-        user_df = filter_df_by_single_value(solutions_df,
-                                            ColumnName.USER.value, user).sort_values(ColumnName.TIME.value)
-        user_df = drop_duplicates(user_df)
-        traceback = list(map(lambda t: get_issues_from_json(t),
-                             list(user_df[ColumnName.TRACEBACK.value])))
-        # Filter info category
-        traceback = list(filter(lambda issues_list: filter(lambda i: i.type != IssueType.INFO, issues_list), traceback))
-        top_issues = __get_top_freq_issues(traceback, top_n)
-        stat.append(UserStatistics(traceback, top_issues))
-    return stat
+
+ALL_ISSUES_COUNT = DynamicsColumn.ALL_ISSUES_COUNT.value
+FORMATTING_ISSUES_COUNT = DynamicsColumn.FORMATTING_ISSUES_COUNT.value
+OTHER_ISSUES_COUNT = DynamicsColumn.OTHER_ISSUES_COUNT.value
 
 
-def __get_student_dynamics(stats: List[UserStatistics]) -> pd.DataFrame:
-    dynamics = map(lambda s: s.get_traceback_dynamics(), stats)
-    dynamics_dict = {i: ','.join(map(lambda d: str(d), dyn)) for (i, dyn) in enumerate(dynamics)}
-    return pd.DataFrame(dynamics_dict.items(), columns=[ColumnName.USER.value, ColumnName.TRACEBACK.value])
+def __get_all_issues(traceback: str) -> List[PenaltyIssue]:
+    return list(filter(lambda i: i.type != IssueType.INFO, get_issues_from_json(traceback)))
 
 
-def __get_total_top(stats: List[UserStatistics]) -> Dict[str, int]:
-    total_top_n = {}
-    for d in map(lambda s: s.top_issues, stats):
-        for k, v in d.items():
-            total_top_n.setdefault(k, 0)
-            total_top_n[k] += v
-    return sort_freq_dict(total_top_n)
+def __get_formatting_issues(traceback: str) -> List[PenaltyIssue]:
+    return list(filter(lambda i: i.type == IssueType.CODE_STYLE, __get_all_issues(traceback)))
+
+
+def __write_dynamics(output_path: Path, user_fragments: pd.DataFrame, index: int) -> None:
+    output_path.mkdir(parents=True, exist_ok=True)
+    user_fragments.columns = [DynamicsColumn.ISSUE_COUNT.value]
+    user_fragments[ColumnName.TIME.value] = np.arange(len(user_fragments))
+    write_dataframe_to_csv(output_path / f'user_{index}{Extension.CSV.value}', user_fragments)
+
+
+def __get_users_statistics(solutions_df: pd.DataFrame, output_path: Path) -> None:
+    users = solutions_df[ColumnName.USER.value].unique()
+    for index, user in enumerate(users):
+        user_df = filter_df_by_single_value(solutions_df,
+                                            ColumnName.USER.value, user).sort_values(ColumnName.TIME.value)
+        user_df[ALL_ISSUES_COUNT] = user_df.apply(lambda row:
+                                                  len(__get_all_issues(
+                                                      row[ColumnName.TRACEBACK.value])),
+                                                  axis=1)
+        user_df[FORMATTING_ISSUES_COUNT] = user_df.apply(lambda row:
+                                                         len(__get_formatting_issues(
+                                                             row[ColumnName.TRACEBACK.value])),
+                                                         axis=1)
+        user_df[OTHER_ISSUES_COUNT] = user_df[ALL_ISSUES_COUNT] - user_df[FORMATTING_ISSUES_COUNT]
+
+        __write_dynamics(output_path / 'all', user_df[[ALL_ISSUES_COUNT]], index)
+        __write_dynamics(output_path / 'formatting', user_df[[FORMATTING_ISSUES_COUNT]], index)
+        __write_dynamics(output_path / 'other', user_df[[OTHER_ISSUES_COUNT]], index)
 
 
 def main() -> int:
@@ -86,16 +72,10 @@ def main() -> int:
         solutions_file_path = args.solutions_file_path
         extension = get_restricted_extension(solutions_file_path, [Extension.CSV])
         solutions_df = get_solutions_df(extension, solutions_file_path)
-        solutions_df = filter_df_by_single_value(solutions_df, ColumnName.IS_PUBLIC.value, 'YES')
-        stats = __get_user_statistics(solutions_df, freq_boundary=args.freq_boundary, top_n=args.n)
-        dynamics = __get_student_dynamics(stats)
-        output_path = get_parent_folder(Path(solutions_file_path)) / f'student_issues_dynamics{Extension.CSV.value}'
-        write_dataframe_to_csv(output_path, dynamics)
-        print(f'The students dynamics was saved here: {output_path}')
-        total_top = __get_total_top(stats)
-        print('Total top issues:')
-        for i, (key, freq) in enumerate(total_top.items()):
-            print(f'{i}. {key} was found {freq} times')
+
+        output_path = get_parent_folder(Path(solutions_file_path)) / 'dynamics'
+        output_path.mkdir(parents=True, exist_ok=True)
+        __get_users_statistics(solutions_df, output_path)
         return 0
 
     except FileNotFoundError:

src/python/evaluation/paper_evaluation/user_dynamics/dynamics_visualization.py

@@ -0,0 +1,60 @@
+import argparse
+import sys
+from pathlib import Path
+from statistics import median
+
+import pandas as pd
+import plotly.graph_objects as go
+from src.python.evaluation.common.pandas_util import logger
+from src.python.evaluation.common.util import ColumnName
+from src.python.evaluation.paper_evaluation.user_dynamics.user_statistics import DynamicsColumn
+from src.python.review.common.file_system import Extension, extension_file_condition, get_all_file_system_items
+
+
+def configure_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument('dynamics_folder_path',
+                        type=lambda value: Path(value).absolute(),
+                        help='Add description here')
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    configure_arguments(parser)
+
+    try:
+        args = parser.parse_args()
+        dynamics_folder_path = args.dynamics_folder_path
+        dynamics_paths = get_all_file_system_items(dynamics_folder_path, extension_file_condition(Extension.CSV))
+        dynamic_fig = go.Figure()
+
+        medians = []
+        for i, dynamic in enumerate(dynamics_paths):
+            dynamic_df = pd.read_csv(dynamic)
+
+            dynamic_df = dynamic_df.head(100)
+            dynamic_fig.add_trace(go.Scatter(
+                x=dynamic_df[ColumnName.TIME.value],
+                y=dynamic_df[DynamicsColumn.ISSUE_COUNT.value],
+                name=f'user {i}',
+            ))
+            medians.append(median(dynamic_df[DynamicsColumn.ISSUE_COUNT.value]))
+
+        dynamic_fig.update_layout(title='Code quality issues dynamics for Python',
+                                  xaxis_title='Submission number',
+                                  yaxis_title='Code quality issues count')
+        dynamic_fig.show()
+
+        medians = go.Figure(data=go.Scatter(x=list(range(len(medians))), y=medians))
+        medians.update_layout(title='Median values for code quality issues dynamics for Python',
+                              xaxis_title='Student number',
+                              yaxis_title='Median of code quality issues count')
+        medians.show()
+        return 0
+
+    except Exception:
+        logger.exception('An unexpected error.')
+        return 2
+
+
+if __name__ == '__main__':
+    sys.exit(main())

src/python/evaluation/paper_evaluation/user_dynamics/unpack_solutions.py

@@ -0,0 +1,72 @@
+import argparse
+import sys
+import uuid
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+from src.python.common.tool_arguments import RunToolArgument
+from src.python.evaluation.common.csv_util import write_dataframe_to_csv
+from src.python.evaluation.common.pandas_util import get_solutions_df, logger
+from src.python.evaluation.common.util import ColumnName
+from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension
+
+
+'''
+This scripts allows unpacking solutions to the solutions dataframe.
+The initial dataframe has only several obligatory columns user_id,times,codes,
+where <times> is an array with times separated by ; symbol and
+<codes> is an array with code fragments separated by ₣ symbol.
+The <times> and <codes> arrays have to has the same length.
+The resulting dataset will have several: columns user_id,time,code,
+where each row contains obly one time and one code fragment
+'''
+
+
+def configure_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(RunToolArgument.SOLUTIONS_FILE_PATH.value.long_name,
+                        type=lambda value: Path(value).absolute(),
+                        help='Path to the compressed solutions')
+
+
+def __parse_time_and_solutions(times_str: str, solutions_str: str) -> pd.DataFrame:
+    times = times_str.split(',')
+    solutions = solutions_str.split('₣')
+    time_to_solution = dict(zip(times, solutions))
+    user_df = pd.DataFrame(time_to_solution.items(), columns=[ColumnName.TIME.value, ColumnName.CODE.value])
+    user_df[ColumnName.USER.value] = uuid.uuid4()
+    return user_df
+
+
+def __add_user_df(user_df_list: List[pd.DataFrame], user_df: pd.DataFrame):
+    user_df_list.append(user_df)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    configure_arguments(parser)
+
+    try:
+        args = parser.parse_args()
+        solutions_file_path = args.solutions_file_path
+        extension = get_restricted_extension(solutions_file_path, [Extension.CSV])
+        solutions_df = get_solutions_df(extension, solutions_file_path)
+        user_df_list = []
+        solutions_df.apply(lambda row: __add_user_df(user_df_list,
+                                                     __parse_time_and_solutions(row['times'], row['codes'])), axis=1)
+        unpacked_solutions = pd.concat(user_df_list)
+        output_path = get_parent_folder(Path(solutions_file_path)) / f'unpacked_solutions{Extension.CSV.value}'
+        write_dataframe_to_csv(output_path, unpacked_solutions)
+        return 0
+
+    except FileNotFoundError:
+        logger.error('CSV-file with the specified name does not exists.')
+        return 2
+
+    except Exception:
+        logger.exception('An unexpected error.')
+        return 2
+
+
+if __name__ == '__main__':
+    sys.exit(main())

src/python/evaluation/paper_evaluation/user_dynamics/user_statistics.py

@@ -1,9 +1,19 @@
 from dataclasses import dataclass
+from enum import Enum, unique
 from typing import Dict, List
 
 from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 
 
+@unique
+class DynamicsColumn(Enum):
+    ALL_ISSUES_COUNT = 'all_issues_count'
+    FORMATTING_ISSUES_COUNT = 'formatting_issues_count'
+    OTHER_ISSUES_COUNT = 'other_issues_count'
+
+    ISSUE_COUNT = 'issue_count'
+
+
 @dataclass
 class UserStatistics:
     traceback: List[List[PenaltyIssue]]

test/python/evaluation/testing_config.py

@@ -20,5 +20,6 @@ def get_testing_arguments(to_add_traceback=None, to_add_tool_path=None, to_add_h
         testing_arguments.with_history = True
 
     testing_arguments.solutions_file_path = None
+    testing_arguments.to_drop_nan = False
 
     return testing_arguments