Add penalty statistics

src/python/evaluation/common/pandas_util.py

@@ -8,9 +8,9 @@
 from src.python.evaluation.common.csv_util import write_dataframe_to_csv
 from src.python.evaluation.common.util import ColumnName, EvaluationArgument
 from src.python.evaluation.common.xlsx_util import create_workbook, remove_sheet, write_dataframe_to_xlsx_sheet
+from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 from src.python.review.application_config import LanguageVersion
 from src.python.review.common.file_system import Extension, get_restricted_extension
-from src.python.review.inspectors.issue import BaseIssue
 from src.python.review.reviewers.utils.print_review import convert_json_to_issues
 
 logger = logging.getLogger(__name__)
@@ -94,10 +94,10 @@ def write_df_to_file(df: pd.DataFrame, output_file_path: Path, extension: Extens
         remove_sheet(output_file_path, 'Sheet')
 
 
-def get_issues_from_json(str_json: str) -> List[BaseIssue]:
+def get_issues_from_json(str_json: str) -> List[PenaltyIssue]:
     parsed_json = json.loads(str_json)['issues']
     return convert_json_to_issues(parsed_json)
 
 
-def get_issues_by_row(df: pd.DataFrame, row: int) -> List[BaseIssue]:
+def get_issues_by_row(df: pd.DataFrame, row: int) -> List[PenaltyIssue]:
     return get_issues_from_json(df.iloc[row][EvaluationArgument.TRACEBACK.value])

src/python/evaluation/common/util.py

@@ -16,6 +16,10 @@ class ColumnName(Enum):
     OLD = 'old'
     NEW = 'new'
     IS_PUBLIC = 'is_public'
+    DECREASED_GRADE = 'decreased_grade'
+    PENALTY = 'penalty'
+    USER = 'user'
+    HISTORY = 'history'
 
 
 @unique

src/python/evaluation/inspectors/README.md

@@ -114,9 +114,11 @@ An example of the pickle` file is:
 ```json
 {
     grade: [2, 3],
+    decreased_grade: [1],
+    user: 2,
     traceback: {
         1: {
-            BaseIssue(
+            PenaltyIssue(
                 origin_class='C0305',
                 description='Trailing newlines',
                 line_no=15,
@@ -125,7 +127,8 @@ An example of the pickle` file is:
 
                 file_path=Path(),
                 inspector_type=InspectorType.UNDEFINED,
-            ), BaseIssue(
+                influence_on_penalty=0,
+            ), PenaltyIssue(
                 origin_class='E211',
                 description='whitespace before \'(\'',
                 line_no=1,
@@ -134,13 +137,32 @@ An example of the pickle` file is:
 
                 file_path=Path(),
                 inspector_type=InspectorType.UNDEFINED,
+                influence_on_penalty=0.6,
             ),
           }
     },
+    penalty: {
+          1: {
+              PenaltyIssue(
+                  origin_class='E211',
+                  description='whitespace before \'(\'',
+                  line_no=1,
+                  column_no=6,
+                  type=IssueType('CODE_STYLE'),
+
+                  file_path=Path(),
+                  inspector_type=InspectorType.UNDEFINED,
+                  influence_on_penalty=0.6,
+              ),
+            }
+      }
 }
 ```
 In the `grade` field are stored fragments ids for which grade was increased in the new data.
+In the `decreased_grade` field are stored fragments ids for which grade was decreased in the new data.
+In the `user` field are stored count unique users in the new dataset.
 In the `traceback` field for fragments ids are stored set of issues. These issues were found in the new data and were not found in the old data.
+In the `penalty` field for fragments ids are stored set of issues. These issues have not zero `influence_on_penalty` coefficient.
 
 ___
 
@@ -168,21 +190,28 @@ The statistics will be printed into console.
 
 The output contains:
 - was found incorrect grades or not;
-- how many fragments has additional issues;
-- how many unique issues was found;
-- top N issues in the format: (issue_key, frequency);
-- short categorized statistics: for each category how many issues were found and how many 
-  fragments have these issues;
-- \[Optional\] full categorized statistics: for each category for each issue how many 
-  fragments have this issue
+- how many grades have decreased value;
+- how many unique users was found in the new dataset;
+- for new issues and for penalty statistics:
+  - how many fragments has additional issues;
+  - how many unique issues was found;
+  - top N issues in the format: (issue_key, frequency);
+  - short categorized statistics: for each category how many issues were found and how many 
+    fragments have these issues;
+  - \[Optional\] full categorized statistics: for each category for each issue how many 
+    fragments have this issue
+- for each category base influence on the penalty statistics: min, max and median values
 
 An example of the printed statistics (without full categorized statistics):
 
 ```json
 SUCCESS! Was not found incorrect grades.
+All grades are equal.
 ______
+NEW INSPECTIONS STATISTICS:
 39830 fragments has additional issues
 139 unique issues was found
+4671 unique users was found!        
 ______
 Top 10 issues:
 SC200: 64435 times
@@ -202,6 +231,20 @@ ERROR_PRONE: 17 issues, 2363 fragments
 COMPLEXITY: 17 issues, 13928 fragments
 COHESION: 1 issues, 3826 fragments
 ______
+______
+PENALTY INSPECTIONS STATISTICS;
+Statistics is empty!
+______
+______
+INFLUENCE ON PENALTY STATISTICS;
+CODE_STYLE issues: min=1, max=100, median=86
+BEST_PRACTICES issues: min=1, max=100, median=98.0
+COMPLEXITY issues: min=1, max=100, median=16.0
+MAINTAINABILITY issues: min=1, max=7, median=2.0
+CYCLOMATIC_COMPLEXITY issues: min=1, max=58, median=11.5
+COHESION issues: min=1, max=100, median=56
+BOOL_EXPR_LEN issues: min=6, max=6, median=6
+______
 ```
 
 ---

src/python/evaluation/inspectors/common/statistics.py

@@ -1,16 +1,45 @@
 from collections import defaultdict
 from dataclasses import dataclass
+from statistics import median
 from typing import Dict, List, Tuple
 
-from src.python.review.inspectors.issue import IssueType, ShortIssue
+from src.python.review.inspectors.issue import BaseIssue, IssueType, ShortIssue
+
+
+@dataclass(frozen=True, eq=True)
+class PenaltyIssue(BaseIssue):
+    influence_on_penalty: int
 
 
 @dataclass(frozen=True)
 class IssuesStatistics:
     stat: Dict[ShortIssue, int]
-    changed_grades_count: int
+    fragments_in_stat: int
+
+    def print_full_statistics(self, n: int, full_stat: bool, separator: str = '') -> None:
+        if self.fragments_in_stat == 0:
+            print('Statistics is empty!')
+            return
+
+        print(f'{self.fragments_in_stat} fragments has additional issues')
+        print(f'{self.count_unique_issues()} unique issues was found')
+
+        self.print_top_n(n, separator)
+        self.print_short_categorized_statistics()
+        print(separator)
+
+        if full_stat:
+            self.print_full_inspectors_statistics()
+
+    def print_top_n(self, n: int, separator: str) -> None:
+        top_n = self.get_top_n_issues(n)
+        print(separator)
+        print(f'Top {n} issues:')
+        for issue, freq in top_n:
+            IssuesStatistics.print_issue_with_freq(issue, freq)
+        print(separator)
 
-    def print_full_statistics(self, to_categorize: bool = True):
+    def print_full_inspectors_statistics(self, to_categorize: bool = True) -> None:
         if to_categorize:
             categorized_statistics: Dict[IssueType, Dict[ShortIssue, int]] = self.get_categorized_statistics()
             for category, issues in categorized_statistics.items():
@@ -20,7 +49,7 @@ def print_full_statistics(self, to_categorize: bool = True):
             self.__print_stat(self.stat)
 
     @classmethod
-    def __print_stat(cls, stat: Dict[ShortIssue, int]):
+    def __print_stat(cls, stat: Dict[ShortIssue, int]) -> None:
         for issue, freq in stat.items():
             cls.print_issue_with_freq(issue, freq, prefix='- ')
 
@@ -54,3 +83,26 @@ def get_top_n_issues(self, n: int) -> List[ShortIssue]:
 
     def count_unique_issues(self) -> int:
         return len(self.stat)
+
+
+# Store list of penalty influences for each category
+@dataclass
+class PenaltyInfluenceStatistics:
+    stat: Dict[IssueType, List[float]]
+
+    def __init__(self, issues_stat_dict: Dict[int, List[PenaltyIssue]]):
+        self.stat = defaultdict(list)
+        for _, issues in issues_stat_dict.items():
+            for issue in issues:
+                self.stat[issue.type].append(issue.influence_on_penalty)
+
+    def print_stat(self):
+        for category, issues in self.stat.items():
+            print(f'{category.value} issues: min={min(issues)}, max={max(issues)}, median={median(issues)}')
+
+
+@dataclass(frozen=True)
+class GeneralInspectorsStatistics:
+    new_issues_stat: IssuesStatistics
+    penalty_issues_stat: IssuesStatistics
+    penalty_influence_stat: PenaltyInfluenceStatistics

src/python/evaluation/inspectors/diffs_between_df.py

@@ -30,18 +30,35 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None:
 # Find difference between two dataframes. Return dict:
 # {
 #  grade: [list_of_fragment_ids],
+#  decreased_grade: [list_of_fragment_ids],
+#  user: count_unique_users,
 #  traceback: {
 #       fragment_id: [list of issues]
 #     },
+#  penalty: {
+#       fragment_id: [list of issues]
+#     },
 # }
 # The key <grade> contains only fragments that increase quality in new df
+# The key <decreased_grade> contains only fragments that decrease quality in new df
+# The key <user> count number of unique users in the new dataset
 # The key <traceback> contains list of new issues for each fragment
+# The key <penalty> contains list of issues with not zero influence_on_penalty coefficient
 def find_diffs(old_df: pd.DataFrame, new_df: pd.DataFrame) -> dict:
+    if ColumnName.HISTORY.value in new_df.columns:
+        del new_df[ColumnName.HISTORY.value]
+    new_df = new_df.reindex(columns=old_df.columns)
     inconsistent_positions = get_inconsistent_positions(old_df, new_df)
     diffs = {
         ColumnName.GRADE.value: [],
+        ColumnName.DECREASED_GRADE.value: [],
         EvaluationArgument.TRACEBACK.value: {},
+        ColumnName.PENALTY.value: {},
     }
+    if ColumnName.USER.value in new_df.columns:
+        diffs[ColumnName.USER.value] = len(new_df[ColumnName.USER.value].unique())
+    else:
+        diffs[ColumnName.USER.value] = 0
     # Keep only diffs in the TRACEBACK column
     for row, _ in filter(lambda t: t[1] == EvaluationArgument.TRACEBACK.value, inconsistent_positions.index):
         old_value = old_df.iloc[row][ColumnName.GRADE.value]
@@ -53,13 +70,21 @@ def find_diffs(old_df: pd.DataFrame, new_df: pd.DataFrame) -> dict:
             # It is an unexpected keys, we should check the algorithm
             diffs[ColumnName.GRADE.value].append(fragment_id)
         else:
-            # Find difference between issues
+            if new_quality < old_quality:
+                diffs[ColumnName.DECREASED_GRADE.value].append(fragment_id)
             old_issues = get_issues_by_row(old_df, row)
             new_issues = get_issues_by_row(new_df, row)
+            # Find difference between issues
             if len(old_issues) > len(new_issues):
                 raise ValueError(f'New dataframe contains less issues than old for fragment {id}')
             difference = set(set(new_issues) - set(old_issues))
-            diffs[EvaluationArgument.TRACEBACK.value][fragment_id] = difference
+            if len(difference) > 0:
+                diffs[EvaluationArgument.TRACEBACK.value][fragment_id] = difference
+
+            # Find issues with influence_in_penalty > 0
+            penalty = set(filter(lambda i: i.influence_on_penalty > 0, new_issues))
+            if len(penalty) > 0:
+                diffs[ColumnName.PENALTY.value][fragment_id] = penalty
     return diffs
 
 

src/python/evaluation/inspectors/filter_issues.py

@@ -6,8 +6,8 @@
 from src.python.common.tool_arguments import RunToolArgument
 from src.python.evaluation.common.pandas_util import get_issues_from_json, get_solutions_df_by_file_path
 from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 from src.python.review.common.file_system import Extension, get_parent_folder, serialize_data_and_write_to_file
-from src.python.review.inspectors.issue import BaseIssue
 
 
 TRACEBACK = EvaluationArgument.TRACEBACK.value
@@ -30,12 +30,12 @@ def __parse_issues_arg(str_issues: str) -> Set[str]:
     return set(str_issues.split(','))
 
 
-def __get_new_issues(traceback: str, new_issues_classes: Set[str]) -> List[BaseIssue]:
+def __get_new_issues(traceback: str, new_issues_classes: Set[str]) -> List[PenaltyIssue]:
     all_issues = get_issues_from_json(traceback)
     return list(filter(lambda i: i.origin_class in new_issues_classes, all_issues))
 
 
-def __add_issues_for_fragment(fragment_id: int, new_issues: List[BaseIssue], diffs: dict) -> None:
+def __add_issues_for_fragment(fragment_id: int, new_issues: List[PenaltyIssue], diffs: dict) -> None:
     if len(new_issues) > 0:
         diffs[TRACEBACK][fragment_id] = new_issues
 

src/python/evaluation/inspectors/get_worse_public_examples.py

@@ -7,8 +7,8 @@
 from src.python.evaluation.common.csv_util import write_dataframe_to_csv
 from src.python.evaluation.common.pandas_util import filter_df_by_condition, get_solutions_df_by_file_path
 from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 from src.python.review.common.file_system import deserialize_data_from_file, Extension, get_parent_folder
-from src.python.review.inspectors.issue import BaseIssue
 
 
 def configure_arguments(parser: argparse.ArgumentParser) -> None:
@@ -26,7 +26,7 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None:
                         default=10)
 
 
-def __get_new_inspections(fragment_id_to_issues: Dict[int, List[BaseIssue]], fragment_id: int) -> str:
+def __get_new_inspections(fragment_id_to_issues: Dict[int, List[PenaltyIssue]], fragment_id: int) -> str:
     return ','.join(set(map(lambda i: i.origin_class, fragment_id_to_issues.get(fragment_id, []))))