Add support for branch coverage

analysis/benchmark_results.py

@@ -95,30 +95,30 @@ def _benchmark_snapshot_df(self):
     @property
     @functools.lru_cache()
     def _benchmark_coverage_dict(self):
-        """Covered regions of each fuzzer on this benchmark."""
+        """Covered branches of each fuzzer on this benchmark."""
         return coverage_data_utils.get_benchmark_cov_dict(
             self._coverage_dict, self.name)
 
     @property
     @functools.lru_cache()
     def _benchmark_aggregated_coverage_df(self):
-        """Aggregated covered regions of each fuzzer on this benchmark."""
+        """Aggregated covered branches of each fuzzer on this benchmark."""
         return coverage_data_utils.get_benchmark_aggregated_cov_df(
             self._coverage_dict, self.name)
 
     @property
     @functools.lru_cache()
-    def _unique_region_dict(self):
-        """Unique regions with the fuzzers that cover it."""
-        return coverage_data_utils.get_unique_region_dict(
+    def _unique_branch_dict(self):
+        """Unique branches with the fuzzers that cover it."""
+        return coverage_data_utils.get_unique_branch_dict(
             self._benchmark_coverage_dict)
 
     @property
     @functools.lru_cache()
-    def unique_region_cov_df(self):
-        """Fuzzers with the number of covered unique regions."""
-        return coverage_data_utils.get_unique_region_cov_df(
-            self._unique_region_dict, self.fuzzer_names)
+    def unique_branch_cov_df(self):
+        """Fuzzers with the number of covered unique branches."""
+        return coverage_data_utils.get_unique_branch_cov_df(
+            self._unique_branch_dict, self.fuzzer_names)
 
     @property
     def fuzzers_with_not_enough_samples(self):
@@ -344,12 +344,12 @@ def _generic_violin_plot(self, filename, bugs=False):
 
     @property
     def violin_plot(self):
-        """Region coverage violin plot."""
+        """Branch coverage violin plot."""
         return self._generic_violin_plot('violin.svg')
 
     @property
     def bug_violin_plot(self):
-        """Region coverage violin plot."""
+        """Branch coverage violin plot."""
         return self._generic_violin_plot('bug_violin.svg', bugs=True)
 
     def _generic_box_plot(self, filename, bugs=False):
@@ -362,7 +362,7 @@ def _generic_box_plot(self, filename, bugs=False):
 
     @property
     def box_plot(self):
-        """Region coverage boxplot."""
+        """Branch coverage boxplot."""
         return self._generic_box_plot('boxplot.svg')
 
     @property
@@ -399,19 +399,19 @@ def better_than_plot(self):
     @property
     def unique_coverage_ranking_plot(self):
         """Ranking plot for unique coverage."""
-        plot_filename = self._prefix_with_benchmark('ranking_unique_region.svg')
-        unique_region_cov_df_combined = self.unique_region_cov_df.merge(
+        plot_filename = self._prefix_with_benchmark('ranking_unique_branch.svg')
+        unique_branch_cov_df_combined = self.unique_branch_cov_df.merge(
             self._benchmark_aggregated_coverage_df, on='fuzzer')
         self._plotter.write_unique_coverage_ranking_plot(
-            unique_region_cov_df_combined, self._get_full_path(plot_filename))
+            unique_branch_cov_df_combined, self._get_full_path(plot_filename))
         return plot_filename
 
     @property
     @functools.lru_cache()
     def pairwise_unique_coverage_table(self):
         """Pairwise unique coverage table for each pair of fuzzers."""
-        fuzzers = self.unique_region_cov_df.sort_values(
-            by='unique_regions_covered', ascending=False).fuzzer
+        fuzzers = self.unique_branch_cov_df.sort_values(
+            by='unique_branches_covered', ascending=False).fuzzer
         return coverage_data_utils.get_pairwise_unique_coverage_table(
             self._benchmark_coverage_dict, fuzzers)
 

analysis/coverage_data_utils.py

@@ -74,109 +74,109 @@ def get_coverage_report_filestore_path(fuzzer: str, benchmark: str,
                           fuzzer, 'index.html')
 
 
-def get_covered_regions_dict(experiment_df: pd.DataFrame) -> Dict:
+def get_covered_branches_dict(experiment_df: pd.DataFrame) -> Dict:
     """Combines json files for different fuzzer-benchmark pair in
-    |experiment_df| and returns a dictionary of the covered regions."""
+    |experiment_df| and returns a dictionary of the covered branches."""
     fuzzers_and_benchmarks = set(
         zip(experiment_df.fuzzer, experiment_df.benchmark))
     arguments = [(fuzzer, benchmark,
                   get_experiment_filestore_path_for_fuzzer_benchmark(
                       fuzzer, benchmark, experiment_df))
                  for fuzzer, benchmark in fuzzers_and_benchmarks]
-    result = itertools.starmap(get_fuzzer_benchmark_covered_regions_and_key,
+    result = itertools.starmap(get_fuzzer_benchmark_covered_branches_and_key,
                                arguments)
     return dict(result)
 
 
-def get_fuzzer_benchmark_covered_regions_filestore_path(
+def get_fuzzer_benchmark_covered_branches_filestore_path(
         fuzzer: str, benchmark: str, exp_filestore_path: str) -> str:
-    """Returns the path to the covered regions json file in the |filestore| for
+    """Returns the path to the covered branches json file in the |filestore| for
     |fuzzer| and |benchmark|."""
     return posixpath.join(exp_filestore_path, 'coverage', 'data', benchmark,
-                          fuzzer, 'covered_regions.json')
+                          fuzzer, 'covered_branches.json')
 
 
-def get_fuzzer_covered_regions(fuzzer: str, benchmark: str, filestore: str):
-    """Returns the covered regions dict for |fuzzer| from the json file in the
+def get_fuzzer_covered_branches(fuzzer: str, benchmark: str, filestore: str):
+    """Returns the covered branches dict for |fuzzer| from the json file in the
     filestore."""
-    src_file = get_fuzzer_benchmark_covered_regions_filestore_path(
+    src_file = get_fuzzer_benchmark_covered_branches_filestore_path(
         fuzzer, benchmark, filestore)
     with tempfile.NamedTemporaryFile() as dst_file:
         if filestore_utils.cp(src_file, dst_file.name,
                               expect_zero=False).retcode:
-            logger.warning('covered_regions.json file: %s could not be copied.',
-                           src_file)
+            logger.warning(
+                'covered_branches.json file: %s could not be copied.', src_file)
             return {}
         with open(dst_file.name) as json_file:
             return json.load(json_file)
 
 
-def get_fuzzer_benchmark_covered_regions_and_key(
+def get_fuzzer_benchmark_covered_branches_and_key(
         fuzzer: str, benchmark: str, filestore: str) -> Tuple[str, Dict]:
     """Accepts |fuzzer|, |benchmark|, |filestore|.
-    Returns a tuple containing the fuzzer benchmark key and the regions covered
+    Returns a tuple containing the fuzzer benchmark key and the branches covered
     by the fuzzer on the benchmark."""
-    fuzzer_benchmark_covered_regions = get_fuzzer_covered_regions(
+    fuzzer_benchmark_covered_branches = get_fuzzer_covered_branches(
         fuzzer, benchmark, filestore)
     key = fuzzer_and_benchmark_to_key(fuzzer, benchmark)
-    return key, fuzzer_benchmark_covered_regions
+    return key, fuzzer_benchmark_covered_branches
 
 
-def get_unique_region_dict(benchmark_coverage_dict: Dict) -> Dict:
+def get_unique_branch_dict(benchmark_coverage_dict: Dict) -> Dict:
     """Returns a dictionary containing the covering fuzzers for each unique
-    region, where the |threshold| defines which regions are unique."""
-    region_dict = collections.defaultdict(list)
-    unique_region_dict = {}
+    branch, where the |threshold| defines which branches are unique."""
+    branch_dict = collections.defaultdict(list)
+    unique_branch_dict = {}
     threshold_count = 1
     for fuzzer in benchmark_coverage_dict:
-        for region in benchmark_coverage_dict[fuzzer]:
-            region_dict[region].append(fuzzer)
-    for region, fuzzers in region_dict.items():
+        for branch in benchmark_coverage_dict[fuzzer]:
+            branch_dict[branch].append(fuzzer)
+    for branch, fuzzers in branch_dict.items():
         if len(fuzzers) <= threshold_count:
-            unique_region_dict[region] = fuzzers
-    return unique_region_dict
+            unique_branch_dict[branch] = fuzzers
+    return unique_branch_dict
 
 
-def get_unique_region_cov_df(unique_region_dict: Dict,
+def get_unique_branch_cov_df(unique_branch_dict: Dict,
                              fuzzer_names: List[str]) -> pd.DataFrame:
     """Returns a DataFrame where the two columns are fuzzers and the number of
-    unique regions covered."""
+    unique branches covered."""
     fuzzers = collections.defaultdict(int)
-    for region in unique_region_dict:
-        for fuzzer in unique_region_dict[region]:
+    for branch in unique_branch_dict:
+        for fuzzer in unique_branch_dict[branch]:
             fuzzers[fuzzer] += 1
-    dict_to_transform = {'fuzzer': [], 'unique_regions_covered': []}
+    dict_to_transform = {'fuzzer': [], 'unique_branches_covered': []}
     for fuzzer in fuzzer_names:
         covered_num = fuzzers[fuzzer]
         dict_to_transform['fuzzer'].append(fuzzer)
-        dict_to_transform['unique_regions_covered'].append(covered_num)
+        dict_to_transform['unique_branches_covered'].append(covered_num)
     return pd.DataFrame(dict_to_transform)
 
 
 def get_benchmark_cov_dict(coverage_dict, benchmark):
-    """Returns a dictionary to store the covered regions of each fuzzer. Uses a
-    set of tuples to store the covered regions."""
+    """Returns a dictionary to store the covered branches of each fuzzer. Uses a
+    set of tuples to store the covered branches."""
     benchmark_cov_dict = {}
-    for key, covered_regions in coverage_dict.items():
+    for key, covered_braches in coverage_dict.items():
         current_fuzzer, current_benchmark = key_to_fuzzer_and_benchmark(key)
         if current_benchmark == benchmark:
-            covered_regions_in_set = set()
-            for region in covered_regions:
-                covered_regions_in_set.add(tuple(region))
-            benchmark_cov_dict[current_fuzzer] = covered_regions_in_set
+            covered_braches_in_set = set()
+            for branch in covered_braches:
+                covered_braches_in_set.add(tuple(branch))
+            benchmark_cov_dict[current_fuzzer] = covered_braches_in_set
     return benchmark_cov_dict
 
 
 def get_benchmark_aggregated_cov_df(coverage_dict, benchmark):
     """Returns a dataframe where each row represents a fuzzer and its aggregated
     coverage number."""
     dict_to_transform = {'fuzzer': [], 'aggregated_edges_covered': []}
-    for key, covered_regions in coverage_dict.items():
+    for key, covered_branches in coverage_dict.items():
         current_fuzzer, current_benchmark = key_to_fuzzer_and_benchmark(key)
         if current_benchmark == benchmark:
             dict_to_transform['fuzzer'].append(current_fuzzer)
             dict_to_transform['aggregated_edges_covered'].append(
-                len(covered_regions))
+                len(covered_branches))
     return pd.DataFrame(dict_to_transform)
 
 
@@ -186,7 +186,7 @@ def get_pairwise_unique_coverage_table(benchmark_coverage_dict, fuzzers):
 
     The pairwise unique coverage table is a square matrix where each
     row and column represents a fuzzer, and each cell contains a number
-    showing the regions covered by the fuzzer of the column but not by
+    showing the branches covered by the fuzzer of the column but not by
     the fuzzer of the row."""
 
     pairwise_unique_coverage_values = []
@@ -204,16 +204,16 @@ def get_pairwise_unique_coverage_table(benchmark_coverage_dict, fuzzers):
                         columns=fuzzers)
 
 
-def get_unique_covered_percentage(fuzzer_row_covered_regions,
-                                  fuzzer_col_covered_regions):
-    """Returns the number of regions covered by the fuzzer of the column but not
-    by the fuzzer of the row."""
+def get_unique_covered_percentage(fuzzer_row_covered_branches,
+                                  fuzzer_col_covered_branches):
+    """Returns the number of branches covered by the fuzzer of the
+    column but not by the fuzzer of the row."""
 
-    unique_region_count = 0
-    for region in fuzzer_col_covered_regions:
-        if region not in fuzzer_row_covered_regions:
-            unique_region_count += 1
-    return unique_region_count
+    unique_branch_count = 0
+    for branch in fuzzer_col_covered_branches:
+        if branch not in fuzzer_row_covered_branches:
+            unique_branch_count += 1
+    return unique_branch_count
 
 
 def rank_by_average_normalized_score(benchmarks_unique_coverage_list):

analysis/experiment_results.py

@@ -264,7 +264,7 @@ def rank_by_unique_coverage_average_normalized_score(self):
         """Rank fuzzers using average normalized score on unique code coverage
         across benchmarks."""
         benchmarks_unique_coverage_list = [
-            benchmark.unique_region_cov_df for benchmark in self.benchmarks
+            benchmark.unique_branch_cov_df for benchmark in self.benchmarks
         ]
         return coverage_data_utils.rank_by_average_normalized_score(
             benchmarks_unique_coverage_list)

analysis/generate_report.py

@@ -226,7 +226,7 @@ def generate_report(experiment_names,
     coverage_dict = {}
     if coverage_report:
         logger.info('Generating coverage report info.')
-        coverage_dict = coverage_data_utils.get_covered_regions_dict(
+        coverage_dict = coverage_data_utils.get_covered_branches_dict(
             experiment_df)
         logger.info('Finished generating coverage report info.')
 

analysis/plotting.py

@@ -181,7 +181,7 @@ def coverage_growth_plot(self,
                     loc='upper left',
                     frameon=False)
 
-        axes.set(ylabel='Bug coverage' if bugs else 'Code region coverage')
+        axes.set(ylabel='Bug coverage' if bugs else 'Code branch coverage')
         axes.set(xlabel='Time (hour:minute)')
 
         if self._logscale or logscale:
@@ -260,7 +260,7 @@ def box_or_violin_plot(self,
             sns.stripplot(**common_args, size=3, color="black", alpha=0.6)
 
         axes.set_title(_formatted_title(benchmark_snapshot_df))
-        ylabel = 'Reached {} coverage'.format('bug' if bugs else 'region')
+        ylabel = 'Reached {} coverage'.format('bug' if bugs else 'branch')
         axes.set(ylabel=ylabel)
         axes.set(xlabel='Fuzzer (highest median coverage on the left)')
         axes.set_xticklabels(axes.get_xticklabels(),
@@ -307,7 +307,7 @@ def distribution_plot(self, benchmark_snapshot_df, axes=None, bugs=False):
         axes.set_title(_formatted_title(benchmark_snapshot_df))
         axes.legend(loc='upper right', frameon=False)
 
-        axes.set(xlabel='Bug coverage' if bugs else 'Code region coverage')
+        axes.set(xlabel='Bug coverage' if bugs else 'Code branch coverage')
         axes.set(ylabel='Density')
         axes.set_xticklabels(axes.get_xticklabels(),
                              rotation=_DEFAULT_LABEL_ROTATION,
@@ -339,7 +339,7 @@ def ranking_plot(self, benchmark_snapshot_df, axes=None, bugs=False):
                            ax=axes)
 
         axes.set_title(_formatted_title(benchmark_snapshot_df))
-        ylabel = 'Reached {} coverage'.format('bug' if bugs else 'region')
+        ylabel = 'Reached {} coverage'.format('bug' if bugs else 'branch')
         axes.set(ylabel=ylabel)
         axes.set(xlabel='Fuzzer (highest median coverage on the left)')
         axes.set_xticklabels(axes.get_xticklabels(),
@@ -491,17 +491,17 @@ def write_critical_difference_plot(self, average_ranks, num_of_benchmarks,
             plt.close(fig)
 
     def unique_coverage_ranking_plot(self,
-                                     unique_region_cov_df_combined,
+                                     unique_branch_cov_df_combined,
                                      axes=None):
         """Draws unique_coverage_ranking plot. The fuzzer labels will be in
         the order of their coverage."""
 
-        fuzzer_order = unique_region_cov_df_combined.sort_values(
-            by='unique_regions_covered', ascending=False).fuzzer
+        fuzzer_order = unique_branch_cov_df_combined.sort_values(
+            by='unique_branches_covered', ascending=False).fuzzer
 
-        axes = sns.barplot(y='unique_regions_covered',
+        axes = sns.barplot(y='unique_branches_covered',
                            x='fuzzer',
-                           data=unique_region_cov_df_combined,
+                           data=unique_branch_cov_df_combined,
                            order=fuzzer_order,
                            palette=self._fuzzer_colors,
                            ax=axes)
@@ -517,7 +517,7 @@ def unique_coverage_ranking_plot(self,
 
         sns.barplot(y='aggregated_edges_covered',
                     x='fuzzer',
-                    data=unique_region_cov_df_combined,
+                    data=unique_branch_cov_df_combined,
                     order=fuzzer_order,
                     facecolor=(1, 1, 1, 0),
                     edgecolor='0.2',
@@ -531,11 +531,11 @@ def unique_coverage_ranking_plot(self,
 
         sns.despine(ax=axes, trim=True)
 
-    def write_unique_coverage_ranking_plot(self, unique_region_cov_df_combined,
+    def write_unique_coverage_ranking_plot(self, unique_branch_cov_df_combined,
                                            image_path):
         """Writes ranking plot for unique coverage."""
         self._write_plot_to_image(self.unique_coverage_ranking_plot,
-                                  unique_region_cov_df_combined,
+                                  unique_branch_cov_df_combined,
                                   image_path,
                                   wide=True)
 

analysis/report_templates/default.html

@@ -385,12 +385,12 @@ <h5 class="center-align">Mann-Whitney U test</h4>
 
                         <div class="row">
                             <div class="col s6 offset-s3">
-                                <h5 class="center-align">Ranking by unique code regions covered</h4>
+                                <h5 class="center-align">Ranking by unique code branches covered</h4>
                                     <img class="responsive-img materialboxed"
                                          src="{{ benchmark.unique_coverage_ranking_plot }}">
-                                    Each bar shows the total number of code regions found by a given fuzzer.
-                                    The colored area shows the number of unique code regions
-                                    (i.e., regions that were not covered by any other fuzzers).
+                                    Each bar shows the total number of code branches found by a given fuzzer.
+                                    The colored area shows the number of unique code branches
+                                    (i.e., branches that were not covered by any other fuzzers).
                             </div>
                         </div> <!-- row -->
 
@@ -399,7 +399,7 @@ <h5 class="center-align">Ranking by unique code regions covered</h4>
                                 <h5 class="center-align">Pairwise unique code coverage</h4>
                                     <img class="responsive-img materialboxed"
                                          src="{{ benchmark.pairwise_unique_coverage_plot }}">
-                                    Each cell represents the number of code regions covered by the fuzzer
+                                    Each cell represents the number of code branches covered by the fuzzer
                                     of the column but not by the fuzzer of the row
                             </div>
                         </div> <!-- row -->