From 057f8764e6ff27bce02babf8cbf40de823130b50 Mon Sep 17 00:00:00 2001 From: vodorok Date: Thu, 10 Oct 2024 23:46:56 +0200 Subject: [PATCH 1/8] Add repr to BuildAction class --- analyzer/codechecker_analyzer/buildlog/build_action.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/analyzer/codechecker_analyzer/buildlog/build_action.py b/analyzer/codechecker_analyzer/buildlog/build_action.py index 692e164a6a..d576fefe18 100644 --- a/analyzer/codechecker_analyzer/buildlog/build_action.py +++ b/analyzer/codechecker_analyzer/buildlog/build_action.py @@ -76,3 +76,6 @@ def with_attr(self, attr, value): details = {key: getattr(self, key) for key in BuildAction.__slots__} details[attr] = value return BuildAction(**details) + + def __repr__(self): + return str(self) From fff7b5e52edc71409ca4991c2ed586b1a8999a7e Mon Sep 17 00:00:00 2001 From: vodorok Date: Mon, 14 Oct 2024 14:46:41 +0200 Subject: [PATCH 2/8] Add single plist option This patch adds the logic of report output merging into one singluar `.plist` file. Now analyze and check can be invoked with the `--plist-file-name` option, which if provided will merge all new `.plist` result files into the specified file name under the specified output report folder. --- .../codechecker_analyzer/analysis_manager.py | 34 ++++++++++++++++--- analyzer/codechecker_analyzer/analyzer.py | 1 + analyzer/codechecker_analyzer/cmd/analyze.py | 9 +++++ analyzer/codechecker_analyzer/cmd/check.py | 9 +++++ .../report/parser/plist.py | 25 ++++++++------ 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index c087262fa2..1c466d9efb 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -17,11 +17,14 @@ import traceback import zipfile +from pathlib import Path from threading import Timer import multiprocess import psutil +import plistlib + from codechecker_common.logger import get_logger from codechecker_common.review_status_handler import ReviewStatusHandler @@ -57,8 +60,29 @@ def print_analyzer_statistic_summary(metadata_analyzers, status, msg=None): LOG.info(" %s: %s", analyzer_type, res) -def worker_result_handler(results, metadata_tool, output_path): - """ Print the analysis summary. """ +def worker_result_handler(results, metadata_tool, output_path, plist_file_name): + """ + Handle analysis results after all the analyzer threads returned. It may + merge all the plist output files into one, and print the analysis summary. + """ + LOG.info("Merging plist files into %s", plist_file_name) + if plist_file_name: + plist_data = [] + single_plist = Path(output_path, plist_file_name) + for _, _, _, _, original_plist, _ in results: + original_plist = Path(original_plist) + if os.path.exists(original_plist): + with open(original_plist, 'rb') as plist: + LOG.debug(f"Merging original plist {original_plist}") + plist_data.append(plistlib.load(plist)) + + with open(single_plist, 'wb') as plist: + LOG.debug(f"Dumping merged plist file into {single_plist}") + plistlib.dump(plist_data, plist) + + LOG.debug(f"Removing original plist file {original_plist}") + original_plist.unlink() + skipped_num = 0 reanalyzed_num = 0 metadata_analyzers = metadata_tool['analyzers'] @@ -719,8 +743,8 @@ def skip_cpp(compile_actions, skip_handlers): return analyze, skip -def start_workers(actions_map, actions, analyzer_config_map, - jobs, output_path, skip_handlers, filter_handlers, +def start_workers(actions_map, actions, analyzer_config_map, jobs, + output_path, plist_file_name, skip_handlers, filter_handlers, rs_handler: ReviewStatusHandler, metadata_tool, quiet_analyze, capture_analysis_output, generate_reproducer, timeout, ctu_reanalyze_on_failure, statistics_data, manager, @@ -815,7 +839,7 @@ def signal_handler(signum, _): analyzed_actions, 1, callback=lambda results: worker_result_handler( - results, metadata_tool, output_path) + results, metadata_tool, output_path, plist_file_name) ).get(timeout) pool.close() diff --git a/analyzer/codechecker_analyzer/analyzer.py b/analyzer/codechecker_analyzer/analyzer.py index b788461e9f..a07f1c863f 100644 --- a/analyzer/codechecker_analyzer/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzer.py @@ -333,6 +333,7 @@ def perform_analysis(args, skip_handlers, filter_handlers, analysis_manager.start_workers(actions_map, actions, config_map, args.jobs, args.output_path, + args.plist_file_name, skip_handlers, filter_handlers, rs_handler, diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index 440018e243..e636ed8393 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -220,6 +220,15 @@ def add_arguments_to_parser(parser): default=argparse.SUPPRESS, help="Store the analysis output in the given folder.") + parser.add_argument('--plist-file-name', + type=str, + dest="plist_file_name", + required=False, + help="If given, all the `.plist` files containing " + "the analyzer result files will be merged " + "into a single `.plist` file in the report " + "output folder given by `-o/--output`.") + parser.add_argument('--compiler-info-file', dest="compiler_info_file", required=False, diff --git a/analyzer/codechecker_analyzer/cmd/check.py b/analyzer/codechecker_analyzer/cmd/check.py index fa2ef4e505..267990e2e1 100644 --- a/analyzer/codechecker_analyzer/cmd/check.py +++ b/analyzer/codechecker_analyzer/cmd/check.py @@ -111,6 +111,15 @@ def add_arguments_to_parser(parser): "If it is not given then the results go into a " "temporary directory which will be removed after " "the analysis.") + + parser.add_argument('--plist-file-name', + type=str, + dest="plist_file_name", + required=False, + help="If given, all the `.plist` files containing " + "the analyzer result files will be merged " + "into a single `.plist` file in the report " + "output folder given by `-o/--output`.") parser.add_argument('-t', '--type', '--output-format', dest="output_format", diff --git a/tools/report-converter/codechecker_report_converter/report/parser/plist.py b/tools/report-converter/codechecker_report_converter/report/parser/plist.py index 3f79878f06..471b079ec6 100644 --- a/tools/report-converter/codechecker_report_converter/report/parser/plist.py +++ b/tools/report-converter/codechecker_report_converter/report/parser/plist.py @@ -206,20 +206,25 @@ def get_reports( if not plist: return reports - metadata = plist.get('metadata') + if not isinstance(plist, list): + plist=[plist] - files = get_file_index_map( - plist, source_dir_path, self._file_cache) + for sub_plist in plist: - for diag in plist.get('diagnostics', []): - report = self.__create_report( - analyzer_result_file_path, diag, files, metadata) + metadata = sub_plist.get('metadata') - if report.report_hash is None: - report.report_hash = get_report_hash( - report, HashType.PATH_SENSITIVE) + files = get_file_index_map( + sub_plist, source_dir_path, self._file_cache) + + for diag in sub_plist.get('diagnostics', []): + report = self.__create_report( + analyzer_result_file_path, diag, files, metadata) + + if report.report_hash is None: + report.report_hash = get_report_hash( + report, HashType.PATH_SENSITIVE) - reports.append(report) + reports.append(report) except KeyError as ex: LOG.warning("Failed to get file path id! Found files: %s. " "KeyError: %s", files, ex) From 02a167f47d4aee1450bbdbd3060dd26b4efca4b6 Mon Sep 17 00:00:00 2001 From: vodorok Date: Mon, 14 Oct 2024 15:00:33 +0200 Subject: [PATCH 3/8] Fix Linter errors --- .../codechecker_analyzer/analysis_manager.py | 16 +++++++++++----- .../buildlog/build_action.py | 2 +- analyzer/codechecker_analyzer/cmd/check.py | 2 +- .../report/parser/plist.py | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index 1c466d9efb..2525dee29f 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -60,10 +60,13 @@ def print_analyzer_statistic_summary(metadata_analyzers, status, msg=None): LOG.info(" %s: %s", analyzer_type, res) -def worker_result_handler(results, metadata_tool, output_path, plist_file_name): +def worker_result_handler(results, + metadata_tool, + output_path, + plist_file_name): """ - Handle analysis results after all the analyzer threads returned. It may - merge all the plist output files into one, and print the analysis summary. + Handle analysis results after all the analyzer threads returned. It may + merge all the plist output files into one, and print the analysis summary. """ LOG.info("Merging plist files into %s", plist_file_name) if plist_file_name: @@ -72,7 +75,7 @@ def worker_result_handler(results, metadata_tool, output_path, plist_file_name): for _, _, _, _, original_plist, _ in results: original_plist = Path(original_plist) if os.path.exists(original_plist): - with open(original_plist, 'rb') as plist: + with open(original_plist, 'rb') as plist: LOG.debug(f"Merging original plist {original_plist}") plist_data.append(plistlib.load(plist)) @@ -839,7 +842,10 @@ def signal_handler(signum, _): analyzed_actions, 1, callback=lambda results: worker_result_handler( - results, metadata_tool, output_path, plist_file_name) + results, + metadata_tool, + output_path, + plist_file_name) ).get(timeout) pool.close() diff --git a/analyzer/codechecker_analyzer/buildlog/build_action.py b/analyzer/codechecker_analyzer/buildlog/build_action.py index d576fefe18..1a43fcd5a8 100644 --- a/analyzer/codechecker_analyzer/buildlog/build_action.py +++ b/analyzer/codechecker_analyzer/buildlog/build_action.py @@ -76,6 +76,6 @@ def with_attr(self, attr, value): details = {key: getattr(self, key) for key in BuildAction.__slots__} details[attr] = value return BuildAction(**details) - + def __repr__(self): return str(self) diff --git a/analyzer/codechecker_analyzer/cmd/check.py b/analyzer/codechecker_analyzer/cmd/check.py index 267990e2e1..67cc795a35 100644 --- a/analyzer/codechecker_analyzer/cmd/check.py +++ b/analyzer/codechecker_analyzer/cmd/check.py @@ -111,7 +111,7 @@ def add_arguments_to_parser(parser): "If it is not given then the results go into a " "temporary directory which will be removed after " "the analysis.") - + parser.add_argument('--plist-file-name', type=str, dest="plist_file_name", diff --git a/tools/report-converter/codechecker_report_converter/report/parser/plist.py b/tools/report-converter/codechecker_report_converter/report/parser/plist.py index 471b079ec6..e6e7811d74 100644 --- a/tools/report-converter/codechecker_report_converter/report/parser/plist.py +++ b/tools/report-converter/codechecker_report_converter/report/parser/plist.py @@ -207,7 +207,7 @@ def get_reports( return reports if not isinstance(plist, list): - plist=[plist] + plist = [plist] for sub_plist in plist: From d47286afdce1a005d8c76a0fe3feb0c6becb920f Mon Sep 17 00:00:00 2001 From: vodorok Date: Mon, 14 Oct 2024 16:11:48 +0200 Subject: [PATCH 4/8] False value if parameter is not present --- analyzer/codechecker_analyzer/cmd/analyze.py | 1 + analyzer/codechecker_analyzer/cmd/check.py | 1 + 2 files changed, 2 insertions(+) diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index e636ed8393..13dc3a57c2 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -223,6 +223,7 @@ def add_arguments_to_parser(parser): parser.add_argument('--plist-file-name', type=str, dest="plist_file_name", + default='', required=False, help="If given, all the `.plist` files containing " "the analyzer result files will be merged " diff --git a/analyzer/codechecker_analyzer/cmd/check.py b/analyzer/codechecker_analyzer/cmd/check.py index 67cc795a35..7727857f0d 100644 --- a/analyzer/codechecker_analyzer/cmd/check.py +++ b/analyzer/codechecker_analyzer/cmd/check.py @@ -116,6 +116,7 @@ def add_arguments_to_parser(parser): type=str, dest="plist_file_name", required=False, + default='', help="If given, all the `.plist` files containing " "the analyzer result files will be merged " "into a single `.plist` file in the report " From ec3a181cdda56c881280aa13445c32c2b2cd80bf Mon Sep 17 00:00:00 2001 From: vodorok Date: Mon, 14 Oct 2024 17:36:48 +0200 Subject: [PATCH 5/8] Pass plist_file_name along to analyze command --- analyzer/codechecker_analyzer/cmd/check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/analyzer/codechecker_analyzer/cmd/check.py b/analyzer/codechecker_analyzer/cmd/check.py index 7727857f0d..7fde58677c 100644 --- a/analyzer/codechecker_analyzer/cmd/check.py +++ b/analyzer/codechecker_analyzer/cmd/check.py @@ -925,6 +925,7 @@ def __update_if_key_exists(source, target, key): 'skipfile', 'drop_skipped_reports', 'files', + 'plist_file_name', 'analyzers', 'add_compiler_defaults', 'cppcheck_args_cfg_file', From 3f0718513021d228e39c44795a3d2f5d349337ca Mon Sep 17 00:00:00 2001 From: vodorok Date: Mon, 14 Oct 2024 17:37:38 +0200 Subject: [PATCH 6/8] Move logging inside the handling condition --- analyzer/codechecker_analyzer/analysis_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index 2525dee29f..5871c122ae 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -68,8 +68,8 @@ def worker_result_handler(results, Handle analysis results after all the analyzer threads returned. It may merge all the plist output files into one, and print the analysis summary. """ - LOG.info("Merging plist files into %s", plist_file_name) if plist_file_name: + LOG.info("Merging plist files into %s", plist_file_name) plist_data = [] single_plist = Path(output_path, plist_file_name) for _, _, _, _, original_plist, _ in results: From e84961bf9e854de67752335e277d09aeed9dd33d Mon Sep 17 00:00:00 2001 From: vodorok Date: Tue, 15 Oct 2024 14:13:36 +0200 Subject: [PATCH 7/8] Factor out merging logic into a function --- .../codechecker_analyzer/analysis_manager.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index 5871c122ae..680685f1e1 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -60,6 +60,29 @@ def print_analyzer_statistic_summary(metadata_analyzers, status, msg=None): LOG.info(" %s: %s", analyzer_type, res) +def merge_plists(results, output_path, plist_file_name): + """ + Merge the plist files generated by the analyzers into a single plist file. + Deletes the original plist files after merging. + """ + LOG.info("Merging plist files into %s", plist_file_name) + plist_data = [] + single_plist = Path(output_path, plist_file_name) + for _, _, _, _, original_plist, _ in results: + original_plist = Path(original_plist) + if os.path.exists(original_plist): + with open(original_plist, 'rb') as plist: + LOG.debug(f"Merging original plist {original_plist}") + plist_data.append(plistlib.load(plist)) + + with open(single_plist, 'wb') as plist: + LOG.debug(f"Dumping merged plist file into {single_plist}") + plistlib.dump(plist_data, plist) + + LOG.debug(f"Removing original plist file {original_plist}") + original_plist.unlink() + + def worker_result_handler(results, metadata_tool, output_path, @@ -69,22 +92,7 @@ def worker_result_handler(results, merge all the plist output files into one, and print the analysis summary. """ if plist_file_name: - LOG.info("Merging plist files into %s", plist_file_name) - plist_data = [] - single_plist = Path(output_path, plist_file_name) - for _, _, _, _, original_plist, _ in results: - original_plist = Path(original_plist) - if os.path.exists(original_plist): - with open(original_plist, 'rb') as plist: - LOG.debug(f"Merging original plist {original_plist}") - plist_data.append(plistlib.load(plist)) - - with open(single_plist, 'wb') as plist: - LOG.debug(f"Dumping merged plist file into {single_plist}") - plistlib.dump(plist_data, plist) - - LOG.debug(f"Removing original plist file {original_plist}") - original_plist.unlink() + merge_plists(results, output_path, plist_file_name) skipped_num = 0 reanalyzed_num = 0 From 9e66f538c45c927e1855c3964498a717dbe76485 Mon Sep 17 00:00:00 2001 From: vodorok Date: Tue, 15 Oct 2024 18:11:03 +0200 Subject: [PATCH 8/8] Add analyze test case --- .../tests/functional/analyze/test_analyze.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/analyzer/tests/functional/analyze/test_analyze.py b/analyzer/tests/functional/analyze/test_analyze.py index ecabdaec4f..61cfe8ac44 100644 --- a/analyzer/tests/functional/analyze/test_analyze.py +++ b/analyzer/tests/functional/analyze/test_analyze.py @@ -21,6 +21,8 @@ import unittest import zipfile +from pathlib import Path + from libtest import env from codechecker_report_converter.report import report_file @@ -1200,6 +1202,50 @@ def test_disable_all_checkers(self): # Checkers of all 3 analyzers are disabled. self.assertEqual(out.count("No checkers enabled for"), 5) + def test_single_plist(self): + """ + Test if specified with the `--plist-file-name` flag. + Analyze output should contain the indication of merging. + Merged plist should be created at the end of the analysis. + Only one `.plist` should remain at the end of the analysis. + """ + build_json = os.path.join( + self.test_workspace, "build_success.json") + source_file = os.path.join( + self.test_dir, "success.c") + build_log = [{"directory": self.test_workspace, + "command": "gcc -c " + source_file, + "file": source_file}] + + with open(build_json, 'w', + encoding="utf-8", errors="ignore") as outfile: + json.dump(build_log, outfile) + + merged_plist_name = "merged.plist" + + analyze_cmd = [self._codechecker_cmd, "analyze", + "--plist-file-name", merged_plist_name, + "-o", self.report_dir, build_json] + + print(analyze_cmd) + process = subprocess.Popen( + analyze_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.test_dir, + encoding="utf-8", + errors="ignore") + out, _ = process.communicate() + + # Output show merging + self.assertIn("Merging plist files into " + merged_plist_name, out) + + # Only the merged plist should remain + for file in Path(self.report_dir).glob("*.plist"): + self.assertEqual(file.name, merged_plist_name) + + print(out) + def test_analyzer_and_checker_config(self): """Test analyzer configuration through command line flags.""" build_json = os.path.join(self.test_workspace, "build_success.json")