From c36d5a2a8223e426dd63a66b41cf86ded7b7e870 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid Al Rashid Date: Sat, 13 Feb 2021 20:06:28 +0600 Subject: [PATCH 1/2] feat: save report in junit format This format is a standard across the industry and hence can be parsed by differnet reporting tools, including but not limited to AWS Device Farm and Jenkins. --- Framework/MainDriverApi.py | 19 +++++++---- reporting/junit_report.py | 65 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 reporting/junit_report.py diff --git a/Framework/MainDriverApi.py b/Framework/MainDriverApi.py index 1933cadc5..bde4b76ce 100644 --- a/Framework/MainDriverApi.py +++ b/Framework/MainDriverApi.py @@ -28,6 +28,7 @@ BuiltInFunctionSharedResources as shared, ) from Framework.Utilities import ws +from reporting import junit_report top_path = os.path.dirname(os.getcwd()) drivers_path = os.path.join(top_path, "Drivers") @@ -1655,15 +1656,15 @@ def get_all_run_id_info(Userid, sModuleInfo): def upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info): - zip_path = ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file) / Path(run_id.replace(":", "-")) - path = zip_path / Path("execution_log.json") + zip_path = Path(ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file)) / run_id.replace(":", "-") + path = zip_path / "execution_log.json" json_report = CommonUtil.get_all_logs(json=True) with open(path, "w") as f: json.dump(json_report, f) if ConfigModule.get_config_value("RunDefinition", "local_run") == "False": FL.ZipFolder(str(zip_path), str(zip_path) + ".zip") - for i in range(720): # 1 hour + for _ in range(720): # 1 hour res = requests.get(RequestFormatter.form_uri("is_copied_api/"), {"runid": run_id}, verify=False) r = res.json() if r["flag"]: @@ -1671,7 +1672,7 @@ def upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info): time.sleep(5) else: print("Run history was not created in server so couldn't upload the report for run_id '%s'." % run_id + - "Get the report from below path-\n" + path) + "Get the report from below path-\n" + path) return with open(str(zip_path) + ".zip", "rb") as fzip: @@ -1702,12 +1703,18 @@ def upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info): else: print("Could not Upload the report to server of run_id '%s'" % run_id) os.unlink(str(zip_path) + ".zip") + with open(path, "w") as f: json.dump(json_report, f, indent=2) - path = zip_path / Path("test_cases_data.json") + path = zip_path / "test_cases_data.json" with open(path, "w") as f: json.dump(all_run_id_info, f, indent=2) + # Create a standard report format to be consumed by other tools. + junit_report_path = zip_path / "report.xml" + junit_report.process(all_run_id_info, str(junit_report_path)) + + # main function def main(device_dict, user_info_object): @@ -1798,7 +1805,7 @@ def main(device_dict, user_info_object): if "debug_step_actions" in run_id_info: debug_info["debug_step_actions"] = run_id_info["debug_step_actions"] driver_list = ['Built_In_Selenium_Driver', 'Built_In_RestApi', 'Built_In_Appium_Driver', 'Built_In_Selenium', - 'Built_In_Driver', 'deepak', 'Built_In_Appium', 'Built_In_NET_Win', 'Jarvis'] + 'Built_In_Driver', 'deepak', 'Built_In_Appium', 'Built_In_NET_Win', 'Jarvis'] final_run_params = {} for param in final_run_params_from_server: final_run_params[param] = CommonUtil.parse_value_into_object(list(final_run_params_from_server[param].items())[1][1]) diff --git a/reporting/junit_report.py b/reporting/junit_report.py new file mode 100644 index 000000000..473ceb6cb --- /dev/null +++ b/reporting/junit_report.py @@ -0,0 +1,65 @@ +import xml.etree.ElementTree as ET +import json + +# Spec: +# https://www.ibm.com/support/knowledgecenter/SSQ2R2_9.1.1/com.ibm.rsar.analysis.codereview.cobol.doc/topics/cac_useresults_junit.html + + +def process(data, save_path="report.xml"): + # For now we'll only deal with one run-id + data = data[0] + + testsuite = ET.Element("testsuite") + testsuite.set("id", data["run_id"]) + testsuite.set("name", data["TestObjective"]) + testsuite.set("tests", str(len(data["test_cases"]))) + testsuite.set("duration", data["execution_detail"]["duration"]) + testsuite.set("timestamp", data["execution_detail"]["teststarttime"]) + + # Number of failed test cases. + failures = 0 + + for tc in data["test_cases"]: + testcase = ET.SubElement(testsuite, "testcase") + testcase.set("id", tc["testcase_no"]) + testcase.set("name", tc["title"]) + testcase.set("time", tc["execution_detail"]["duration"]) + + if tc["execution_detail"]["status"].lower() == "passed": + # Skip passed test cases. + continue + + # Failure/error details related code. + failures += 1 + try: + for step in tc["steps"]: + if step["execution_detail"]["status"].lower() == "passed": + # Skip passed steps. + continue + failure = ET.SubElement(testcase, "failure") + failure.set("type", step["step_name"]) + failure.set("message", step["step_name"] + " failed") + failure.text = tc["execution_detail"]["failreason"] + except: + pass + + testsuite.set("failures", str(failures)) + + ET.ElementTree(testsuite).write( + save_path, + encoding="UTF-8", + xml_declaration=True + ) + + +def main(): + data = None + with open("reporting/sample.json", "r") as f: + data = json.loads(f.read()) + + #print(data) + process(data, "reporting/report.xml") + + +if __name__ == "__main__": + main() From c2869c34c4615a52da28b294b0a4398b2241a173 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid Al Rashid Date: Sat, 13 Feb 2021 20:38:40 +0600 Subject: [PATCH 2/2] feat: Copy logs/reports to expected dir in farms --- Framework/MainDriverApi.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Framework/MainDriverApi.py b/Framework/MainDriverApi.py index bde4b76ce..e25b97100 100644 --- a/Framework/MainDriverApi.py +++ b/Framework/MainDriverApi.py @@ -1711,7 +1711,7 @@ def upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info): json.dump(all_run_id_info, f, indent=2) # Create a standard report format to be consumed by other tools. - junit_report_path = zip_path / "report.xml" + junit_report_path = zip_path / "junitreport.xml" junit_report.process(all_run_id_info, str(junit_report_path)) @@ -1963,6 +1963,17 @@ def kill(process): CommonUtil.ExecLog(sModuleInfo, "Test Set Cancelled by the User", 1) # add log elif not run_id.startswith("debug"): upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info) + + # If node is running in device farm, copy the logs and reports to + # the expected directory. + if "DEVICEFARM_LOG_DIR" in os.environ: + log_dir = Path(os.environ["DEVICEFARM_LOG_DIR"]) + zeuz_log_dir = Path(ConfigModule.get_config_value( + "sectionOne", "test_case_folder", temp_ini_file + )).parent + + shutil.copytree(str(zeuz_log_dir), str(log_dir)) + # executor.submit(upload_json_report) # Close websocket connection.