diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 57783c679..7eb2c0b0f 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -2197,6 +2197,7 @@ def excel_read(data_set): # Save file so that we don't see the "Want to save" dailog. wb.save() + wb.close() return "passed" except: @@ -2286,6 +2287,7 @@ def excel_comparison(data_set): # Save workbook. wb.save() + wb.close() return "passed" except: diff --git a/Framework/Built_In_Automation/Shared_Resources/BuiltInFunctionSharedResources.py b/Framework/Built_In_Automation/Shared_Resources/BuiltInFunctionSharedResources.py index da1ccdb43..6a38244fb 100644 --- a/Framework/Built_In_Automation/Shared_Resources/BuiltInFunctionSharedResources.py +++ b/Framework/Built_In_Automation/Shared_Resources/BuiltInFunctionSharedResources.py @@ -53,11 +53,13 @@ def Set_Shared_Variables(key, value, protected=False, allowEmpty=False, print_va shared_variables[key] = value if print_variable: + try: val = json.dumps(CommonUtil.parse_value_into_object(value), indent=2, sort_keys=True) + except: val = str(value) CommonUtil.ExecLog( sModuleInfo, "Saved variable: %s" % key, 1, variable={ "key": key, - "val": json.dumps(CommonUtil.parse_value_into_object(value), indent=2, sort_keys=True) + "val": val } ) diff --git a/Framework/MainDriverApi.py b/Framework/MainDriverApi.py index 3a946fba8..b0025aaeb 100644 --- a/Framework/MainDriverApi.py +++ b/Framework/MainDriverApi.py @@ -1327,31 +1327,31 @@ def write_log_file_for_test_case( ) # upload will go here. - upload_zip( - ConfigModule.get_config_value("Authentication", "server_address"), - ConfigModule.get_config_value("Authentication", "server_port"), - ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file), - run_id, - ConfigModule.get_config_value("sectionOne", "test_case", temp_ini_file) + ".zip", - ConfigModule.get_config_value("Advanced Options", "_file_upload_path"), - ) - TCLogFile = ( - os.sep - + ConfigModule.get_config_value("Advanced Options", "_file_upload_path") - + os.sep - + run_id.replace(":", "-") - + "/" - + ConfigModule.get_config_value( - "sectionOne", "test_case", temp_ini_file - ) - + ".zip" - ) - FL.DeleteFile( - ConfigModule.get_config_value( - "sectionOne", "test_case_folder", temp_ini_file - ) - + ".zip" - ) + # upload_zip( + # ConfigModule.get_config_value("Authentication", "server_address"), + # ConfigModule.get_config_value("Authentication", "server_port"), + # ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file), + # run_id, + # ConfigModule.get_config_value("sectionOne", "test_case", temp_ini_file) + ".zip", + # ConfigModule.get_config_value("Advanced Options", "_file_upload_path"), + # ) + # TCLogFile = ( + # os.sep + # + ConfigModule.get_config_value("Advanced Options", "_file_upload_path") + # + os.sep + # + run_id.replace(":", "-") + # + "/" + # + ConfigModule.get_config_value( + # "sectionOne", "test_case", temp_ini_file + # ) + # + ".zip" + # ) + # FL.DeleteFile( + # ConfigModule.get_config_value( + # "sectionOne", "test_case_folder", temp_ini_file + # ) + # + ".zip" + # ) else: TCLogFile = "" # upload the log file ID @@ -1433,17 +1433,19 @@ def run_test_case( testcase_info, executor, debug_info, + all_file_specific_steps, send_log_file_only_for_fail=True, performance=False, browserDriver=None, ): shared.Set_Shared_Variables("run_id", run_id) test_case = str(TestCaseID).replace("#", "no") - ConfigModule.add_config_value("sectionOne", "sTestStepExecLogId", "MainDriver", temp_ini_file) - file_specific_steps = download_attachments_for_test_case( + ConfigModule.add_config_value("sectionOne", "sTestStepExecLogId", sModuleInfo, temp_ini_file) + download_attachments_for_test_case( sModuleInfo, run_id, test_case, temp_ini_file, - testcase_info["testcase_attachments_links"], testcase_info["step_attachments"] + [],[] ) + file_specific_steps = all_file_specific_steps[TestCaseID] if TestCaseID in all_file_specific_steps else {} TestCaseName = testcase_info["title"] log_line = "# EXECUTING TEST CASE : %s :: %s #" % (test_case, TestCaseName) print("#"*(len(log_line))) @@ -1471,6 +1473,13 @@ def run_test_case( performance ) + ConfigModule.add_config_value( + "sectionOne", + "sTestStepExecLogId", + run_id + "|" + test_case + "|" + "none" + "|" + "none", + temp_ini_file, + ) + # get test case end time TestCaseEndTime = time.time() sTestCaseEndTime = datetime.fromtimestamp(TestCaseEndTime).strftime("%Y-%m-%d %H:%M:%S") @@ -1738,19 +1747,15 @@ def get_all_run_id_info(Userid, sModuleInfo): # return response["json"] -def upload_json_report(Userid, temp_ini_file, run_id): - # path = os.path.join(os.path.abspath(__file__).split("Framework")[0])/Path("AutomationLog")/Path("execution_log.json") - path = ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file) / Path(run_id.replace(":", "-")) - zip_path = path / Path("execution_log.zip") - path = path / Path("execution_log.json") +def upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info): + zip_path = ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file) / Path(run_id.replace(":", "-")) + path = zip_path / Path("execution_log.json") json_report = CommonUtil.get_all_logs(json=True) with open(path, "w") as f: - json.dump(json_report, f, indent=2) - - FL.ZipFile(path, zip_path) - FL.DeleteFile(path) + json.dump(json_report, f) if ConfigModule.get_config_value("RunDefinition", "local_run") == "False": + FL.ZipFolder(str(zip_path), str(zip_path) + ".zip") for i in range(720): # 1 hour res = requests.get(RequestFormatter.form_uri("is_copied_api/"), {"runid": run_id}, verify=False) r = res.json() @@ -1762,7 +1767,8 @@ def upload_json_report(Userid, temp_ini_file, run_id): "Get the report from below path-\n" + path) return - with open(zip_path, "rb") as fzip: + with open(str(zip_path) + ".zip", "rb") as fzip: + print("Uploading report of %s KB. Please wait" % (os.stat(str(zip_path) + ".zip").st_size / 1000)) for i in range(5): res = requests.post( RequestFormatter.form_uri("create_report_log_api/"), @@ -1776,17 +1782,22 @@ def upload_json_report(Userid, temp_ini_file, run_id): print("Could not Upload json report to server") return if isinstance(res_json, dict) and 'message' in res_json and res_json["message"]: - print("Successfully Uploaded json report to server") + print("Successfully Uploaded the report to server of run_id '%s'" % run_id) else: - print("Could not Upload json report to server") + print("Could not Upload the report to server of run_id '%s'" % run_id) break time.sleep(1) else: print("Could not Upload json report to server") - + os.unlink(str(zip_path) + ".zip") + with open(path, "w") as f: + json.dump(json_report, f, indent=2) + path = zip_path / Path("test_cases_data.json") + with open(path, "w") as f: + json.dump(all_run_id_info, f, indent=2) # main function -def main(device_dict, user_info_object, all_run_id_info): +def main(device_dict, user_info_object): # get module info sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME @@ -1807,13 +1818,31 @@ def main(device_dict, user_info_object, all_run_id_info): Userid = (CommonUtil.MachineInfo().getLocalUser()).lower() # all_run_id_info = get_all_run_id_info(Userid, sModuleInfo) - + get_json, all_file_specific_steps = True, {} + save_path = Path(ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file)) / "attachments" + cnt = 0 + for i in os.walk(save_path): + if get_json: + get_json = False + json_path = Path(i[0]) / i[2][0] + folder_list = i[1] + for j in folder_list: + all_file_specific_steps[j] = {} + else: + for j in i[2]: + all_file_specific_steps[folder_list[cnt]][j] = str(Path(i[0]) / j) + cnt += 1 + with open(json_path, "r") as f: + all_run_id_info = json.loads(f.read()) + with open(Path.cwd().parent / "Projects" / "Local_run.json", "w") as f: + f.write(json.dumps(all_run_id_info)) if len(all_run_id_info) == 0: CommonUtil.ExecLog("", "No Test Run Schedule found for the current user : %s" % Userid, 2) return False executor = CommonUtil.GetExecutor() for run_id_info in all_run_id_info: + run_id_info["base_path"] = ConfigModule.get_config_value("Advanced Options", "_file_upload_path") run_id = run_id_info["run_id"] run_cancelled = "" debug_info = "" @@ -1822,9 +1851,7 @@ def main(device_dict, user_info_object, all_run_id_info): # Write testcase json path = ConfigModule.get_config_value("sectionOne", "temp_run_file_path", temp_ini_file) / Path(run_id.replace(":", "-")) FL.CreateFolder(path) - path = path / Path("test_cases_data.json") - with open(path, "w") as f: - json.dump(all_run_id_info, f, indent=2) + # Start websocket server if we're in debug mode. if run_id.lower().startswith("debug"): @@ -1884,7 +1911,7 @@ def main(device_dict, user_info_object, all_run_id_info): else: CommonUtil.ExecLog("", "No Automated test cases found for the current user : %s" % Userid, 2) return "pass" - + num_of_tc = len(all_testcases_info) CommonUtil.all_logs_json = all_run_id_info cnt = 1 for testcase_info in all_testcases_info: @@ -1986,6 +2013,7 @@ def kill(process): testcase_info, executor, debug_info, + all_file_specific_steps, send_log_file_only_for_fail, ) CommonUtil.clear_all_logs() # clear logs @@ -2008,6 +2036,7 @@ def kill(process): "duration": TestSetDuration } CommonUtil.CreateJsonReport(setInfo=after_execution_dict) + print("Report creation time = %s sec for %s testcases" % (CommonUtil.report_json_time, num_of_tc)) CommonUtil.ExecLog("", "Test Set Completed", 4, False) ConfigModule.add_config_value("sectionOne", "sTestStepExecLogId", "MainDriver", temp_ini_file) @@ -2015,7 +2044,7 @@ def kill(process): if run_cancelled == CANCELLED_TAG: CommonUtil.ExecLog(sModuleInfo, "Test Set Cancelled by the User", 1) # add log elif not run_id.startswith("debug"): - upload_json_report(Userid, temp_ini_file, run_id) + upload_json_report(Userid, temp_ini_file, run_id, all_run_id_info) # executor.submit(upload_json_report) # Close websocket connection. diff --git a/Framework/Utilities/CommonUtil.py b/Framework/Utilities/CommonUtil.py index 28b9c9395..15be8f2b6 100644 --- a/Framework/Utilities/CommonUtil.py +++ b/Framework/Utilities/CommonUtil.py @@ -93,6 +93,7 @@ all_logs = {} all_logs_json, json_log_cond = [], False +tc_error_logs = [] all_logs_count = 0 all_logs_list = [] skip_list = ["step_data"] @@ -290,9 +291,13 @@ def Result_Analyzer(sTestStepReturnStatus, temp_q): except Exception as e: return Exception_Handler(sys.exc_info()) +report_json_time = 0.0 +import time + def CreateJsonReport(logs=None, stepInfo=None, TCInfo=None, setInfo=None): - global all_logs_json + global all_logs_json, report_json_time, tc_error_logs + start = time.perf_counter() if logs or stepInfo or TCInfo or setInfo: log_id = ConfigModule.get_config_value("sectionOne", "sTestStepExecLogId", temp_config) if not log_id: @@ -300,9 +305,7 @@ def CreateJsonReport(logs=None, stepInfo=None, TCInfo=None, setInfo=None): log_id_vals = log_id.split("|") if logs: log_id, now, iLogLevel, status, sModuleInfo, sDetails = logs - if len(log_id_vals) != 4: - pass - else: + if len(log_id_vals) == 4: # these loops can be optimized by saving the previous log_id_vals and comparing it with current one runID, testcase_no, step_id, step_no = log_id_vals for run_id_info in all_logs_json: @@ -316,31 +319,35 @@ def CreateJsonReport(logs=None, stepInfo=None, TCInfo=None, setInfo=None): if testcase_no == testcase_info["testcase_no"].replace("#", "no"): if TCInfo: testcase_info["execution_detail"] = TCInfo + fail_reason_str = "" + if TCInfo["status"] in ("Failed", "Blocked"): + count = -min(len(tc_error_logs), 3) + while count <= -1: + fail_reason_str += tc_error_logs[count] + if count != -1: + fail_reason_str += "\n---------------------------------------------\n" + count += 1 + testcase_info["execution_detail"]["failreason"] = fail_reason_str + break + if step_id == "none": break all_step_info = testcase_info["steps"] for step_info in all_step_info: if step_no == str(step_info["step_sequence"]) and step_id == str(step_info["step_id"]): if stepInfo: step_info["execution_detail"] = stepInfo - fail_reason_log = [] - fail_reason_str = "" + step_error_logs = [] if stepInfo["status"].lower() == "failed": - count = 0 - for each_log in reversed(step_info["log"]): - if count == 4: - break - if each_log["status"].lower() == "error": - fail_reason_log.append(each_log["details"]) - count += 1 - fail_reason_log.reverse() - if fail_reason_log[-1].endswith(to_dlt_from_fail_reason): - del fail_reason_log[-1] - if len(fail_reason_log) > 3: - del fail_reason_log[0] - for i in fail_reason_log: - fail_reason_str += i + "\n" - fail_reason_str = fail_reason_str[:-1] - step_info["failreason"] = fail_reason_str + count, err_count, max_count = -1, 0, -len(step_info["log"]) + # Can be optimized by taking error when occurs and append it if the step fails only + while count >= max_count and err_count < 3: + each_log = step_info["log"][count] + if each_log["status"].lower() == "error" and not each_log["details"].endswith(to_dlt_from_fail_reason): + step_error_logs.append(each_log["details"]) + err_count += 1 + count -= 1 + step_error_logs.reverse() + tc_error_logs += step_error_logs break log_info = { "status": status, @@ -362,6 +369,7 @@ def CreateJsonReport(logs=None, stepInfo=None, TCInfo=None, setInfo=None): break elif stepInfo: pass + report_json_time += (time.perf_counter() - start) def ExecLog( diff --git a/README.md b/README.md index 9f8db21b5..78613a4ec 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,17 @@ # Zeuz Node +--- -An open source cross-platform automation tool + framework that can automate -web, mobile, desktop, rest api & databases. +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -## Minimum requirements +We follow and use the [black](https://github.com/psf/black) (PEP8 compliant, from Python Software Foudndation) code formatter. Please format your code before submitting them to the repository. -- Python 3.7+ is recommended. -- For Windows, we recommend 32-bit Python (required for windows - desktop automation). +```bash +$ pip install black +$ black /path/to/ZeuzPythonNode +``` -## Usage +## Description -1. Install the initial requirements if not already installed. After first - installation, newly added modules in the future will be auto installed. +Client side application for running automated testing. - ```shell - $ pip install -r requiremenets-win.txt - ``` - - - `requirements-win.txt` for Windows. - - `requiremenets-mac.txt` for Mac. - - `requiremenets-linux.txt` for Linux. - -2. Run Zeuz Node. - ```shell - $ python node_cli.py - ``` - -## Docs - -https://automationsolutionz.github.io +To run Zeuz Node in daemon mode (as a background process), execute the `daemon.sh` script. diff --git a/node_cli.py b/node_cli.py index 3be39d2a2..0f59ae98d 100755 --- a/node_cli.py +++ b/node_cli.py @@ -1,17 +1,18 @@ # -*- coding: utf-8 -*- # -*- coding: cp1252 -*- -import os, sys, time, os.path, base64, signal, argparse, requests, json +from Framework.module_installer import install_missing_modules +install_missing_modules() + +import os, sys, time, os.path, base64, signal, argparse, requests, json, io, zipfile, shutil from pathlib import Path from getpass import getpass from urllib3.exceptions import InsecureRequestWarning +from tqdm import tqdm # Suppress the InsecureRequestWarning since we use verify=False parameter. requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) -from Framework.module_installer import install_missing_modules -install_missing_modules() - from Framework.Utilities import ConfigModule PROJECT_ROOT = os.path.abspath(os.curdir) @@ -64,7 +65,7 @@ from Framework.Utilities import ( RequestFormatter, CommonUtil, - FileUtilities, + FileUtilities as FL, All_Device_Info, self_updater ) @@ -407,27 +408,36 @@ def RunProcess(sTesterid, user_info_object): if exit_script: return False if time.time() > etime: + print("30 minutes over, logging in again") return True # Timeout reached, re-login. We do this because after about 3-4 hours this function will hang, and thus not be available for deployment - # r = RequestFormatter.Get("is_run_submitted_api", {"machine_name": sTesterid}) + r = RequestFormatter.Get("is_run_submitted_api", {"machine_name": sTesterid}) Userid = (CommonUtil.MachineInfo().getLocalUser()).lower() - r = requests.get(RequestFormatter.form_uri("getting_json_data_api"), {"machine_name": Userid}, verify=False).json() - # if r and "run_submit" in r and r["run_submit"]: - if r["found"]: - all_run_id_info = r["json"] - # Auto save the json data for local run. - with open(Path.cwd().parent / "Projects" / "Local_run.json", "w") as f: - f.write(json.dumps(all_run_id_info)) - processing_test_case = True + if r and "run_submit" in r and r["run_submit"]: + CommonUtil.ExecLog("", "Downloading dataset and attachments. Please wait", 4) + save_path = temp_ini_file.parent/"attachments" + FL.CreateFolder(save_path) + response = requests.get(RequestFormatter.form_uri("getting_json_data_api"), {"machine_name": Userid}, stream=True) + total_size_in_bytes = int(response.headers.get('content-length', 0)) + chunk_size = 4096 + progress_bar = tqdm(total=total_size_in_bytes, unit='B', mininterval=0, unit_scale=True, miniters=1, leave=True) + with open(save_path/"input.zip", 'wb') as file: + first = True + for data in response.iter_content(chunk_size): + progress_bar.update(len(data)) + file.write(data) + z = zipfile.ZipFile(save_path/"input.zip") + z.extractall(save_path) + z.close() + os.unlink(save_path/"input.zip") CommonUtil.ExecLog( "", - "**************************\n* STARTING SESSION *\n**************************", + "\n**************************\n* STARTING SESSION *\n**************************", 4, False, ) PreProcess() - value = MainDriverApi.main(device_dict, user_info_object, all_run_id_info) - # value = Old_MainDriverApi.main(device_dict, user_info_object) + value = MainDriverApi.main(device_dict, user_info_object) if value == "pass": if exit_script: return False diff --git a/requirements-linux.txt b/requirements-linux.txt index 40c4343c5..49f457850 100644 --- a/requirements-linux.txt +++ b/requirements-linux.txt @@ -31,3 +31,4 @@ attrs pytest==6.1.2 yapf websocket-client +tqdm diff --git a/requirements-mac.txt b/requirements-mac.txt index e847f9c80..18c7ed0a4 100644 --- a/requirements-mac.txt +++ b/requirements-mac.txt @@ -39,3 +39,4 @@ attrs pytest==6.1.2 yapf websocket-client +tqdm diff --git a/requirements-win.txt b/requirements-win.txt index dd39d4f62..eb9eed4cf 100644 --- a/requirements-win.txt +++ b/requirements-win.txt @@ -47,3 +47,4 @@ attrs pytest==6.1.2 yapf websocket-client +tqdm diff --git a/requirements.txt b/requirements.txt index 1785b8615..3a102b18d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ attrs pytest==6.1.2 yapf websocket-client +tqdm