From 0c55a7d5ea341dbab4d9e3d8bfb9703bfbfadd97 Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Sun, 6 Oct 2024 16:18:05 +0600 Subject: [PATCH 01/11] =?UTF-8?q?=F0=9F=8E=89=20add=20mitm=20into=20requir?= =?UTF-8?q?ements=20txt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements-linux.txt | 3 ++- requirements-mac.txt | 3 ++- requirements-win.txt | 3 ++- requirements.txt | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/requirements-linux.txt b/requirements-linux.txt index d12fdcc31..f92ca49dd 100644 --- a/requirements-linux.txt +++ b/requirements-linux.txt @@ -58,4 +58,5 @@ pyperclip thefuzz backports-datetime-fromisoformat; python_version < '3.11' genson -google-cloud-storage \ No newline at end of file +google-cloud-storage +mitmproxy \ No newline at end of file diff --git a/requirements-mac.txt b/requirements-mac.txt index 813082181..6c71e21fc 100644 --- a/requirements-mac.txt +++ b/requirements-mac.txt @@ -62,4 +62,5 @@ pyperclip backports-datetime-fromisoformat; python_version < '3.11' thefuzz genson -google-cloud-storage \ No newline at end of file +google-cloud-storage +mitmproxy \ No newline at end of file diff --git a/requirements-win.txt b/requirements-win.txt index 06f990dbc..a47ace846 100644 --- a/requirements-win.txt +++ b/requirements-win.txt @@ -72,4 +72,5 @@ pyperclip backports-datetime-fromisoformat; python_version < '3.11' thefuzz genson -google-cloud-storage \ No newline at end of file +google-cloud-storage +mitmproxy \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9b1a167ac..3a8d920e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,4 +53,5 @@ pandas pyperclip backports-datetime-fromisoformat; python_version < '3.11' genson -google-cloud-storage \ No newline at end of file +google-cloud-storage +mitmproxy \ No newline at end of file From 4935375784987cafc10728a4fcc0f03bda86d3d0 Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Sun, 6 Oct 2024 16:20:24 +0600 Subject: [PATCH 02/11] =?UTF-8?q?=E2=9C=A8=20mitm=20proxy=20integration=20?= =?UTF-8?q?-=20mitm=20proxy=20server=20py=20file=20added=20-=20Declared=20?= =?UTF-8?q?action=20-=20Added=20start=20and=20stop=20action=20-=20mitm=5Fp?= =?UTF-8?q?roxy=5Fpids=20list=20added=20in=20CommonUtil?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../action_declarations/common.py | 2 + .../Sequential_Actions/common_functions.py | 60 +++++++++++++++++++ .../Sequential_Actions/mitm_proxy.py | 60 +++++++++++++++++++ Framework/Utilities/CommonUtil.py | 2 + 4 files changed, 124 insertions(+) create mode 100644 Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py diff --git a/Framework/Built_In_Automation/Sequential_Actions/action_declarations/common.py b/Framework/Built_In_Automation/Sequential_Actions/action_declarations/common.py index d2e963bf9..d0098821f 100644 --- a/Framework/Built_In_Automation/Sequential_Actions/action_declarations/common.py +++ b/Framework/Built_In_Automation/Sequential_Actions/action_declarations/common.py @@ -131,6 +131,8 @@ {"name": "connect to google service client", "function": "connect_to_google_service_account", "screenshot": "none" }, {"name": "upload to google storage bucket", "function": "upload_to_google_storage_bucket", "screenshot": "none" }, + + {"name": "proxy server", "function": "proxy_server", "screenshot": "none"} ) # yapf: disable diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 04b9332d7..99a880290 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6740,3 +6740,63 @@ def stop_ssh_tunnel(data_set): return "passed" +@logger +def proxy_server(data_set): + import os + sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME + + proxy_var = None + action = None + port = 8080 + for left, mid, right in data_set: + if left.lower().strip() == 'action': + action = 'start' if right.lower().strip() == 'start' else 'stop' + if left.lower().strip() == 'port': + port = int(right.strip()) + if left.lower().strip() == 'proxy server': + proxy_var = right.strip() + + if action == None: + CommonUtil.ExecLog(sModuleInfo, "Incorrect dataset", 3) + return "zeuz_failed" + + if action == 'start': + CommonUtil.ExecLog(sModuleInfo, f"{action.capitalize()}ing proxy server on port {port}", 1) + + proxy_log_dir = Path(sr.Get_Shared_Variables("zeuz_download_folder")).parent / 'proxy_log' + os.makedirs(proxy_log_dir, exist_ok=True) + mitm_proxy_path = Path(__file__).parent / "mitm_proxy.py" + output_file_path = proxy_log_dir / 'mitm.log' # Output file to save the logs + CommonUtil.ExecLog(sModuleInfo, f"Proxy Log file: {output_file_path}", 1) + + captured_network_file_path = proxy_log_dir / 'captured_network_data.csv' + CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {output_file_path}", 1) + # Open the output file in append mode + with open(output_file_path, 'a') as output_file: + # Start the subprocess + process = subprocess.Popen( + ['mitmdump', '-s', mitm_proxy_path, captured_network_file_path], + stdout=output_file, # Redirect stdout to the file + stderr=output_file # Redirect stderr to the file + ) + + pid = process.pid + CommonUtil.mitm_proxy_pids.append(pid) + CommonUtil.ExecLog(sModuleInfo, f"Started process with PID: {pid}", 1) + + sr.Set_Shared_Variables(proxy_var, {"pid":pid,"captured_network_file_path":captured_network_file_path,"log_file":output_file_path}) + return "passed" + else: + import signal + + if CommonUtil.mitm_proxy_pids: + try: + pid = CommonUtil.mitm_proxy_pids[0] + os.kill(pid, signal.SIGTERM) + CommonUtil.ExecLog(sModuleInfo,f"Process with PID {pid} has been terminated.",1) + CommonUtil.mitm_proxy_pids.pop() + except OSError as e: + CommonUtil.ExecLog(sModuleInfo,f"Error: {e}", 3) + + CommonUtil.ExecLog(sModuleInfo, f"{action.capitalize()}ing proxy server on port {port}", 1) + return "passed" \ No newline at end of file diff --git a/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py b/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py new file mode 100644 index 000000000..c338e1488 --- /dev/null +++ b/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py @@ -0,0 +1,60 @@ +from mitmproxy import http +import time +import sys +import csv +import os + +# Initialize a list to track request details +requests_data = [] + +# Get the output file path from command-line arguments +output_file_path = sys.argv[3] if len(sys.argv) > 3 else 'default_output.csv' +print(f"OUTPUT: {output_file_path}") + +# Create a CSV file and write headers if the file does not exist +if not os.path.exists(output_file_path): + with open(output_file_path, 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["url", "status_code", "duration in seconds", "content_length in bytes", "timestamp"]) # Write CSV header + +def request(flow: http.HTTPFlow) -> None: + # Capture request data when it's made + start_time = time.time() + requests_data.append({ + 'url': flow.request.url, + 'start_time': start_time, + 'status_code': None, + 'end_time': None, + 'duration': None, + 'content_length': None + }) + +def response(flow: http.HTTPFlow) -> None: + res = flow.response + end_time = time.time() + + # Find the matching request based on the URL + for req in requests_data: + if req['url'] == flow.request.url: + req['status_code'] = res.status_code + req['end_time'] = end_time + req['duration'] = end_time - req['start_time'] + req['content_length'] = len(res.content) + break + + # Create a list to hold the captured details + captured_details = [ + flow.request.url, + res.status_code, + req.get('duration', None), + len(res.content), + end_time + ] + + # Append the captured details as a row in the CSV file + with open(output_file_path, 'a', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(captured_details) # Write CSV row + + # Optionally print captured details for console output + print(f"Captured: {captured_details}") diff --git a/Framework/Utilities/CommonUtil.py b/Framework/Utilities/CommonUtil.py index 3e8af15b4..0a9f9e647 100644 --- a/Framework/Utilities/CommonUtil.py +++ b/Framework/Utilities/CommonUtil.py @@ -189,6 +189,8 @@ global_sleep = {"selenium":{}, "appium":{}, "windows":{}, "desktop":{}} zeuz_disable_var_print = {} +mitm_proxy_pids = [] + def clear_performance_metrics(): """reset everything to initial value""" global browser_perf, action_perf, step_perf, test_case_perf, perf_test_perf, api_performance_data, load_testing, processed_performance_data From 7bc83e27022f49d94a2408e2ae0cad992d9da66c Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 12:46:56 +0600 Subject: [PATCH 03/11] use PurePosixPath --- .../Sequential_Actions/common_functions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 99a880290..a344b4d4f 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6762,15 +6762,20 @@ def proxy_server(data_set): if action == 'start': CommonUtil.ExecLog(sModuleInfo, f"{action.capitalize()}ing proxy server on port {port}", 1) + from pathlib import PurePosixPath - proxy_log_dir = Path(sr.Get_Shared_Variables("zeuz_download_folder")).parent / 'proxy_log' + proxy_log_dir = PurePosixPath(sr.Get_Shared_Variables("zeuz_download_folder")).parent / 'proxy_log' os.makedirs(proxy_log_dir, exist_ok=True) - mitm_proxy_path = Path(__file__).parent / "mitm_proxy.py" + + # Paths for mitm_proxy and output files + mitm_proxy_path = PurePosixPath(__file__).parent / "mitm_proxy.py" output_file_path = proxy_log_dir / 'mitm.log' # Output file to save the logs + + # Logging the output paths CommonUtil.ExecLog(sModuleInfo, f"Proxy Log file: {output_file_path}", 1) captured_network_file_path = proxy_log_dir / 'captured_network_data.csv' - CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {output_file_path}", 1) + CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {captured_network_file_path}", 1) # Open the output file in append mode with open(output_file_path, 'a') as output_file: # Start the subprocess From 945755236cdfd78bed1d1e17b2ced57c8966cd4e Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 12:54:40 +0600 Subject: [PATCH 04/11] convert to string --- .../Built_In_Automation/Sequential_Actions/common_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index a344b4d4f..65525b4ce 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6780,7 +6780,7 @@ def proxy_server(data_set): with open(output_file_path, 'a') as output_file: # Start the subprocess process = subprocess.Popen( - ['mitmdump', '-s', mitm_proxy_path, captured_network_file_path], + ['mitmdump', '-s', str(mitm_proxy_path), str(captured_network_file_path)], stdout=output_file, # Redirect stdout to the file stderr=output_file # Redirect stderr to the file ) From 62bf881c285ee5c85e9428292f3c0f5d8c52b404 Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 12:58:15 +0600 Subject: [PATCH 05/11] remove subprocess --- .../Sequential_Actions/common_functions.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 65525b4ce..6e4a96ad1 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6777,19 +6777,24 @@ def proxy_server(data_set): captured_network_file_path = proxy_log_dir / 'captured_network_data.csv' CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {captured_network_file_path}", 1) # Open the output file in append mode - with open(output_file_path, 'a') as output_file: - # Start the subprocess - process = subprocess.Popen( - ['mitmdump', '-s', str(mitm_proxy_path), str(captured_network_file_path)], - stdout=output_file, # Redirect stdout to the file - stderr=output_file # Redirect stderr to the file - ) - - pid = process.pid - CommonUtil.mitm_proxy_pids.append(pid) - CommonUtil.ExecLog(sModuleInfo, f"Started process with PID: {pid}", 1) + def start_mitm_proxy(mitm_proxy_path, captured_network_file_path, output_file_path): + # Open the log file in append mode + with open(output_file_path, 'a') as output_file: + # Fork the process + pid = os.fork() + + if pid == 0: # In the child process + # Replace the child process with the mitmdump command + os.execvp('mitmdump', ['mitmdump', '-s', str(mitm_proxy_path), str(captured_network_file_path)]) + else: + # In the parent process, log the PID + CommonUtil.ExecLog(sModuleInfo, f"Started mitmdump process with PID: {pid}", 1) + CommonUtil.mitm_proxy_pids.append(pid) + + # Call the function + start_mitm_proxy(mitm_proxy_path, captured_network_file_path, output_file_path) - sr.Set_Shared_Variables(proxy_var, {"pid":pid,"captured_network_file_path":captured_network_file_path,"log_file":output_file_path}) + sr.Set_Shared_Variables(proxy_var, {"pid":CommonUtil.mitm_proxy_pids[0],"captured_network_file_path":captured_network_file_path,"log_file":output_file_path}) return "passed" else: import signal From ae19a307c038355bf5b50d2a77d8115d3d5c0d0e Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 13:03:33 +0600 Subject: [PATCH 06/11] use os.system --- .../Sequential_Actions/common_functions.py | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 6e4a96ad1..384a87381 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6762,39 +6762,28 @@ def proxy_server(data_set): if action == 'start': CommonUtil.ExecLog(sModuleInfo, f"{action.capitalize()}ing proxy server on port {port}", 1) - from pathlib import PurePosixPath - proxy_log_dir = PurePosixPath(sr.Get_Shared_Variables("zeuz_download_folder")).parent / 'proxy_log' + proxy_log_dir = Path(sr.Get_Shared_Variables("zeuz_download_folder")).parent / 'proxy_log' os.makedirs(proxy_log_dir, exist_ok=True) - - # Paths for mitm_proxy and output files - mitm_proxy_path = PurePosixPath(__file__).parent / "mitm_proxy.py" + mitm_proxy_path = Path(__file__).parent / "mitm_proxy.py" output_file_path = proxy_log_dir / 'mitm.log' # Output file to save the logs - - # Logging the output paths CommonUtil.ExecLog(sModuleInfo, f"Proxy Log file: {output_file_path}", 1) captured_network_file_path = proxy_log_dir / 'captured_network_data.csv' - CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {captured_network_file_path}", 1) + CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {output_file_path}", 1) # Open the output file in append mode - def start_mitm_proxy(mitm_proxy_path, captured_network_file_path, output_file_path): - # Open the log file in append mode - with open(output_file_path, 'a') as output_file: - # Fork the process - pid = os.fork() - - if pid == 0: # In the child process - # Replace the child process with the mitmdump command - os.execvp('mitmdump', ['mitmdump', '-s', str(mitm_proxy_path), str(captured_network_file_path)]) - else: - # In the parent process, log the PID - CommonUtil.ExecLog(sModuleInfo, f"Started mitmdump process with PID: {pid}", 1) - CommonUtil.mitm_proxy_pids.append(pid) + command = f'mitmdump -s {mitm_proxy_path} {captured_network_file_path} -p {port} >> {output_file_path} 2>&1 &' + os.system(command) + + # Retrieve the PID of the process running on the specified port + pid_command = f"lsof -t -i:{port}" + pid = subprocess.check_output(pid_command, shell=True).decode().strip() - # Call the function - start_mitm_proxy(mitm_proxy_path, captured_network_file_path, output_file_path) + # Append the PID to the list and log + CommonUtil.mitm_proxy_pids.append(int(pid)) + CommonUtil.ExecLog(sModuleInfo, f"Started process with PID: {pid}", 1) - sr.Set_Shared_Variables(proxy_var, {"pid":CommonUtil.mitm_proxy_pids[0],"captured_network_file_path":captured_network_file_path,"log_file":output_file_path}) + sr.Set_Shared_Variables(proxy_var, {"pid":pid,"captured_network_file_path":captured_network_file_path,"log_file":output_file_path}) return "passed" else: import signal From 2ce6e14da6987f56820771617d0b9deba168e97a Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 17:15:39 +0600 Subject: [PATCH 07/11] test scraping added --- Framework/test.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 Framework/test.py diff --git a/Framework/test.py b/Framework/test.py new file mode 100644 index 000000000..a3eea49d7 --- /dev/null +++ b/Framework/test.py @@ -0,0 +1,57 @@ +import os +import subprocess +from Utilities import CommonUtil +from pathlib import Path + +OUTPUT_FILEPATH = "/Users/sakib/Documents/zeuz/Zeuz_Python_Node/AutomationLog/debug_sakib_dd446e56-a_AaRlG/session_1/TEST-10894/zeuz_download_folder/proxy_log/mitm.log" +CAPTURED_CSV_FILEPATH = "/Users/sakib/Documents/zeuz/Zeuz_Python_Node/AutomationLog/debug_sakib_dd446e56-a_AaRlG/session_1/TEST-10894/zeuz_download_folder/proxy_log/captured_network_data.csv" +MITM_PROXY_PATH = "/Users/sakib/Documents/zeuz/Zeuz_Python_Node/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py" +PORT = 8080 + + +print(f"Starting proxy server on port {PORT}") + +print(f"MITM Proxy path: {MITM_PROXY_PATH}") +print(f"Proxy Log file: {OUTPUT_FILEPATH}") + +print(f"Captured Network file: {CAPTURED_CSV_FILEPATH}") + +# Open the output file in append mode +with open(OUTPUT_FILEPATH, 'a') as output_file: + # Start the subprocess + process = subprocess.Popen( + ['mitmdump', '-s', MITM_PROXY_PATH, '-w', str(CAPTURED_CSV_FILEPATH), '-p', str(PORT)], + stdout=output_file, # Redirect stdout to the file + stderr=output_file # Redirect stderr to the file + ) + +pid = process.pid + +# Assuming CommonUtil.mitm_proxy_pids is a list, make sure it's initialized properly +if not hasattr(CommonUtil, 'mitm_proxy_pids'): + CommonUtil.mitm_proxy_pids = [] + +CommonUtil.mitm_proxy_pids.append(pid) + +import time +time.sleep(2) + +# Verify if the service is running on the specified port +def verify_port_in_use(port): + if os.name == 'posix': # macOS/Linux + result = subprocess.run(['lsof', '-i', f':{port}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return result.stdout + elif os.name == 'nt': # Windows + result = subprocess.run(['netstat', '-aon'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return result.stdout if f':{port}' in result.stdout else '' + +# Check if the port is in use +port_status = verify_port_in_use(PORT) + +if port_status: + print(f"Service is running on port {PORT}:\n{port_status}") +else: + print(f"Service is NOT running on port {PORT}. Check if the subprocess started correctly.") + +# Prevent the script from exiting immediately +input("Press Enter to exit...\n") \ No newline at end of file From 07b7984f767a5cf314b9ec33323e08b9b3d5610c Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 19:06:42 +0600 Subject: [PATCH 08/11] args added --- .../Sequential_Actions/common_functions.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 384a87381..51f93b25a 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6760,6 +6760,7 @@ def proxy_server(data_set): CommonUtil.ExecLog(sModuleInfo, "Incorrect dataset", 3) return "zeuz_failed" + if action == 'start': CommonUtil.ExecLog(sModuleInfo, f"{action.capitalize()}ing proxy server on port {port}", 1) @@ -6772,15 +6773,16 @@ def proxy_server(data_set): captured_network_file_path = proxy_log_dir / 'captured_network_data.csv' CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {output_file_path}", 1) # Open the output file in append mode - command = f'mitmdump -s {mitm_proxy_path} {captured_network_file_path} -p {port} >> {output_file_path} 2>&1 &' - os.system(command) - - # Retrieve the PID of the process running on the specified port - pid_command = f"lsof -t -i:{port}" - pid = subprocess.check_output(pid_command, shell=True).decode().strip() - - # Append the PID to the list and log - CommonUtil.mitm_proxy_pids.append(int(pid)) + with open(output_file_path, 'a') as output_file: + # Start the subprocess + process = subprocess.Popen( + ['mitmdump', '-s', str(mitm_proxy_path), '-w', str(captured_network_file_path), '-p', str(port)], + stdout=output_file, # Redirect stdout to the file + stderr=output_file # Redirect stderr to the file + ) + + pid = process.pid + CommonUtil.mitm_proxy_pids.append(pid) CommonUtil.ExecLog(sModuleInfo, f"Started process with PID: {pid}", 1) sr.Set_Shared_Variables(proxy_var, {"pid":pid,"captured_network_file_path":captured_network_file_path,"log_file":output_file_path}) From c8d12fdd430c0d81f7736782f6ae920e9dd789f0 Mon Sep 17 00:00:00 2001 From: Sakib75 Date: Wed, 9 Oct 2024 19:58:37 +0600 Subject: [PATCH 09/11] raw path --- .../Sequential_Actions/common_functions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index 51f93b25a..c66be63fe 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6773,13 +6773,13 @@ def proxy_server(data_set): captured_network_file_path = proxy_log_dir / 'captured_network_data.csv' CommonUtil.ExecLog(sModuleInfo, f"Captured Network file: {output_file_path}", 1) # Open the output file in append mode - with open(output_file_path, 'a') as output_file: + with open(r'{}'.format(output_file_path), 'a') as output_file: # Start the subprocess process = subprocess.Popen( - ['mitmdump', '-s', str(mitm_proxy_path), '-w', str(captured_network_file_path), '-p', str(port)], - stdout=output_file, # Redirect stdout to the file - stderr=output_file # Redirect stderr to the file - ) + ['mitmdump', '-s', r'{}'.format(str(mitm_proxy_path)), '-w', r'{}'.format(str(captured_network_file_path)), '-p', str(port)], + stdout=output_file, # Redirect stdout to the file + stderr=output_file # Redirect stderr to the file + ) pid = process.pid CommonUtil.mitm_proxy_pids.append(pid) From 01ed5451640475d84ff141c278818b3c22b7d264 Mon Sep 17 00:00:00 2001 From: sazid Date: Thu, 10 Oct 2024 11:56:56 +0600 Subject: [PATCH 10/11] remove backported datetime module --- requirements-linux.txt | 1 - requirements-mac.txt | 1 - requirements-win.txt | 1 - requirements.txt | 1 - 4 files changed, 4 deletions(-) diff --git a/requirements-linux.txt b/requirements-linux.txt index f92ca49dd..341ffca61 100644 --- a/requirements-linux.txt +++ b/requirements-linux.txt @@ -56,7 +56,6 @@ jinja2 pandas pyperclip thefuzz -backports-datetime-fromisoformat; python_version < '3.11' genson google-cloud-storage mitmproxy \ No newline at end of file diff --git a/requirements-mac.txt b/requirements-mac.txt index 6c71e21fc..b74e1892e 100644 --- a/requirements-mac.txt +++ b/requirements-mac.txt @@ -59,7 +59,6 @@ configobj jinja2 pandas pyperclip -backports-datetime-fromisoformat; python_version < '3.11' thefuzz genson google-cloud-storage diff --git a/requirements-win.txt b/requirements-win.txt index a47ace846..f22d0b91f 100644 --- a/requirements-win.txt +++ b/requirements-win.txt @@ -69,7 +69,6 @@ configobj jinja2 pandas pyperclip -backports-datetime-fromisoformat; python_version < '3.11' thefuzz genson google-cloud-storage diff --git a/requirements.txt b/requirements.txt index 3a8d920e7..48e3b6c22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,6 @@ configobj boto3 pandas pyperclip -backports-datetime-fromisoformat; python_version < '3.11' genson google-cloud-storage mitmproxy \ No newline at end of file From fcdd959af9d6d1f7d2fd39a74c96e7185cf97af3 Mon Sep 17 00:00:00 2001 From: sazid Date: Thu, 10 Oct 2024 14:27:55 +0600 Subject: [PATCH 11/11] Fix mitm script not recognizing output file path properly --- .../Sequential_Actions/common_functions.py | 12 +++++- .../Sequential_Actions/mitm_proxy.py | 41 ++++++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py index c66be63fe..f0eaa26a1 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/common_functions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/common_functions.py @@ -6776,9 +6776,17 @@ def proxy_server(data_set): with open(r'{}'.format(output_file_path), 'a') as output_file: # Start the subprocess process = subprocess.Popen( - ['mitmdump', '-s', r'{}'.format(str(mitm_proxy_path)), '-w', r'{}'.format(str(captured_network_file_path)), '-p', str(port)], + [ + "mitmdump", + "-s", + f"{mitm_proxy_path}", + "-p", + str(port), + "--set", + f"output_file_path={captured_network_file_path}", + ], stdout=output_file, # Redirect stdout to the file - stderr=output_file # Redirect stderr to the file + stderr=output_file, # Redirect stderr to the file ) pid = process.pid diff --git a/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py b/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py index c338e1488..1e63c1a3b 100644 --- a/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py +++ b/Framework/Built_In_Automation/Sequential_Actions/mitm_proxy.py @@ -1,21 +1,16 @@ -from mitmproxy import http +from mitmproxy import http, ctx import time -import sys import csv import os # Initialize a list to track request details requests_data = [] -# Get the output file path from command-line arguments -output_file_path = sys.argv[3] if len(sys.argv) > 3 else 'default_output.csv' -print(f"OUTPUT: {output_file_path}") -# Create a CSV file and write headers if the file does not exist -if not os.path.exists(output_file_path): - with open(output_file_path, 'w', newline='') as csvfile: - writer = csv.writer(csvfile) - writer.writerow(["url", "status_code", "duration in seconds", "content_length in bytes", "timestamp"]) # Write CSV header +def load(l): + # Define the custom option `output_file_path` + ctx.options.add_option("output_file_path", str, "", "Path to output CSV file") + def request(flow: http.HTTPFlow) -> None: # Capture request data when it's made @@ -30,6 +25,12 @@ def request(flow: http.HTTPFlow) -> None: }) def response(flow: http.HTTPFlow) -> None: + output_file_path = ctx.options.output_file_path + create_file_if_not_exists(output_file_path) + + # print("Flow", flow) + # print("Response", flow.response) + res = flow.response end_time = time.time() @@ -58,3 +59,23 @@ def response(flow: http.HTTPFlow) -> None: # Optionally print captured details for console output print(f"Captured: {captured_details}") + +def create_file_if_not_exists(filepath): + """ + Check if the output CSV file exists. + If it does not exist, create the file and add csv headers. + """ + + if not os.path.exists(filepath): + with open(filepath, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow( + [ + "url", + "status_code", + "duration_in_seconds", + "content_length_in_bytes", + "timestamp", + ] + ) + print(f"Created output file: {filepath}")