diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 7e6ee062..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "pip" - directory: "/" # Location of your pyproject.toml or requirements.txt - schedule: - interval: "weekly" # Checks for updates every week - commit-message: - prefix: "deps" # Prefix for pull request titles - open-pull-requests-limit: 5 # Limit the number of open PRs at a time diff --git a/.gitignore b/.gitignore index df6d98b8..d65d045e 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ workspaces-flashtaggerviewer **/__pycache__ .DS_Store *-embed-amd64 +get-pip.py +run_app.bat +python* +gdpr_consent/node_modules/ *~ - - diff --git a/LICENSE b/LICENSE index 713de5ec..aac5ba3c 100644 --- a/LICENSE +++ b/LICENSE @@ -13,7 +13,7 @@ This software is released under a three-clause BSD license: * Neither the name of any author or any participating institution may be used to endorse or promote products derived from this software without specific prior written permission. -For a full list of authors, refer to the file AUTHORS. +For a full list of authors, refer to the git contributions. -------------------------------------------------------------------------- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE diff --git a/app.py b/app.py index c19c1e36..d739697f 100644 --- a/app.py +++ b/app.py @@ -36,4 +36,4 @@ } pg = st.navigation(pages, expanded=True) - pg.run() \ No newline at end of file + pg.run() diff --git a/docker-compose.yml b/docker-compose.yml index 450b16e1..20098ba8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: openms-streamlit-template: build: diff --git a/openms-streamlit-vue-component b/openms-streamlit-vue-component index 0a45f30e..fad86f46 160000 --- a/openms-streamlit-vue-component +++ b/openms-streamlit-vue-component @@ -1 +1 @@ -Subproject commit 0a45f30e2223cf2f24392149a4e190ac88e31b27 +Subproject commit fad86f46c3aa788387ec5201fc5cceda5db30fd9 diff --git a/requirements.txt b/requirements.txt index ddeedfcd..0ea13b0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,140 @@ -# the requirements.txt file is intended for deployment on streamlit cloud and if the simple container is built -# note that it is much more restricted in terms of installing third-parties / etc. -# preferably use the batteries included or simple docker file for local hosting -streamlit==1.43.0 -numpy==1.26.4 # pandas and numpy are dependencies of pyopenms, however, pyopenms needs numpy<=1.26.4 -plotly==5.22.0 +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements.txt pyproject.toml +# +altair==5.5.0 + # via streamlit +attrs==25.3.0 + # via + # jsonschema + # referencing +blinker==1.9.0 + # via streamlit +cachetools==5.5.2 + # via streamlit captcha==0.7.1 -pyopenms>=3.2.0 -pyarrow<16 -streamlit-js-eval + # via src (pyproject.toml) +certifi==2025.1.31 + # via requests +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via streamlit +contourpy==1.3.1 + # via matplotlib +cycler==0.12.1 + # via matplotlib +fonttools==4.56.0 + # via matplotlib +gitdb==4.0.12 + # via gitpython +gitpython==3.1.44 + # via streamlit +idna==3.10 + # via requests +jinja2==3.1.6 + # via + # altair + # pydeck +jsonschema==4.23.0 + # via altair +jsonschema-specifications==2024.10.1 + # via jsonschema +kiwisolver==1.4.8 + # via matplotlib +markupsafe==3.0.2 + # via jinja2 +matplotlib==3.10.1 + # via pyopenms +narwhals==1.32.0 + # via altair +numpy==1.26.4 + # via + # contourpy + # matplotlib + # pandas + # pydeck + # pyopenms + # src (pyproject.toml) + # streamlit +packaging==24.2 + # via + # altair + # matplotlib + # plotly + # streamlit +pandas==2.2.3 + # via + # pyopenms + # pyopenms-viz + # streamlit +pillow==11.1.0 + # via + # captcha + # matplotlib + # streamlit +plotly==5.22.0 + # via src (pyproject.toml) +protobuf==5.29.4 + # via streamlit +psutil==7.0.0 + # via src (pyproject.toml) +pyarrow==19.0.1 + # via streamlit +pydeck==0.9.1 + # via streamlit +pyopenms==3.3.0 + # via src (pyproject.toml) +pyopenms-viz==1.0.0 + # via src (pyproject.toml) +pyparsing==3.2.3 + # via matplotlib +python-dateutil==2.9.0.post0 + # via + # matplotlib + # pandas +pytz==2025.2 + # via pandas +referencing==0.36.2 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via streamlit +rpds-py==0.24.0 + # via + # jsonschema + # referencing +six==1.17.0 + # via python-dateutil +smmap==5.0.2 + # via gitdb +streamlit==1.49.0 + # via + # src (pyproject.toml) + # streamlit-js-eval +streamlit-js-eval==0.1.7 + # via src (pyproject.toml) +tenacity==9.0.0 + # via + # plotly + # streamlit +toml==0.10.2 + # via streamlit +tornado==6.4.2 + # via streamlit +typing-extensions==4.13.0 + # via + # altair + # referencing + # streamlit +tzdata==2025.2 + # via pandas +urllib3==2.3.0 + # via requests +watchdog==6.0.0 + # via streamlit +xlsxwriter scipy>=1.15 -psutil diff --git a/src/common/captcha_.py b/src/common/captcha_.py index 288eed03..2da672d1 100644 --- a/src/common/captcha_.py +++ b/src/common/captcha_.py @@ -1,14 +1,23 @@ -from pathlib import Path import streamlit as st import streamlit.components.v1 as st_components -from streamlit.source_util import page_icon_and_name, calc_md5, get_pages, _on_pages_changed +from streamlit.source_util import page_icon_and_name from captcha.image import ImageCaptcha +from pathlib import Path +import hashlib import random import string import os +def calc_md5(string : str): + return hashlib.md5(string.encode()).hexdigest() + +def get_pages(): + return st.runtime.get_pages() + +def set_pages(pages : dict): + st.runtime.set_pages(pages) def delete_all_pages(main_script_path_str: str) -> None: """ @@ -22,7 +31,7 @@ def delete_all_pages(main_script_path_str: str) -> None: """ # Get all pages from the app's configuration - current_pages = get_pages(main_script_path_str) + current_pages = get_pages() # Create a list to store keys pages to delete keys_to_delete = [] @@ -37,7 +46,7 @@ def delete_all_pages(main_script_path_str: str) -> None: del current_pages[key] # Refresh the pages configuration - _on_pages_changed.send() + set_pages(current_pages) def delete_page(main_script_path_str: str, page_name: str) -> None: @@ -52,7 +61,7 @@ def delete_page(main_script_path_str: str, page_name: str) -> None: None """ # Get all pages - current_pages = get_pages(main_script_path_str) + current_pages = get_pages() # Iterate over all pages and delete the desired page if found for key, value in current_pages.items(): @@ -60,7 +69,7 @@ def delete_page(main_script_path_str: str, page_name: str) -> None: del current_pages[key] # Refresh the pages configuration - _on_pages_changed.send() + set_pages(current_pages) def restore_all_pages(main_script_path_str: str) -> None: @@ -74,7 +83,7 @@ def restore_all_pages(main_script_path_str: str) -> None: None """ # Get all pages - pages = get_pages(main_script_path_str) + pages = get_pages() # Obtain the path to the main script main_script_path = Path(main_script_path_str) @@ -126,7 +135,7 @@ def restore_all_pages(main_script_path_str: str) -> None: } # Refresh the page configuration - _on_pages_changed.send() + set_pages(pages) def add_page(main_script_path_str: str, page_name: str) -> None: @@ -141,7 +150,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None: None """ # Get all pages - pages = get_pages(main_script_path_str) + pages = get_pages() # Obtain the path to the main script main_script_path = Path(main_script_path_str) @@ -168,7 +177,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None: } # Refresh the page configuration - _on_pages_changed.send() + set_pages(pages) length_captcha = 5 diff --git a/src/common/common.py b/src/common/common.py index c9c063cc..f42f3e1d 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -1,17 +1,18 @@ -import json import os -import shutil import sys import uuid +import json import time +import psutil +import shutil + +import pandas as pd +import streamlit as st + from typing import Any from pathlib import Path from streamlit.components.v1 import html -import streamlit as st -import pandas as pd -import psutil - try: from tkinter import Tk, filedialog @@ -340,30 +341,25 @@ def render_sidebar(page: str = "") -> None: # The main page has workspace switcher # Display workspace switcher if workspace is enabled in local mode if st.session_state.settings["enable_workspaces"]: - with st.expander("🖥️ **Workspaces**"): - # Workspaces directory specified in the settings.json - if ( - st.session_state.settings["workspaces_dir"] - and st.session_state.location == "local" - ): - workspaces_dir = Path( - st.session_state.settings["workspaces_dir"], - "workspaces-" + st.session_state.settings["repository-name"], - ) - else: - workspaces_dir = ".." - # Online: show current workspace name in info text and option to change to other existing workspace - if st.session_state.location == "local": + # Workspaces directory specified in the settings.json + if ( + st.session_state.settings["workspaces_dir"] + and st.session_state.location == "local" + ): + workspaces_dir = Path( + st.session_state.settings["workspaces_dir"], + "workspaces-" + st.session_state.settings["repository-name"], + ) + else: + workspaces_dir = ".." + # Online: show current workspace name in info text and option to change to other existing workspace + if st.session_state.location == "local": + with st.expander("🖥️ **Workspaces**"): # Define callback function to change workspace def change_workspace(): for key in params.keys(): - if key in ['controllo']: - continue if key in st.session_state.keys(): del st.session_state[key] - st.session_state.workspace = Path( - workspaces_dir, st.session_state["chosen-workspace"] - ) st.query_params.workspace = st.session_state["chosen-workspace"] # Get all available workspaces as options diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 297adf8f..4c469f71 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -77,8 +77,15 @@ def run_command(self, command: list[str]) -> None: self.logger.log(f"Running command:\n"+' '.join(command)+"\nWaiting for command to finish...", 1) start_time = time.time() - # Execute the command - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Execute the command with real-time output capture + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # Line buffered + universal_newlines=True + ) child_pid = process.pid # Record the PID to keep track of running processes associated with this workspace/workflow @@ -86,25 +93,69 @@ def run_command(self, command: list[str]) -> None: pid_file_path = self.pid_dir / str(child_pid) pid_file_path.touch() - # Wait for command completion and capture output - stdout, stderr = process.communicate() + # Real-time output capture + self._stream_output(process) + + # Wait for process completion + process.wait() # Cleanup PID file pid_file_path.unlink() end_time = time.time() execution_time = end_time - start_time - # Format the logging prefix + + # Log completion self.logger.log(f"Process finished:\n"+' '.join(command)+f"\nTotal time to run command: {execution_time:.2f} seconds", 1) - # Log stdout if present - if stdout: - self.logger.log(stdout.decode(), 2) + # Check for errors + if process.returncode != 0: + self.logger.log(f"ERRORS OCCURRED: Process exited with code {process.returncode}", 2) + + def _stream_output(self, process: subprocess.Popen) -> None: + """ + Streams stdout and stderr from a running process in real-time to the logger. + This method runs in the workflow process, not the GUI thread, so it's safe to block. + + Args: + process: The subprocess.Popen object to stream from + """ + def read_stdout(): + """Read stdout in real-time""" + try: + for line in iter(process.stdout.readline, ''): + if line: + self.logger.log(line.rstrip(), 2) + if process.poll() is not None: + break + except Exception as e: + self.logger.log(f"Error reading stdout: {e}", 2) + finally: + process.stdout.close() + + def read_stderr(): + """Read stderr in real-time""" + try: + for line in iter(process.stderr.readline, ''): + if line: + self.logger.log(f"STDERR: {line.rstrip()}", 2) + if process.poll() is not None: + break + except Exception as e: + self.logger.log(f"Error reading stderr: {e}", 2) + finally: + process.stderr.close() + + # Start threads to read stdout and stderr simultaneously + stdout_thread = threading.Thread(target=read_stdout, daemon=True) + stderr_thread = threading.Thread(target=read_stderr, daemon=True) + + stdout_thread.start() + stderr_thread.start() - # Log stderr and raise an exception if errors occurred - if stderr or process.returncode != 0: - error_message = stderr.decode().strip() - self.logger.log(f"ERRORS OCCURRED:\n{error_message}", 2) + # Wait for both threads to complete + stdout_thread.join() + stderr_thread.join() def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> None: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index bb76e9d3..cece3a75 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -1033,45 +1033,71 @@ def execution_section(self, start_workflow_function) -> None: with st.expander("**Summary**"): st.markdown(self.export_parameters_markdown()) - if OS_PLATFORM == 'win32': - self.show_log(start_workflow_function) - return - c1, c2 = st.columns(2) # Select log level, this can be changed at run time or later without re-running the workflow log_level = c1.selectbox( "log details", ["minimal", "commands and run times", "all"], key="log_level" ) - if self.executor.pid_dir.exists(): + + # Real-time display options + if "log_lines_count" not in st.session_state: + st.session_state.log_lines_count = 100 + + log_lines_count = c2.selectbox( + "lines to show", [50, 100, 200, 500, "all"], + index=1, key="log_lines_select" + ) + if log_lines_count != "all": + st.session_state.log_lines_count = log_lines_count + + pid_exists = self.executor.pid_dir.exists() + log_path = Path(self.workflow_dir, "logs", log_level.replace(" ", "-") + ".log") + log_exists = log_path.exists() + + if pid_exists: if c1.button("Stop Workflow", type="primary", use_container_width=True): self.executor.stop() st.rerun() elif c1.button("Start Workflow", type="primary", use_container_width=True): start_workflow_function() - time.sleep(3) - st.rerun() - log_path = Path(self.workflow_dir, "logs", log_level.replace(" ", "-") + ".log") - if log_path.exists(): - if self.executor.pid_dir.exists(): - with st.spinner("**Workflow running...**"): - with open(log_path, "r", encoding="utf-8") as f: - st.code( - "".join(f.readlines()[-30:]), - language="neon", - line_numbers=False, - ) - time.sleep(2) + with st.spinner("**Workflow running...**"): + time.sleep(1) st.rerun() - else: - st.markdown( - f"**Workflow log file: {datetime.fromtimestamp(log_path.stat().st_ctime).strftime('%Y-%m-%d %H:%M')} CET**" - ) + + if log_exists and pid_exists: + # Real-time display during execution + with st.spinner("**Workflow running...**"): with open(log_path, "r", encoding="utf-8") as f: - content = f.read() - # Check if workflow finished successfully - if not "WORKFLOW FINISHED" in content: - st.error("**Errors occurred, check log file.**") - st.code(content, language="neon", line_numbers=False) + lines = f.readlines() + if log_lines_count == "all": + display_lines = lines + else: + display_lines = lines[-st.session_state.log_lines_count:] + st.code( + "".join(display_lines), + language="neon", + line_numbers=False, + ) + # Faster polling for real-time updates + time.sleep(1) + st.rerun() + + elif log_exists and not pid_exists: + # Static display after completion + st.markdown( + f"**Workflow log file: {datetime.fromtimestamp(log_path.stat().st_ctime).strftime('%Y-%m-%d %H:%M')} CET**" + ) + with open(log_path, "r", encoding="utf-8") as f: + content = f.read() + # Check if workflow finished successfully + if not "WORKFLOW FINISHED" in content: + st.error("**Errors occurred, check log file.**") + st.code(content, language="neon", line_numbers=False) + elif pid_exists: + with st.spinner("**Workflow running...**"): + time.sleep(1) + st.rerun() + def results_section(self, custom_results_function) -> None: custom_results_function()