From 1da8973323608181205163a1ca06409558a02b3a Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 12:22:54 +0000
Subject: [PATCH 01/35] WiP - interaction script

---
 bci/browser/interaction/__init__.py           |  0
 bci/browser/interaction/browsers/__init__.py  |  0
 bci/browser/interaction/browsers/browser.py   | 82 +++++++++++++++++++
 bci/browser/interaction/browsers/chromium.py  | 51 ++++++++++++
 bci/browser/interaction/browsers/firefox.py   | 74 +++++++++++++++++
 bci/browser/interaction/interaction.py        | 13 +++
 bci/evaluations/custom/custom_evaluation.py   | 51 ++++++++----
 .../UserInteraction/a.test/main/headers.json  |  1 +
 .../UserInteraction/a.test/main/index.html    |  6 ++
 .../UserInteraction/interaction_script.cmd    |  1 +
 requirements.in                               |  1 +
 requirements.txt                              |  4 +-
 12 files changed, 269 insertions(+), 15 deletions(-)
 create mode 100644 bci/browser/interaction/__init__.py
 create mode 100644 bci/browser/interaction/browsers/__init__.py
 create mode 100644 bci/browser/interaction/browsers/browser.py
 create mode 100644 bci/browser/interaction/browsers/chromium.py
 create mode 100644 bci/browser/interaction/browsers/firefox.py
 create mode 100644 bci/browser/interaction/interaction.py
 create mode 100644 experiments/pages/Support/UserInteraction/a.test/main/headers.json
 create mode 100644 experiments/pages/Support/UserInteraction/a.test/main/index.html
 create mode 100644 experiments/pages/Support/UserInteraction/interaction_script.cmd

diff --git a/bci/browser/interaction/__init__.py b/bci/browser/interaction/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/bci/browser/interaction/browsers/__init__.py b/bci/browser/interaction/browsers/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/bci/browser/interaction/browsers/browser.py b/bci/browser/interaction/browsers/browser.py
new file mode 100644
index 00000000..a969fcce
--- /dev/null
+++ b/bci/browser/interaction/browsers/browser.py
@@ -0,0 +1,82 @@
+import json
+from abc import ABC, abstractmethod
+
+from websockets.sync.client import ClientConnection, connect
+
+
+# Returns true if:
+# - required == True -> all fields from `pattern` are present in `data` with the same value
+# - required == False -> all fields from `pattern` which are present in `data` have the same value
+def dictionaries_match(pattern: dict, data: dict, required: bool) -> bool:
+    for key in pattern:
+        if required and key not in data:
+            return False
+
+        if key in data:
+            # Equal values, up to slashes
+            if (
+                not dictionaries_match(pattern[key], data[key], required)
+                if isinstance(pattern[key], dict)
+                else str(data[key]).replace('/', '') != str(pattern[key]).replace('/', '')
+            ):
+                return False
+    return True
+
+
+class Browser(ABC):
+    request_id: int = 0
+    ws_timeout: float
+    ws: ClientConnection
+
+    def __init__(self, browser_id: str = '', port: int = 9222, host: str = '127.0.0.1', autoclose_timeout: float = 0.5):
+        self.ws_timeout = autoclose_timeout
+        self.ws = connect(self.get_ws_endpoint(host, port, browser_id), close_timeout=autoclose_timeout)
+        self.initialize_connection(browser_id, port, host)
+
+    def req_id(self) -> int:
+        self.request_id += 1
+        return self.request_id
+
+    def send(self, data: dict) -> dict:
+        data['id'] = self.req_id()
+
+        self.ws.send(json.dumps(data))
+
+        return self.receive({'method': data['method'], 'id': data['id']}, False)
+
+    def listen(self, event: str, params: dict) -> dict:
+        return self.receive({'type': 'event', 'method': event, 'params': params}, True)
+
+    def receive(self, data: dict, required: bool) -> dict:
+        result = None
+
+        while result == None or (not dictionaries_match(data, result, required)):
+            result = json.loads(self.ws.recv(self.ws_timeout))
+
+            if 'type' in result and result['type'] == 'error':
+                raise Exception(f'Received browser error: {result}')
+
+        return result
+
+    # --- BROWSER-SPECIFIC METHODS ---
+    @abstractmethod
+    def get_ws_endpoint(self, host: str, port: int, browser_id: str) -> str:
+        pass
+
+    @abstractmethod
+    def initialize_connection(self, _browserId, _port, _host):
+        pass
+
+    @abstractmethod
+    def close_connection(
+        self,
+    ):
+        pass
+
+    @abstractmethod
+    def navigate(self, _url):
+        pass
+
+    @abstractmethod
+    def click(self, _x, _y):
+        pass
diff --git a/bci/browser/interaction/browsers/chromium.py b/bci/browser/interaction/browsers/chromium.py
new file mode 100644
index 00000000..b82c7014
--- /dev/null
+++ b/bci/browser/interaction/browsers/chromium.py
@@ -0,0 +1,51 @@
+from time import sleep
+
+from .browser import Browser
+
+
+class Chromium(Browser):
+    session_id: str
+
+    def get_ws_endpoint(self, host: str, port: int, browser_id: str) -> str:
+        return f'ws://{host}:{port}/devtools/browser/{browser_id}'
+
+    def initialize_connection(self, browserId, port, host):
+        # Get list of all targets and find a "page" target.
+        target_response = self.send(
+            {
+                'method': 'Target.getTargets',
+            }
+        )
+
+        page_target = list(filter(lambda info: (info['type'] == 'page'), target_response['result']['targetInfos']))[0][
+            'targetId'
+        ]
+
+        # Attach to the page target.
+        session = self.send({'method': 'Target.attachToTarget', 'params': {'targetId': page_target, 'flatten': True}})
+
+        self.session_id = session['result']['sessionId']
+
+    def close_connection(self):
+        pass
+
+    def navigate(self, url):
+        self.send({'sessionId': self.session_id, 'method': 'Page.navigate', 'params': {'url': url}})
+        sleep(0.01)
+
+    def click(self, x, y):
+        self.send(
+            {
+                'sessionId': self.session_id,
+                'method': 'Input.dispatchMouseEvent',
+                'params': {'x': x, 'y': y, 'type': 'mousePressed', 'clickCount': 1, 'button': 'left'},
+            }
+        )
+
+        self.send(
+            {
+                'sessionId': self.session_id,
+                'method': 'Input.dispatchMouseEvent',
+                'params': {'x': x, 'y': y, 'type': 'mouseReleased', 'button': 'left'},
+            }
+        )
diff --git a/bci/browser/interaction/browsers/firefox.py b/bci/browser/interaction/browsers/firefox.py
new file mode 100644
index 00000000..97dd7202
--- /dev/null
+++ b/bci/browser/interaction/browsers/firefox.py
@@ -0,0 +1,74 @@
+from .browser import Browser
+
+
+class Firefox(Browser):
+    browsing_context: str
+
+    def get_ws_endpoint(self, host: str, port: int, browser_id: str) -> str:
+        return f'ws://{host}:{port}/session'
+
+    def initialize_connection(self, browserId, port, host):
+        # Initiate the session
+        session_id = self.send({'method': 'session.new', 'params': {'capabilities': {}}})['result']['sessionId']
+
+        # Subscribe to browser events
+        self.send(
+            {
+                'method': 'session.subscribe',
+                'params': {
+                    'events': [
+                        'browsingContext.domContentLoaded',
+                    ]
+                },
+            }
+        )
+
+        # Create the browsing context
+        user_context = self.send({'method': 'browser.createUserContext', 'params': {}})['result']['userContext']
+
+        self.browsing_context = self.send(
+            {'method': 'browsingContext.create', 'params': {'type': 'tab', 'userContext': user_context}}
+        )['result']['context']
+
+    def close_connection(self):
+        self.send(
+            {
+                'method': 'session.end',
+                'params': {},
+            }
+        )
+
+    def navigate(self, url):
+        navigation = self.send(
+            {'method': 'browsingContext.navigate', 'params': {'url': url, 'context': self.browsing_context}}
+        )['result']['navigation']
+
+        # Wait for the DOM to load
+        self.listen(
+            'browsingContext.domContentLoaded',
+            {
+                'url': url,
+                'context': self.browsing_context,
+                'navigation': navigation,
+            },
+        )
+
+    def click(self, x, y):
+        self.send(
+            {
+                'method': 'input.performActions',
+                'params': {
+                    'context': self.browsing_context,
+                    'actions': [
+                        {
+                            'type': 'pointer',
+                            'id': str(self.req_id()),
+                            'actions': [
+                                {'type': 'pointerDown', 'button': 1, 'width': x, 'height': y},
+                                {'type': 'pointerUp', 'button': 1, 'width': x, 'height': y},
+                            ],
+                        }
+                    ],
+                },
+            }
+        )
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
new file mode 100644
index 00000000..8a014ac1
--- /dev/null
+++ b/bci/browser/interaction/interaction.py
@@ -0,0 +1,13 @@
+from bci.browser.configuration.browser import Browser as BrowserConfig
+
+
+class Interaction:
+    browser_config: BrowserConfig
+    script: list[str]
+
+    def __init__(self, browser: BrowserConfig, script: list[str]) -> None:
+        self.browser_config = browser
+        self.script = script
+
+    def execute(self) -> None:
+        print(f'TODO - execute {self.browser_config._get_terminal_args()} with script {", ".join(self.script)}')
diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index 279065e3..3f235378 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -3,6 +3,7 @@
 import textwrap
 
 from bci.browser.configuration.browser import Browser
+from bci.browser.interaction.interaction import Interaction
 from bci.configuration import Global
 from bci.evaluations.collectors.collector import Collector, Type
 from bci.evaluations.evaluation_framework import EvaluationFramework
@@ -16,7 +17,7 @@ class CustomEvaluationFramework(EvaluationFramework):
     def __init__(self):
         super().__init__()
         self.dir_tree = self.initialize_dir_tree()
-        self.tests_per_project = self.initialize_tests_and_url_queues(self.dir_tree)
+        self.tests_per_project = self.initialize_tests_and_interactions(self.dir_tree)
 
     @staticmethod
     def initialize_dir_tree() -> dict:
@@ -35,7 +36,7 @@ def path_to_dict(path):
         return path_to_dict(path)
 
     @staticmethod
-    def initialize_tests_and_url_queues(dir_tree: dict) -> dict:
+    def initialize_tests_and_interactions(dir_tree: dict) -> dict:
         experiments_per_project = {}
         page_folder_path = Global.custom_page_folder
         for project, experiments in dir_tree.items():
@@ -43,13 +44,29 @@ def initialize_tests_and_url_queues(dir_tree: dict) -> dict:
             project_path = os.path.join(page_folder_path, project)
             experiments_per_project[project] = {}
             for experiment in experiments:
-                url_queue = CustomEvaluationFramework.__get_url_queue(project, project_path, experiment)
-                experiments_per_project[project][experiment] = {
-                    'url_queue': url_queue,
-                    'runnable': CustomEvaluationFramework.is_runnable_experiment(project, experiment, dir_tree),
-                }
+                interaction_script = CustomEvaluationFramework.__get_interaction_script(project_path, experiment)
+                data = {}
+
+                if interaction_script is not None:
+                    data['interaction_script'] = interaction_script
+                else:
+                    url_queue = CustomEvaluationFramework.__get_url_queue(project, project_path, experiment)
+                    data['url_queue'] = url_queue
+
+                data['runnable'] = CustomEvaluationFramework.is_runnable_experiment(project, experiment, dir_tree)
+
+                experiments_per_project[project][experiment] = data
         return experiments_per_project
 
+    @staticmethod
+    def __get_interaction_script(project_path: str, experiment: str) -> list[str] | None:
+        interaction_file_path = os.path.join(project_path, experiment, 'interaction_script.cmd')
+        if os.path.isfile(interaction_file_path):
+            # If an interaction script is specified, it is parsed and used
+            with open(interaction_file_path) as file:
+                return file.readlines()
+        return None
+
     @staticmethod
     def __get_url_queue(project: str, project_path: str, experiment: str) -> list[str]:
         url_queue_file_path = os.path.join(project_path, experiment, 'url_queue.txt')
@@ -88,12 +105,18 @@ def perform_specific_evaluation(self, browser: Browser, params: TestParameters)
 
         is_dirty = False
         try:
-            url_queue = self.tests_per_project[params.evaluation_configuration.project][params.mech_group]['url_queue']
-            for url in url_queue:
-                tries = 0
-                while tries < 3:
-                    tries += 1
-                    browser.visit(url)
+            experiment = self.tests_per_project[params.evaluation_configuration.project][params.mech_group]
+
+            if 'interaction_script' in experiment:
+                interaction = Interaction(browser, experiment['interaction_script'])
+                interaction.execute()
+            else:
+                url_queue = experiment['url_queue']
+                for url in url_queue:
+                    tries = 0
+                    while tries < 3:
+                        tries += 1
+                        browser.visit(url)
         except Exception as e:
             logger.error(f'Error during test: {e}', exc_info=True)
             is_dirty = True
@@ -205,5 +228,5 @@ def include_file_headers(file_type: str) -> bool:
 
     def sync_with_folders(self):
         self.dir_tree = self.initialize_dir_tree()
-        self.tests_per_project = self.initialize_tests_and_url_queues(self.dir_tree)
+        self.tests_per_project = self.initialize_tests_and_interactions(self.dir_tree)
         logger.info('Framework is synced with folders')
diff --git a/experiments/pages/Support/UserInteraction/a.test/main/headers.json b/experiments/pages/Support/UserInteraction/a.test/main/headers.json
new file mode 100644
index 00000000..fe51488c
--- /dev/null
+++ b/experiments/pages/Support/UserInteraction/a.test/main/headers.json
@@ -0,0 +1 @@
+[]
diff --git a/experiments/pages/Support/UserInteraction/a.test/main/index.html b/experiments/pages/Support/UserInteraction/a.test/main/index.html
new file mode 100644
index 00000000..ad482c63
--- /dev/null
+++ b/experiments/pages/Support/UserInteraction/a.test/main/index.html
@@ -0,0 +1,6 @@
+<a href="http://a.test/report/?leak=UserInteraction" style="position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: red;" id="leak">
+    CLICK
+</a>
+<script>
+    document.getElementById('leak').click();
+</script>
\ No newline at end of file
diff --git a/experiments/pages/Support/UserInteraction/interaction_script.cmd b/experiments/pages/Support/UserInteraction/interaction_script.cmd
new file mode 100644
index 00000000..5daa56cf
--- /dev/null
+++ b/experiments/pages/Support/UserInteraction/interaction_script.cmd
@@ -0,0 +1 @@
+NAVIGATE https://a.test/Support/UserInteraction/main
diff --git a/requirements.in b/requirements.in
index b7055ad5..385a39ca 100644
--- a/requirements.in
+++ b/requirements.in
@@ -5,3 +5,4 @@ flatten-dict
 gunicorn
 pymongo
 requests
+websockets
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d2968cc8..a5f13430 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile requirements.in
@@ -54,6 +54,8 @@ urllib3==2.2.3
     # via
     #   docker
     #   requests
+websockets==13.1
+    # via -r requirements.in
 werkzeug==3.0.4
     # via flask
 wsproto==1.2.0

From 5a516ba2765d2dc8bab09293163390ce1c3f8e93 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 14:10:09 +0000
Subject: [PATCH 02/35] Connect browser interaction to terminal automation

---
 bci/browser/automation/terminal.py          | 35 +++++++++++++--------
 bci/browser/configuration/browser.py        | 21 ++++++++++---
 bci/browser/configuration/chromium.py       |  2 ++
 bci/browser/interaction/interaction.py      | 17 ++++++++--
 bci/evaluations/custom/custom_evaluation.py |  8 +++--
 5 files changed, 61 insertions(+), 22 deletions(-)

diff --git a/bci/browser/automation/terminal.py b/bci/browser/automation/terminal.py
index d1fe6f70..ed2235e1 100644
--- a/bci/browser/automation/terminal.py
+++ b/bci/browser/automation/terminal.py
@@ -7,22 +7,31 @@
 
 
 class TerminalAutomation:
-
     @staticmethod
-    def run(url: str, args: list[str], seconds_per_visit: int):
-        logger.debug("Starting browser process...")
+    def visit_url(url: str, args: list[str], seconds_per_visit: int):
         args.append(url)
+        proc, _ = TerminalAutomation.open_browser(args)
+        logger.debug(f'Visiting the page for {seconds_per_visit}s')
+        time.sleep(seconds_per_visit)
+        TerminalAutomation.terminate_browser(proc, args)
+
+    @staticmethod
+    def open_browser(args: list[str]) -> tuple[subprocess.Popen, str]:
+        logger.debug('Starting browser process...')
         logger.debug(f'Command string: \'{" ".join(args)}\'')
-        with open('/tmp/browser.log', 'a') as file:
-            proc = subprocess.Popen(
-                args,
-                stdout=file,
-                stderr=file
-            )
+        with open('/tmp/browser.log', 'a+') as file:
+            proc = subprocess.Popen(args, stdout=file, stderr=file)
+            time.sleep(0.5)
+            position = file.tell()
+            file.seek(0)
+            output = file.read()
+            file.seek(position)
+            return proc, output
 
-        time.sleep(seconds_per_visit)
+    @staticmethod
+    def terminate_browser(proc: subprocess.Popen, args: list[str]) -> None:
+        logger.debug('Terminating browser process using SIGINT...')
 
-        logger.debug(f'Terminating browser process after {seconds_per_visit}s using SIGINT...')
         # Use SIGINT and SIGTERM to end process such that cookies remain saved.
         proc.send_signal(signal.SIGINT)
         proc.send_signal(signal.SIGTERM)
@@ -30,8 +39,8 @@ def run(url: str, args: list[str], seconds_per_visit: int):
         try:
             stdout, stderr = proc.communicate(timeout=5)
         except subprocess.TimeoutExpired:
-            logger.info("Browser process did not terminate after 5s. Killing process through pkill...")
+            logger.info('Browser process did not terminate after 5s. Killing process through pkill...')
             subprocess.run(['pkill', '-2', args[0].split('/')[-1]])
 
         proc.wait()
-        logger.debug("Browser process terminated.")
+        logger.debug('Browser process terminated.')
diff --git a/bci/browser/configuration/browser.py b/bci/browser/configuration/browser.py
index 1178ece1..67419cd4 100644
--- a/bci/browser/configuration/browser.py
+++ b/bci/browser/configuration/browser.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import subprocess
 from abc import abstractmethod
 
 import bci.browser.binary.factory as binary_factory
@@ -15,8 +16,11 @@
 
 
 class Browser:
+    process: subprocess.Popen
 
-    def __init__(self, browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, binary: Binary) -> None:
+    def __init__(
+        self, browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, binary: Binary
+    ) -> None:
         self.browser_config = browser_config
         self.eval_config = eval_config
         self.binary = binary
@@ -34,10 +38,17 @@ def visit(self, url: str):
         match self.eval_config.automation:
             case 'terminal':
                 args = self._get_terminal_args()
-                TerminalAutomation.run(url, args, self.eval_config.seconds_per_visit)
+                TerminalAutomation.visit_url(url, args, self.eval_config.seconds_per_visit)
             case _:
                 raise AttributeError('Not implemented')
 
+    def open(self) -> str:
+        self.process, output = TerminalAutomation.open_browser(self._get_terminal_args())
+        return output
+
+    def terminate(self):
+        TerminalAutomation.terminate_browser(self.process, self._get_terminal_args())
+
     def pre_evaluation_setup(self):
         self.__fetch_binary()
 
@@ -80,11 +91,13 @@ def _get_executable_file_path(self) -> str:
         return os.path.join(self.__get_execution_folder_path(), self.binary.executable_name)
 
     @abstractmethod
-    def _get_terminal_args(self):
+    def _get_terminal_args(self) -> list[str]:
         pass
 
     @staticmethod
-    def get_browser(browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, state: State) -> Browser:
+    def get_browser(
+        browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, state: State
+    ) -> Browser:
         from bci.browser.configuration.chromium import Chromium
         from bci.browser.configuration.firefox import Firefox
 
diff --git a/bci/browser/configuration/chromium.py b/bci/browser/configuration/chromium.py
index 6108328d..3aa0f8e3 100644
--- a/bci/browser/configuration/chromium.py
+++ b/bci/browser/configuration/chromium.py
@@ -1,6 +1,7 @@
 from bci.browser.configuration.browser import Browser
 from bci.browser.configuration.options import Default, BlockThirdPartyCookies, PrivateBrowsing
 from bci.browser.configuration.profile import prepare_chromium_profile
+from bci.browser.interaction.interaction import Interaction
 
 SUPPORTED_OPTIONS = [
     Default(),
@@ -41,6 +42,7 @@ def _get_terminal_args(self) -> list[str]:
         args.append('--enable-logging')
         args.append('--v=1')
         args.append('--log-level=0')
+        args.append(f'--remote-debugging-port={Interaction.port}')
         # Headless changed from version +/- 110 onwards: https://developer.chrome.com/docs/chromium/new-headless
         # Using the `--headless` flag will crash the browser for these later versions.
         # Also see: https://github.com/DistriNet/BugHog/issues/12
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 8a014ac1..1f867a0a 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -1,13 +1,24 @@
+from bci.browser.automation.terminal import TerminalAutomation
 from bci.browser.configuration.browser import Browser as BrowserConfig
 
 
 class Interaction:
-    browser_config: BrowserConfig
+    port = 9222
+
+    browser: BrowserConfig
     script: list[str]
 
     def __init__(self, browser: BrowserConfig, script: list[str]) -> None:
-        self.browser_config = browser
+        self.browser = browser
         self.script = script
 
     def execute(self) -> None:
-        print(f'TODO - execute {self.browser_config._get_terminal_args()} with script {", ".join(self.script)}')
+        output = self.browser.open()
+
+        # TODO - identify browser from the output
+        # TODO - initialize the browser
+        # TODO - parse the script and run the commands
+        # TODO - visit the sanity check URL
+        # TODO - destroy the browser
+
+        self.browser.terminate()
diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index 3f235378..b2d97ef3 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -107,14 +107,18 @@ def perform_specific_evaluation(self, browser: Browser, params: TestParameters)
         try:
             experiment = self.tests_per_project[params.evaluation_configuration.project][params.mech_group]
 
+            max_tries = 3
             if 'interaction_script' in experiment:
                 interaction = Interaction(browser, experiment['interaction_script'])
-                interaction.execute()
+                tries = 0
+                while tries < max_tries:
+                    tries += 1
+                    interaction.execute()
             else:
                 url_queue = experiment['url_queue']
                 for url in url_queue:
                     tries = 0
-                    while tries < 3:
+                    while tries < max_tries:
                         tries += 1
                         browser.visit(url)
         except Exception as e:

From 6cef7597894fef52a5236a8e5521440453b855ff Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 14:26:05 +0000
Subject: [PATCH 03/35] Install pip requirements in dev container

---
 .devcontainer/devcontainer.json | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 8687890c..16cc9c75 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -24,7 +24,10 @@
 				"Vue.volar"
 			]
 		}
-	}
+	},
+
+	// Install pip requirements
+	"postCreateCommand": "pip install -r requirements.txt"
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	// "features": {},
@@ -35,9 +38,6 @@
 	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
 	// "shutdownAction": "none",
 
-	// Uncomment the next line to run commands after the container is created.
-	// "postCreateCommand": "cat /etc/os-release",
-
 	// Configure tool-specific properties.
 	// "customizations": {},
 }

From 688948fcb7748dd5011c355b1c6e716043a0f56d Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 14:43:36 +0000
Subject: [PATCH 04/35] Simple interaction browser identification

---
 bci/browser/interaction/browsers/chromium.py |  2 +-
 bci/browser/interaction/interaction.py       | 25 +++++++++++++++-----
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/bci/browser/interaction/browsers/chromium.py b/bci/browser/interaction/browsers/chromium.py
index b82c7014..7db878af 100644
--- a/bci/browser/interaction/browsers/chromium.py
+++ b/bci/browser/interaction/browsers/chromium.py
@@ -31,7 +31,7 @@ def close_connection(self):
 
     def navigate(self, url):
         self.send({'sessionId': self.session_id, 'method': 'Page.navigate', 'params': {'url': url}})
-        sleep(0.01)
+        sleep(0.5)
 
     def click(self, x, y):
         self.send(
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 1f867a0a..518d5ee4 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -1,5 +1,8 @@
-from bci.browser.automation.terminal import TerminalAutomation
+import re
+
 from bci.browser.configuration.browser import Browser as BrowserConfig
+from bci.browser.interaction.browsers.browser import Browser
+from bci.browser.interaction.browsers.chromium import Chromium
 
 
 class Interaction:
@@ -15,10 +18,20 @@ def __init__(self, browser: BrowserConfig, script: list[str]) -> None:
     def execute(self) -> None:
         output = self.browser.open()
 
-        # TODO - identify browser from the output
-        # TODO - initialize the browser
-        # TODO - parse the script and run the commands
-        # TODO - visit the sanity check URL
-        # TODO - destroy the browser
+        interaction_browser = self._initiate_browser(output)
+
+        # TODO - parse the script and run the commands instead
+        interaction_browser.navigate('https://a.test/Support/UserInteraction/main')
+
+        interaction_browser.navigate('https://a.test/report/?bughog_sanity_check=OK')
+        interaction_browser.close_connection()
 
         self.browser.terminate()
+
+    def _initiate_browser(self, init_output: str) -> Browser:
+        cdp = re.search(r'DevTools listening on ws:\/\/127\.0\.0\.1:9222\/devtools\/browser\/(.+)\n', init_output)
+
+        if cdp:
+            return Chromium(browser_id=cdp.group(1), port=Interaction.port)
+
+        raise Exception('Unrecognized browser')

From b2f1f828e1bbb3b9615b659787f10f8e7bd35661 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 14:48:26 +0000
Subject: [PATCH 05/35] Fix browser log reading

---
 bci/browser/automation/terminal.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/bci/browser/automation/terminal.py b/bci/browser/automation/terminal.py
index ed2235e1..77869206 100644
--- a/bci/browser/automation/terminal.py
+++ b/bci/browser/automation/terminal.py
@@ -20,12 +20,13 @@ def open_browser(args: list[str]) -> tuple[subprocess.Popen, str]:
         logger.debug('Starting browser process...')
         logger.debug(f'Command string: \'{" ".join(args)}\'')
         with open('/tmp/browser.log', 'a+') as file:
+            initial_position = file.tell()
             proc = subprocess.Popen(args, stdout=file, stderr=file)
             time.sleep(0.5)
-            position = file.tell()
-            file.seek(0)
+            last_position = file.tell()
+            file.seek(initial_position)
             output = file.read()
-            file.seek(position)
+            file.seek(last_position)
             return proc, output
 
     @staticmethod

From e8f56a2903c5868ae76ce12ed82b28476ed91743 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 15:19:17 +0000
Subject: [PATCH 06/35] Browser interaction commands interpreter

---
 bci/browser/interaction/browsers/browser.py |  2 ++
 bci/browser/interaction/interaction.py      | 28 +++++++++++++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/bci/browser/interaction/browsers/browser.py b/bci/browser/interaction/browsers/browser.py
index a969fcce..fcee439b 100644
--- a/bci/browser/interaction/browsers/browser.py
+++ b/bci/browser/interaction/browsers/browser.py
@@ -28,6 +28,8 @@ class Browser(ABC):
     ws_timeout: float
     ws: ClientConnection
 
+    public_methods: list[str] = ['navigate', 'click']
+
     def __init__(self, browser_id: str = '', port: int = 9222, host: str = '127.0.0.1', autoclose_timeout: float = 0.5):
         self.ws_timeout = autoclose_timeout
         self.ws = connect(self.get_ws_endpoint(host, port, browser_id), close_timeout=autoclose_timeout)
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 518d5ee4..65b582b0 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -1,9 +1,13 @@
+import logging
 import re
+from inspect import signature
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
 from bci.browser.interaction.browsers.browser import Browser
 from bci.browser.interaction.browsers.chromium import Chromium
 
+logger = logging.getLogger(__name__)
+
 
 class Interaction:
     port = 9222
@@ -20,8 +24,7 @@ def execute(self) -> None:
 
         interaction_browser = self._initiate_browser(output)
 
-        # TODO - parse the script and run the commands instead
-        interaction_browser.navigate('https://a.test/Support/UserInteraction/main')
+        self._interpret(interaction_browser)
 
         interaction_browser.navigate('https://a.test/report/?bughog_sanity_check=OK')
         interaction_browser.close_connection()
@@ -35,3 +38,24 @@ def _initiate_browser(self, init_output: str) -> Browser:
             return Chromium(browser_id=cdp.group(1), port=Interaction.port)
 
         raise Exception('Unrecognized browser')
+
+    def _interpret(self, browser: Browser) -> None:
+        for statement in self.script:
+            cmd, *args = statement.split()
+            method_name = cmd.lower()
+
+            if method_name not in Browser.public_methods:
+                raise Exception(
+                    f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Browser.public_methods))}.'
+                )
+
+            method = getattr(browser, method_name)
+            method_params_len = len(signature(method).parameters)
+
+            if method_params_len != len(args):
+                raise Exception(
+                    f'Invalid number of arguments for command `{cmd}`. Expected {method_params_len}, got {len(args)}.'
+                )
+
+            logger.debug(f'Executing interaction method `{method_name}` with the arguments {args}')
+            method(*args)

From c0e47903e97a42eded3617864d2166275d7af134 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 16:55:44 +0000
Subject: [PATCH 07/35] Basic BiDi implementation

---
 bci/browser/configuration/chromium.py       |  3 +--
 bci/browser/configuration/firefox.py        |  4 ++--
 bci/browser/interaction/browsers/browser.py |  2 +-
 bci/browser/interaction/browsers/firefox.py | 11 ++++++-----
 bci/browser/interaction/interaction.py      | 20 ++++++++++++++++----
 5 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/bci/browser/configuration/chromium.py b/bci/browser/configuration/chromium.py
index 3aa0f8e3..c2d70ea4 100644
--- a/bci/browser/configuration/chromium.py
+++ b/bci/browser/configuration/chromium.py
@@ -1,7 +1,6 @@
 from bci.browser.configuration.browser import Browser
 from bci.browser.configuration.options import Default, BlockThirdPartyCookies, PrivateBrowsing
 from bci.browser.configuration.profile import prepare_chromium_profile
-from bci.browser.interaction.interaction import Interaction
 
 SUPPORTED_OPTIONS = [
     Default(),
@@ -42,7 +41,7 @@ def _get_terminal_args(self) -> list[str]:
         args.append('--enable-logging')
         args.append('--v=1')
         args.append('--log-level=0')
-        args.append(f'--remote-debugging-port={Interaction.port}')
+        args.append('--remote-debugging-port=0')
         # Headless changed from version +/- 110 onwards: https://developer.chrome.com/docs/chromium/new-headless
         # Using the `--headless` flag will crash the browser for these later versions.
         # Also see: https://github.com/DistriNet/BugHog/issues/12
diff --git a/bci/browser/configuration/firefox.py b/bci/browser/configuration/firefox.py
index 643ae57c..c6918d70 100644
--- a/bci/browser/configuration/firefox.py
+++ b/bci/browser/configuration/firefox.py
@@ -2,10 +2,9 @@
 
 from bci import cli
 from bci.browser.configuration.browser import Browser
-from bci.browser.configuration.options import Default, BlockThirdPartyCookies, PrivateBrowsing, TrackingProtection
+from bci.browser.configuration.options import BlockThirdPartyCookies, Default, PrivateBrowsing, TrackingProtection
 from bci.browser.configuration.profile import prepare_firefox_profile
 
-
 SUPPORTED_OPTIONS = [
     Default(),
     BlockThirdPartyCookies(),
@@ -26,6 +25,7 @@ def _get_terminal_args(self) -> list[str]:
 
         args = [self._get_executable_file_path()]
         args.extend(['-profile', self._profile_path])
+        args.append('--remote-debugging-port=0')
         user_prefs = []
 
         def add_user_pref(key: str, value: str | int | bool):
diff --git a/bci/browser/interaction/browsers/browser.py b/bci/browser/interaction/browsers/browser.py
index fcee439b..94dcc5ff 100644
--- a/bci/browser/interaction/browsers/browser.py
+++ b/bci/browser/interaction/browsers/browser.py
@@ -30,7 +30,7 @@ class Browser(ABC):
 
     public_methods: list[str] = ['navigate', 'click']
 
-    def __init__(self, browser_id: str = '', port: int = 9222, host: str = '127.0.0.1', autoclose_timeout: float = 0.5):
+    def __init__(self, browser_id: str = '', port: int = 9222, host: str = '127.0.0.1', autoclose_timeout: float = 2):
         self.ws_timeout = autoclose_timeout
         self.ws = connect(self.get_ws_endpoint(host, port, browser_id), close_timeout=autoclose_timeout)
         self.initialize_connection(browser_id, port, host)
diff --git a/bci/browser/interaction/browsers/firefox.py b/bci/browser/interaction/browsers/firefox.py
index 97dd7202..9f5d6f02 100644
--- a/bci/browser/interaction/browsers/firefox.py
+++ b/bci/browser/interaction/browsers/firefox.py
@@ -46,11 +46,12 @@ def navigate(self, url):
         # Wait for the DOM to load
         self.listen(
             'browsingContext.domContentLoaded',
-            {
-                'url': url,
-                'context': self.browsing_context,
-                'navigation': navigation,
-            },
+            {}
+            #{
+            #    'url': url,
+            #    'context': self.browsing_context,
+            #    'navigation': navigation,
+            #},
         )
 
     def click(self, x, y):
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 65b582b0..a3c98488 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -5,13 +5,12 @@
 from bci.browser.configuration.browser import Browser as BrowserConfig
 from bci.browser.interaction.browsers.browser import Browser
 from bci.browser.interaction.browsers.chromium import Chromium
+from bci.browser.interaction.browsers.firefox import Firefox
 
 logger = logging.getLogger(__name__)
 
 
 class Interaction:
-    port = 9222
-
     browser: BrowserConfig
     script: list[str]
 
@@ -32,10 +31,23 @@ def execute(self) -> None:
         self.browser.terminate()
 
     def _initiate_browser(self, init_output: str) -> Browser:
-        cdp = re.search(r'DevTools listening on ws:\/\/127\.0\.0\.1:9222\/devtools\/browser\/(.+)\n', init_output)
+        print(init_output)
+
+        cdp = re.search(
+            r'DevTools listening on ws:\/\/(.+):(.+)\/devtools\/browser\/(.+)\n',
+            init_output,
+        )
 
         if cdp:
-            return Chromium(browser_id=cdp.group(1), port=Interaction.port)
+            return Chromium(browser_id=cdp.group(3), port=int(cdp.group(2)), host=cdp.group(1))
+
+        bidi = re.search(
+            r'WebDriver BiDi listening on ws:\/\/(.+):(.+)',
+            init_output,
+        )
+
+        if bidi:
+            return Firefox(port=int(bidi.group(2)), host=bidi.group(1))
 
         raise Exception('Unrecognized browser')
 

From 796bf93e22007beaeeae4b518270f19f6e418a22 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Wed, 30 Oct 2024 16:57:12 +0000
Subject: [PATCH 08/35] Remove print

---
 bci/browser/interaction/interaction.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index a3c98488..f0186382 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -31,8 +31,6 @@ def execute(self) -> None:
         self.browser.terminate()
 
     def _initiate_browser(self, init_output: str) -> Browser:
-        print(init_output)
-
         cdp = re.search(
             r'DevTools listening on ws:\/\/(.+):(.+)\/devtools\/browser\/(.+)\n',
             init_output,

From c2c5580561c09294f6082897c3984f74a66c9e6e Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 08:55:56 +0000
Subject: [PATCH 09/35] Simple AutoGUI test experiment

---
 experiments/pages/Support/AutoGUI/a.test/main/headers.json | 1 +
 experiments/pages/Support/AutoGUI/a.test/main/index.html   | 1 +
 experiments/pages/Support/AutoGUI/interaction_script.cmd   | 2 ++
 3 files changed, 4 insertions(+)
 create mode 100644 experiments/pages/Support/AutoGUI/a.test/main/headers.json
 create mode 100644 experiments/pages/Support/AutoGUI/a.test/main/index.html
 create mode 100644 experiments/pages/Support/AutoGUI/interaction_script.cmd

diff --git a/experiments/pages/Support/AutoGUI/a.test/main/headers.json b/experiments/pages/Support/AutoGUI/a.test/main/headers.json
new file mode 100644
index 00000000..fe51488c
--- /dev/null
+++ b/experiments/pages/Support/AutoGUI/a.test/main/headers.json
@@ -0,0 +1 @@
+[]
diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
new file mode 100644
index 00000000..4c9318e8
--- /dev/null
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -0,0 +1 @@
+<a href="https://a.test/report/?leak=AutoGUI" style="position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: red;">CLICK</a>
\ No newline at end of file
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
new file mode 100644
index 00000000..92690174
--- /dev/null
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -0,0 +1,2 @@
+NAVIGATE https://a.test/Support/AutoGUI/main
+CLICK 100 100
\ No newline at end of file

From b8c02dd5be326749c7cadd03504887dd61f28181 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 09:01:13 +0000
Subject: [PATCH 10/35] Install AutoGUI

---
 requirements.in  |  3 ++-
 requirements.txt | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/requirements.in b/requirements.in
index 385a39ca..814ea5bf 100644
--- a/requirements.in
+++ b/requirements.in
@@ -5,4 +5,5 @@ flatten-dict
 gunicorn
 pymongo
 requests
-websockets
\ No newline at end of file
+websockets
+pyautogui
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index a5f13430..c03e9c00 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -38,10 +38,30 @@ markupsafe==3.0.2
     # via
     #   jinja2
     #   werkzeug
+mouseinfo==0.1.3
+    # via pyautogui
 packaging==24.1
     # via gunicorn
+pyautogui==0.9.54
+    # via -r requirements.in
+pygetwindow==0.0.9
+    # via pyautogui
 pymongo==4.10.1
     # via -r requirements.in
+pymsgbox==1.0.9
+    # via pyautogui
+pyperclip==1.9.0
+    # via mouseinfo
+pyrect==0.2.0
+    # via pygetwindow
+pyscreeze==1.0.1
+    # via pyautogui
+python3-xlib==0.15
+    # via
+    #   mouseinfo
+    #   pyautogui
+pytweening==1.2.0
+    # via pyautogui
 requests==2.32.3
     # via
     #   -r requirements.in

From 1444ff2de22fbede606e9c9f3ea64b00c924c1b0 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 10:07:16 +0000
Subject: [PATCH 11/35] Simple AutoGUI usage

---
 Dockerfile                                  |   4 +
 bci/browser/configuration/browser.py        |  13 ++-
 bci/browser/interaction/browsers/browser.py | 106 +++++++-------------
 bci/browser/interaction/interaction.py      |  13 ++-
 requirements.in                             |   6 +-
 requirements.txt                            |  12 ++-
 6 files changed, 67 insertions(+), 87 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b7bd1020..9439b3d8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,7 +54,11 @@ RUN cp /app/scripts/daemon/xvfb /etc/init.d/xvfb
 # Install python packages
 COPY requirements.txt /app/requirements.txt
 RUN pip install --user -r /app/requirements.txt
+RUN apt-get install python3-tk python3-xlib gnome-screenshot -y
 
+# Initiate PyAutoGUI
+RUN touch /root/.Xauthority && \
+    xauth add ${HOST}:0 . $(xxd -l 16 -p /dev/urandom)
 
 FROM base AS core
 # Copy rest of source code
diff --git a/bci/browser/configuration/browser.py b/bci/browser/configuration/browser.py
index 67419cd4..707f53d3 100644
--- a/bci/browser/configuration/browser.py
+++ b/bci/browser/configuration/browser.py
@@ -16,12 +16,13 @@
 
 
 class Browser:
-    process: subprocess.Popen
+    process: subprocess.Popen | None
 
     def __init__(
         self, browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, binary: Binary
     ) -> None:
         self.browser_config = browser_config
+        self.process = None
         self.eval_config = eval_config
         self.binary = binary
         self.state = binary.state
@@ -42,12 +43,18 @@ def visit(self, url: str):
             case _:
                 raise AttributeError('Not implemented')
 
-    def open(self) -> str:
-        self.process, output = TerminalAutomation.open_browser(self._get_terminal_args())
+    def open(self, url: str) -> str:
+        args = self._get_terminal_args()
+        args.append(url)
+        self.process, output = TerminalAutomation.open_browser(args)
         return output
 
     def terminate(self):
+        if self.process is None:
+            return
+
         TerminalAutomation.terminate_browser(self.process, self._get_terminal_args())
+        self.process = None
 
     def pre_evaluation_setup(self):
         self.__fetch_binary()
diff --git a/bci/browser/interaction/browsers/browser.py b/bci/browser/interaction/browsers/browser.py
index 94dcc5ff..cdfec4ae 100644
--- a/bci/browser/interaction/browsers/browser.py
+++ b/bci/browser/interaction/browsers/browser.py
@@ -1,84 +1,46 @@
-import json
-from abc import ABC, abstractmethod
+import base64
+import os
+from io import BytesIO
+from time import sleep
 
-from websockets.sync.client import ClientConnection, connect
+import pyautogui as gui
+import Xlib.display
+from pyvirtualdisplay.display import Display
 
+from bci.browser.configuration.browser import Browser as BrowserConfig
 
-# Returns true if:
-# - required == True -> all fields from `pattern` are present in `data` with the same value
-# - required == False -> all fields from `pattern` which are present in `data` have the same value
-def dictionaries_match(pattern: dict, data: dict, required: bool) -> bool:
-    for key in pattern:
-        if required and key not in data:
-            return False
-
-        if key in data:
-            # Equal values, up to slashes
-            if (
-                not dictionaries_match(pattern[key], data[key], required)
-                if isinstance(pattern[key], dict)
-                else str(data[key]).replace('/', '') != str(pattern[key]).replace('/', '')
-            ):
-                return False
-    return True
-
-
-class Browser(ABC):
-    request_id: int = 0
-    ws_timeout: float
-    ws: ClientConnection
 
+class Browser:
+    browser_config: BrowserConfig
     public_methods: list[str] = ['navigate', 'click']
 
-    def __init__(self, browser_id: str = '', port: int = 9222, host: str = '127.0.0.1', autoclose_timeout: float = 2):
-        self.ws_timeout = autoclose_timeout
-        self.ws = connect(self.get_ws_endpoint(host, port, browser_id), close_timeout=autoclose_timeout)
-        self.initialize_connection(browser_id, port, host)
-
-    def req_id(self) -> int:
-        self.request_id += 1
-        return self.request_id
-
-    def send(self, data: dict) -> dict:
-        data['id'] = self.req_id()
-
-        self.ws.send(json.dumps(data))
-
-        return self.receive({'method': data['method'], 'id': data['id']}, False)
-
-    def listen(self, event: str, params: dict) -> dict:
-        return self.receive({'type': 'event', 'method': event, 'params': params}, True)
-
-    def receive(self, data: dict, required: bool) -> dict:
-        result = None
-
-        while result == None or (not dictionaries_match(data, result, required)):
-            result = json.loads(self.ws.recv(self.ws_timeout))
-
-            if 'type' in result and result['type'] == 'error':
-                raise Exception(f'Received browser error: {result}')
+    def __init__(self, browser_config: BrowserConfig):
+        self.browser_config = browser_config
+        disp = Display(visible=True, size=(1920, 1080), backend='xvfb', use_xauth=True)
+        disp.start()
+        gui._pyautogui_x11._display = Xlib.display.Display(os.environ['DISPLAY'])
 
-        return result
+    def __del__(self):
+        self.browser_config.terminate()
 
-    # --- BROWSER-SPECIFIC METHODS ---
-    @abstractmethod
-    def get_ws_endpoint(self, host: str, port: int, browser_id: str) -> str:
-        pass
+    # --- PUBLIC METHODS ---
+    def navigate(self, url: str):
+        self.browser_config.terminate()
+        self.browser_config.open(url)
 
-    @abstractmethod
-    def initialize_connection(self, _browserId, _port, _host):
-        pass
+        # TODO - convert this into an argument or a separate command
+        sleep(0.5)
 
-    @abstractmethod
-    def close_connection(
-        self,
-    ):
-        pass
+    def click(self, x: str, y: str):
+        # print(gui.size())
+        # print(gui.position())
+        # gui.moveTo(int(x), int(y))
+        gui.moveTo(100, 540)
+        gui.click()
 
-    @abstractmethod
-    def navigate(self, _url):
-        pass
+        # buffered = BytesIO()
+        # print(gui.screenshot().save(buffered, format='JPEG'))
+        # img_str = base64.b64encode(buffered.getvalue())
+        # print(img_str.decode('utf-8'))
 
-    @abstractmethod
-    def click(self, _x, _y):
-        pass
+        sleep(0.5)
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index f0186382..e84c4328 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -19,18 +19,16 @@ def __init__(self, browser: BrowserConfig, script: list[str]) -> None:
         self.script = script
 
     def execute(self) -> None:
-        output = self.browser.open()
-
-        interaction_browser = self._initiate_browser(output)
+        interaction_browser = self._initiate_browser()
 
         self._interpret(interaction_browser)
 
         interaction_browser.navigate('https://a.test/report/?bughog_sanity_check=OK')
-        interaction_browser.close_connection()
-
-        self.browser.terminate()
 
-    def _initiate_browser(self, init_output: str) -> Browser:
+    def _initiate_browser(self) -> Browser:
+        # TODO - possibly return different browser instances
+        return Browser(self.browser)
+        """
         cdp = re.search(
             r'DevTools listening on ws:\/\/(.+):(.+)\/devtools\/browser\/(.+)\n',
             init_output,
@@ -48,6 +46,7 @@ def _initiate_browser(self, init_output: str) -> Browser:
             return Firefox(port=int(bidi.group(2)), host=bidi.group(1))
 
         raise Exception('Unrecognized browser')
+        """
 
     def _interpret(self, browser: Browser) -> None:
         for statement in self.script:
diff --git a/requirements.in b/requirements.in
index 814ea5bf..f0993275 100644
--- a/requirements.in
+++ b/requirements.in
@@ -5,5 +5,7 @@ flatten-dict
 gunicorn
 pymongo
 requests
-websockets
-pyautogui
\ No newline at end of file
+pyautogui
+pyvirtualdisplay
+Pillow
+Xlib
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index c03e9c00..2a367b96 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -42,6 +42,8 @@ mouseinfo==0.1.3
     # via pyautogui
 packaging==24.1
     # via gunicorn
+pillow==11.0.0
+    # via -r requirements.in
 pyautogui==0.9.54
     # via -r requirements.in
 pygetwindow==0.0.9
@@ -62,6 +64,8 @@ python3-xlib==0.15
     #   pyautogui
 pytweening==1.2.0
     # via pyautogui
+pyvirtualdisplay==3.0
+    # via -r requirements.in
 requests==2.32.3
     # via
     #   -r requirements.in
@@ -69,14 +73,16 @@ requests==2.32.3
 simple-websocket==1.1.0
     # via flask-sock
 six==1.16.0
-    # via flatten-dict
+    # via
+    #   flatten-dict
+    #   xlib
 urllib3==2.2.3
     # via
     #   docker
     #   requests
-websockets==13.1
-    # via -r requirements.in
 werkzeug==3.0.4
     # via flask
 wsproto==1.2.0
     # via simple-websocket
+xlib==0.21
+    # via -r requirements.in

From 0acc9c204d9cba8d7862167f7139b9a63be516a4 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 10:17:02 +0000
Subject: [PATCH 12/35] Remove debugging ports

---
 bci/browser/configuration/chromium.py | 1 -
 bci/browser/configuration/firefox.py  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/bci/browser/configuration/chromium.py b/bci/browser/configuration/chromium.py
index c2d70ea4..6108328d 100644
--- a/bci/browser/configuration/chromium.py
+++ b/bci/browser/configuration/chromium.py
@@ -41,7 +41,6 @@ def _get_terminal_args(self) -> list[str]:
         args.append('--enable-logging')
         args.append('--v=1')
         args.append('--log-level=0')
-        args.append('--remote-debugging-port=0')
         # Headless changed from version +/- 110 onwards: https://developer.chrome.com/docs/chromium/new-headless
         # Using the `--headless` flag will crash the browser for these later versions.
         # Also see: https://github.com/DistriNet/BugHog/issues/12
diff --git a/bci/browser/configuration/firefox.py b/bci/browser/configuration/firefox.py
index c6918d70..c336190b 100644
--- a/bci/browser/configuration/firefox.py
+++ b/bci/browser/configuration/firefox.py
@@ -25,7 +25,6 @@ def _get_terminal_args(self) -> list[str]:
 
         args = [self._get_executable_file_path()]
         args.extend(['-profile', self._profile_path])
-        args.append('--remote-debugging-port=0')
         user_prefs = []
 
         def add_user_pref(key: str, value: str | int | bool):

From 840190a8cdd5447039e4c732e32548cae0ffb83d Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 11:14:46 +0000
Subject: [PATCH 13/35] Fix opening Firefox

---
 bci/browser/automation/terminal.py   | 12 +++---------
 bci/browser/configuration/browser.py |  5 ++---
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/bci/browser/automation/terminal.py b/bci/browser/automation/terminal.py
index 77869206..ddc08b8e 100644
--- a/bci/browser/automation/terminal.py
+++ b/bci/browser/automation/terminal.py
@@ -10,24 +10,18 @@ class TerminalAutomation:
     @staticmethod
     def visit_url(url: str, args: list[str], seconds_per_visit: int):
         args.append(url)
-        proc, _ = TerminalAutomation.open_browser(args)
+        proc = TerminalAutomation.open_browser(args)
         logger.debug(f'Visiting the page for {seconds_per_visit}s')
         time.sleep(seconds_per_visit)
         TerminalAutomation.terminate_browser(proc, args)
 
     @staticmethod
-    def open_browser(args: list[str]) -> tuple[subprocess.Popen, str]:
+    def open_browser(args: list[str]) -> subprocess.Popen:
         logger.debug('Starting browser process...')
         logger.debug(f'Command string: \'{" ".join(args)}\'')
         with open('/tmp/browser.log', 'a+') as file:
-            initial_position = file.tell()
             proc = subprocess.Popen(args, stdout=file, stderr=file)
-            time.sleep(0.5)
-            last_position = file.tell()
-            file.seek(initial_position)
-            output = file.read()
-            file.seek(last_position)
-            return proc, output
+            return proc
 
     @staticmethod
     def terminate_browser(proc: subprocess.Popen, args: list[str]) -> None:
diff --git a/bci/browser/configuration/browser.py b/bci/browser/configuration/browser.py
index 707f53d3..0247f28e 100644
--- a/bci/browser/configuration/browser.py
+++ b/bci/browser/configuration/browser.py
@@ -43,11 +43,10 @@ def visit(self, url: str):
             case _:
                 raise AttributeError('Not implemented')
 
-    def open(self, url: str) -> str:
+    def open(self, url: str) -> None:
         args = self._get_terminal_args()
         args.append(url)
-        self.process, output = TerminalAutomation.open_browser(args)
-        return output
+        self.process = TerminalAutomation.open_browser(args)
 
     def terminate(self):
         if self.process is None:

From c85fa2a198af6a45748fbad00606ed692da4e59d Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 11:15:15 +0000
Subject: [PATCH 14/35] Fix opening Firefox

---
 bci/browser/configuration/firefox.py        | 2 ++
 bci/browser/interaction/browsers/browser.py | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/bci/browser/configuration/firefox.py b/bci/browser/configuration/firefox.py
index c336190b..4310bf20 100644
--- a/bci/browser/configuration/firefox.py
+++ b/bci/browser/configuration/firefox.py
@@ -25,6 +25,7 @@ def _get_terminal_args(self) -> list[str]:
 
         args = [self._get_executable_file_path()]
         args.extend(['-profile', self._profile_path])
+        args.append('-setDefaultBrowser')
         user_prefs = []
 
         def add_user_pref(key: str, value: str | int | bool):
@@ -44,6 +45,7 @@ def add_user_pref(key: str, value: str | int | bool):
         # add_user_pref('network.proxy.type', 1)
 
         add_user_pref('app.update.enabled', False)
+        add_user_pref('browser.shell.checkDefaultBrowser', False)
         if 'default' in self.browser_config.browser_setting:
             pass
         elif 'btpc' in self.browser_config.browser_setting:
diff --git a/bci/browser/interaction/browsers/browser.py b/bci/browser/interaction/browsers/browser.py
index cdfec4ae..2d45dafd 100644
--- a/bci/browser/interaction/browsers/browser.py
+++ b/bci/browser/interaction/browsers/browser.py
@@ -29,12 +29,13 @@ def navigate(self, url: str):
         self.browser_config.open(url)
 
         # TODO - convert this into an argument or a separate command
-        sleep(0.5)
+        sleep(2)
 
     def click(self, x: str, y: str):
         # print(gui.size())
         # print(gui.position())
         # gui.moveTo(int(x), int(y))
+
         gui.moveTo(100, 540)
         gui.click()
 

From 44e5dac47981a39b41e81bf5354538c7255781ac Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 12:15:44 +0000
Subject: [PATCH 15/35] Editing config files through the web UI

---
 bci/evaluations/custom/custom_evaluation.py | 17 ++++---
 bci/web/vue/src/components/poc-editor.vue   | 51 ++++++++++++++++++---
 2 files changed, 54 insertions(+), 14 deletions(-)

diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index b2d97ef3..8e81ed90 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -25,11 +25,7 @@ def initialize_dir_tree() -> dict:
 
         def path_to_dict(path):
             if os.path.isdir(path):
-                return {
-                    sub_folder: path_to_dict(os.path.join(path, sub_folder))
-                    for sub_folder in os.listdir(path)
-                    if sub_folder != 'url_queue.txt'
-                }
+                return {sub_folder: path_to_dict(os.path.join(path, sub_folder)) for sub_folder in os.listdir(path)}
             else:
                 return os.path.basename(path)
 
@@ -154,15 +150,22 @@ def get_projects(self) -> list[str]:
     def get_poc_structure(self, project: str, poc: str) -> dict:
         return self.dir_tree[project][poc]
 
+    def _get_poc_file_path(self, project: str, poc: str, domain: str, path: str, file_name: str) -> str:
+        # Top-level config file
+        if domain == 'Config' and path == '_':
+            return os.path.join(Global.custom_page_folder, project, poc, file_name)
+
+        return os.path.join(Global.custom_page_folder, project, poc, domain, path, file_name)
+
     def get_poc_file(self, project: str, poc: str, domain: str, path: str, file_name: str) -> str:
-        file_path = os.path.join(Global.custom_page_folder, project, poc, domain, path, file_name)
+        file_path = self._get_poc_file_path(project, poc, domain, path, file_name)
         if os.path.isfile(file_path):
             with open(file_path) as file:
                 return file.read()
         raise AttributeError(f"Could not find PoC file at expected path '{file_path}'")
 
     def update_poc_file(self, project: str, poc: str, domain: str, path: str, file_name: str, content: str) -> bool:
-        file_path = os.path.join(Global.custom_page_folder, project, poc, domain, path, file_name)
+        file_path = self._get_poc_file_path(project, poc, domain, path, file_name)
         if os.path.isfile(file_path):
             if content == '':
                 logger.warning('Attempt to save empty file ignored')
diff --git a/bci/web/vue/src/components/poc-editor.vue b/bci/web/vue/src/components/poc-editor.vue
index 59e894fb..53a2eaac 100644
--- a/bci/web/vue/src/components/poc-editor.vue
+++ b/bci/web/vue/src/components/poc-editor.vue
@@ -37,6 +37,7 @@
             //     "path1": ["file1", "file2"],
             //     "path2": ["file1"],
             //   },
+            //  'interaction_script.cmd': 'interaction_script.cmd',
             // },
         },
         available_file_types: [
@@ -44,6 +45,10 @@
           'js',
           'py',
         ],
+        poc_tree_config: {
+          domain: 'Config',
+          page: '_',
+        },
         dialog: {
           domain: {
             name: null,
@@ -59,6 +64,33 @@
         timeout: null,
       }
     },
+    computed: {
+      active_poc_tree() {
+        return Object.entries(this.active_poc.tree).reduce((acc, [domain, pages]) => {
+          if (domain !== pages) {
+            return [
+              ...acc,
+              [domain, pages]
+            ]
+          }
+
+          const configDomain = this.poc_tree_config.domain;
+          const configPage = this.poc_tree_config.page;
+
+          // A single top-level file -> create a virtual config folder
+          const config_folder = acc.find(([domain]) => domain === configDomain) ?? [configDomain, {[configPage]: {}}];
+          return [
+            ...acc.filter(([domain]) => domain !== configDomain),
+            [configDomain, {
+              [configPage]: {
+                ...config_folder[1].subfolder,
+                [domain]: pages,
+              }
+            }],
+          ]
+        }, []);
+      }
+    },
     methods: {
       set_active_file(domain, file_path, file_name) {
         this.active_poc.active_domain = domain;
@@ -199,16 +231,21 @@
     <div v-else class="basis-4/12 w-1/12 h-full grow overflow-y-auto">
       <ul class="relative flex flex-col text-gray-700 bg-white shadow-md w-96 rounded-md bg-clip-border dark:bg-dark-3 dark:text-white">
         <nav class="flex min-w-[240px] h-full flex-col gap-1 p-2 font-sans text-base font-normal text-blue-gray-700">
-          <li v-for="([domain, pages, index]) in Object.entries(active_poc.tree)">
+          <li v-for="([domain, pages, index]) in active_poc_tree">
             <ul>
               <li v-for="([path, files, index]) in Object.entries(pages)">
                 <div class="flex border-b-2 p-2 font-bold mb-2" >
-                  <div class="w-full">
-                    {{ domain }}/{{ path }}
-                  </div>
-                  <a :href="'https://'+domain+'/'+this.project+'/'+this.active_poc.name+'/'+path" target="_blank">
-                    <v-icon name="fa-link" class=""/>
-                  </a>
+                  <template v-if="domain !== poc_tree_config.domain">
+                    <div class="w-full">
+                      {{ domain }}/{{ path }}
+                    </div>
+                    <a :href="'https://'+domain+'/'+this.project+'/'+this.active_poc.name+'/'+path" target="_blank">
+                      <v-icon name="fa-link" class=""/>
+                    </a>
+                  </template>
+                  <template v-else>
+                    {{ domain}}
+                  </template>
                 </div>
                 <ul>
                   <li v-for="file in files"

From 025c8ec4824f1ea9df6cded65978c7bc074c3739 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 12:27:47 +0000
Subject: [PATCH 16/35] Interaction script - support blank lines and comments

---
 bci/browser/interaction/interaction.py                   | 9 ++++++---
 experiments/pages/Support/AutoGUI/interaction_script.cmd | 2 ++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index e84c4328..6177983c 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -1,11 +1,8 @@
 import logging
-import re
 from inspect import signature
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
 from bci.browser.interaction.browsers.browser import Browser
-from bci.browser.interaction.browsers.chromium import Chromium
-from bci.browser.interaction.browsers.firefox import Firefox
 
 logger = logging.getLogger(__name__)
 
@@ -50,9 +47,15 @@ def _initiate_browser(self) -> Browser:
 
     def _interpret(self, browser: Browser) -> None:
         for statement in self.script:
+            if statement.strip() == '':
+                continue
+
             cmd, *args = statement.split()
             method_name = cmd.lower()
 
+            if cmd == '#':
+                continue
+
             if method_name not in Browser.public_methods:
                 raise Exception(
                     f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Browser.public_methods))}.'
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index 92690174..26ec60ca 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -1,2 +1,4 @@
 NAVIGATE https://a.test/Support/AutoGUI/main
+
+# test
 CLICK 100 100
\ No newline at end of file

From 32a61492a9360933859b62ddf54b02487caa47f9 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 13:19:20 +0000
Subject: [PATCH 17/35] Interaction script editor code highlighting

---
 bci/web/vue/src/components/poc-editor.vue  |  4 ++
 bci/web/vue/src/interaction_script_mode.js | 47 ++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 bci/web/vue/src/interaction_script_mode.js

diff --git a/bci/web/vue/src/components/poc-editor.vue b/bci/web/vue/src/components/poc-editor.vue
index 53a2eaac..48253620 100644
--- a/bci/web/vue/src/components/poc-editor.vue
+++ b/bci/web/vue/src/components/poc-editor.vue
@@ -4,6 +4,7 @@
   import { Mode as JsMode } from 'ace-builds/src-noconflict/mode-javascript';
   import { Mode as JsonMode } from 'ace-builds/src-noconflict/mode-json';
   import { Mode as PyMode } from 'ace-builds/src-noconflict/mode-python';
+  import { getMode as getInteractionScriptMode } from '../interaction_script_mode';
   import 'ace-builds/src-min-noconflict/ext-modelist';
   import 'ace-builds/src-min-noconflict/theme-twilight';
   import 'ace-builds/src-min-noconflict/theme-xcode';
@@ -175,6 +176,9 @@
           case "py":
             this.editor.session.setMode(new PyMode());
             break;
+          case "cmd":
+            getInteractionScriptMode().then(({ Mode }) => { this.editor.session.setMode(new Mode()) })
+            break;
         }
       },
       add_page() {
diff --git a/bci/web/vue/src/interaction_script_mode.js b/bci/web/vue/src/interaction_script_mode.js
new file mode 100644
index 00000000..fb697e44
--- /dev/null
+++ b/bci/web/vue/src/interaction_script_mode.js
@@ -0,0 +1,47 @@
+const KEYWORDS = "NAVIGATE|CLICK";
+
+ace.define("ace/mode/interaction_script_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module){"use strict";
+    const oop = require("../lib/oop");
+    const TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+    const HighlightRules = function() {
+
+        var keywordMapper = this.createKeywordMapper({
+            "keyword": KEYWORDS,
+        }, "argument", true);
+
+        this.$rules = {
+            "start" : [ {
+                token : "comment",
+                regex : "#.*$"
+            }, {
+                token : "constant.numeric", // float
+                regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+            }, {
+                token : keywordMapper,
+                regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+            } ]
+        };
+        this.normalizeRules();
+    };
+
+    oop.inherits(HighlightRules, TextHighlightRules);
+    exports.HighlightRules = HighlightRules;
+});
+
+ace.define("ace/mode/interaction_script",["require","exports","module","ace/lib/oop","ace/mode/text", "ace/mode/interaction_script_highlight_rules"], function(require, exports){"use strict";
+    const oop = require("ace/lib/oop");
+    const TextMode = require("ace/mode/text").Mode;
+    const HighlightRules = require("ace/mode/interaction_script_highlight_rules").HighlightRules;
+    const Mode = function() {
+        this.HighlightRules = HighlightRules;
+    };
+    oop.inherits(Mode, TextMode);
+    exports.Mode = Mode;
+});
+
+const getMode = () => new Promise((resolve) => ace.require(["ace/mode/interaction_script"], resolve));
+
+export {
+    getMode,
+};
\ No newline at end of file

From b56b17c29602b2d6167b795ef8c734735aa784f1 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 13:58:22 +0000
Subject: [PATCH 18/35] Add new Sleep command

---
 bci/browser/interaction/browsers/__init__.py  |  0
 bci/browser/interaction/browsers/chromium.py  | 51 -------------
 bci/browser/interaction/browsers/firefox.py   | 75 -------------------
 bci/browser/interaction/interaction.py        | 40 +++-------
 .../{browsers/browser.py => simulation.py}    | 19 ++---
 bci/web/vue/src/interaction_script_mode.js    |  2 +-
 .../Support/AutoGUI/interaction_script.cmd    |  4 +-
 7 files changed, 23 insertions(+), 168 deletions(-)
 delete mode 100644 bci/browser/interaction/browsers/__init__.py
 delete mode 100644 bci/browser/interaction/browsers/chromium.py
 delete mode 100644 bci/browser/interaction/browsers/firefox.py
 rename bci/browser/interaction/{browsers/browser.py => simulation.py} (79%)

diff --git a/bci/browser/interaction/browsers/__init__.py b/bci/browser/interaction/browsers/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/bci/browser/interaction/browsers/chromium.py b/bci/browser/interaction/browsers/chromium.py
deleted file mode 100644
index 7db878af..00000000
--- a/bci/browser/interaction/browsers/chromium.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from time import sleep
-
-from .browser import Browser
-
-
-class Chromium(Browser):
-    session_id: str
-
-    def get_ws_endpoint(self, host: str, port: int, browser_id: str) -> str:
-        return f'ws://{host}:{port}/devtools/browser/{browser_id}'
-
-    def initialize_connection(self, browserId, port, host):
-        # Get list of all targets and find a "page" target.
-        target_response = self.send(
-            {
-                'method': 'Target.getTargets',
-            }
-        )
-
-        page_target = list(filter(lambda info: (info['type'] == 'page'), target_response['result']['targetInfos']))[0][
-            'targetId'
-        ]
-
-        # Attach to the page target.
-        session = self.send({'method': 'Target.attachToTarget', 'params': {'targetId': page_target, 'flatten': True}})
-
-        self.session_id = session['result']['sessionId']
-
-    def close_connection(self):
-        pass
-
-    def navigate(self, url):
-        self.send({'sessionId': self.session_id, 'method': 'Page.navigate', 'params': {'url': url}})
-        sleep(0.5)
-
-    def click(self, x, y):
-        self.send(
-            {
-                'sessionId': self.session_id,
-                'method': 'Input.dispatchMouseEvent',
-                'params': {'x': x, 'y': y, 'type': 'mousePressed', 'clickCount': 1, 'button': 'left'},
-            }
-        )
-
-        self.send(
-            {
-                'sessionId': self.session_id,
-                'method': 'Input.dispatchMouseEvent',
-                'params': {'x': x, 'y': y, 'type': 'mouseReleased', 'button': 'left'},
-            }
-        )
diff --git a/bci/browser/interaction/browsers/firefox.py b/bci/browser/interaction/browsers/firefox.py
deleted file mode 100644
index 9f5d6f02..00000000
--- a/bci/browser/interaction/browsers/firefox.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from .browser import Browser
-
-
-class Firefox(Browser):
-    browsing_context: str
-
-    def get_ws_endpoint(self, host: str, port: int, browser_id: str) -> str:
-        return f'ws://{host}:{port}/session'
-
-    def initialize_connection(self, browserId, port, host):
-        # Initiate the session
-        session_id = self.send({'method': 'session.new', 'params': {'capabilities': {}}})['result']['sessionId']
-
-        # Subscribe to browser events
-        self.send(
-            {
-                'method': 'session.subscribe',
-                'params': {
-                    'events': [
-                        'browsingContext.domContentLoaded',
-                    ]
-                },
-            }
-        )
-
-        # Create the browsing context
-        user_context = self.send({'method': 'browser.createUserContext', 'params': {}})['result']['userContext']
-
-        self.browsing_context = self.send(
-            {'method': 'browsingContext.create', 'params': {'type': 'tab', 'userContext': user_context}}
-        )['result']['context']
-
-    def close_connection(self):
-        self.send(
-            {
-                'method': 'session.end',
-                'params': {},
-            }
-        )
-
-    def navigate(self, url):
-        navigation = self.send(
-            {'method': 'browsingContext.navigate', 'params': {'url': url, 'context': self.browsing_context}}
-        )['result']['navigation']
-
-        # Wait for the DOM to load
-        self.listen(
-            'browsingContext.domContentLoaded',
-            {}
-            #{
-            #    'url': url,
-            #    'context': self.browsing_context,
-            #    'navigation': navigation,
-            #},
-        )
-
-    def click(self, x, y):
-        self.send(
-            {
-                'method': 'input.performActions',
-                'params': {
-                    'context': self.browsing_context,
-                    'actions': [
-                        {
-                            'type': 'pointer',
-                            'id': str(self.req_id()),
-                            'actions': [
-                                {'type': 'pointerDown', 'button': 1, 'width': x, 'height': y},
-                                {'type': 'pointerUp', 'button': 1, 'width': x, 'height': y},
-                            ],
-                        }
-                    ],
-                },
-            }
-        )
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 6177983c..4f4e9dd3 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -2,7 +2,7 @@
 from inspect import signature
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
-from bci.browser.interaction.browsers.browser import Browser
+from bci.browser.interaction.simulation import Simulation
 
 logger = logging.getLogger(__name__)
 
@@ -16,36 +16,18 @@ def __init__(self, browser: BrowserConfig, script: list[str]) -> None:
         self.script = script
 
     def execute(self) -> None:
-        interaction_browser = self._initiate_browser()
+        simulation = self._initiate_simulation()
 
-        self._interpret(interaction_browser)
+        self._interpret(simulation)
 
-        interaction_browser.navigate('https://a.test/report/?bughog_sanity_check=OK')
+        simulation.navigate('https://a.test/report/?bughog_sanity_check=OK')
+        simulation.sleep('0.5')
 
-    def _initiate_browser(self) -> Browser:
+    def _initiate_simulation(self) -> Simulation:
         # TODO - possibly return different browser instances
-        return Browser(self.browser)
-        """
-        cdp = re.search(
-            r'DevTools listening on ws:\/\/(.+):(.+)\/devtools\/browser\/(.+)\n',
-            init_output,
-        )
+        return Simulation(self.browser)
 
-        if cdp:
-            return Chromium(browser_id=cdp.group(3), port=int(cdp.group(2)), host=cdp.group(1))
-
-        bidi = re.search(
-            r'WebDriver BiDi listening on ws:\/\/(.+):(.+)',
-            init_output,
-        )
-
-        if bidi:
-            return Firefox(port=int(bidi.group(2)), host=bidi.group(1))
-
-        raise Exception('Unrecognized browser')
-        """
-
-    def _interpret(self, browser: Browser) -> None:
+    def _interpret(self, simulation: Simulation) -> None:
         for statement in self.script:
             if statement.strip() == '':
                 continue
@@ -56,12 +38,12 @@ def _interpret(self, browser: Browser) -> None:
             if cmd == '#':
                 continue
 
-            if method_name not in Browser.public_methods:
+            if method_name not in Simulation.public_methods:
                 raise Exception(
-                    f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Browser.public_methods))}.'
+                    f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Simulation.public_methods))}.'
                 )
 
-            method = getattr(browser, method_name)
+            method = getattr(simulation, method_name)
             method_params_len = len(signature(method).parameters)
 
             if method_params_len != len(args):
diff --git a/bci/browser/interaction/browsers/browser.py b/bci/browser/interaction/simulation.py
similarity index 79%
rename from bci/browser/interaction/browsers/browser.py
rename to bci/browser/interaction/simulation.py
index 2d45dafd..80ff17e5 100644
--- a/bci/browser/interaction/browsers/browser.py
+++ b/bci/browser/interaction/simulation.py
@@ -10,9 +10,9 @@
 from bci.browser.configuration.browser import Browser as BrowserConfig
 
 
-class Browser:
+class Simulation:
     browser_config: BrowserConfig
-    public_methods: list[str] = ['navigate', 'click']
+    public_methods: list[str] = ['navigate', 'click', 'sleep']
 
     def __init__(self, browser_config: BrowserConfig):
         self.browser_config = browser_config
@@ -28,20 +28,17 @@ def navigate(self, url: str):
         self.browser_config.terminate()
         self.browser_config.open(url)
 
-        # TODO - convert this into an argument or a separate command
-        sleep(2)
-
     def click(self, x: str, y: str):
-        # print(gui.size())
-        # print(gui.position())
-        # gui.moveTo(int(x), int(y))
-
         gui.moveTo(100, 540)
         gui.click()
 
+    def sleep(self, duration: str):
+        sleep(float(duration))
+
+    def screenshot(self):
+        # TODO
         # buffered = BytesIO()
         # print(gui.screenshot().save(buffered, format='JPEG'))
         # img_str = base64.b64encode(buffered.getvalue())
         # print(img_str.decode('utf-8'))
-
-        sleep(0.5)
+        pass
diff --git a/bci/web/vue/src/interaction_script_mode.js b/bci/web/vue/src/interaction_script_mode.js
index fb697e44..27fd0d9b 100644
--- a/bci/web/vue/src/interaction_script_mode.js
+++ b/bci/web/vue/src/interaction_script_mode.js
@@ -1,4 +1,4 @@
-const KEYWORDS = "NAVIGATE|CLICK";
+const KEYWORDS = "NAVIGATE|CLICK|SLEEP";
 
 ace.define("ace/mode/interaction_script_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module){"use strict";
     const oop = require("../lib/oop");
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index 26ec60ca..43d2d731 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -1,4 +1,6 @@
 NAVIGATE https://a.test/Support/AutoGUI/main
+SLEEP 0.5
 
 # test
-CLICK 100 100
\ No newline at end of file
+CLICK 100 540
+SLEEP 1
\ No newline at end of file

From 47757967ebe650b2618624e226b2402fc03b07be Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 14:41:45 +0000
Subject: [PATCH 19/35] Clicking on element IDs

---
 bci/browser/interaction/elements/five.png     | Bin 0 -> 4220 bytes
 bci/browser/interaction/elements/four.png     | Bin 0 -> 4220 bytes
 bci/browser/interaction/elements/one.png      | Bin 0 -> 4219 bytes
 bci/browser/interaction/elements/six.png      | Bin 0 -> 4219 bytes
 bci/browser/interaction/elements/three.png    | Bin 0 -> 4220 bytes
 bci/browser/interaction/elements/two.png      | Bin 0 -> 4220 bytes
 bci/browser/interaction/simulation.py         |  20 +++---
 bci/web/vue/src/interaction_script_mode.js    |   2 +-
 .../Support/AutoGUI/a.test/main/index.html    |  63 +++++++++++++++++-
 .../Support/AutoGUI/interaction_script.cmd    |   8 ++-
 experiments/res/bughog.css                    |  58 ++++++++++++++++
 11 files changed, 138 insertions(+), 13 deletions(-)
 create mode 100644 bci/browser/interaction/elements/five.png
 create mode 100644 bci/browser/interaction/elements/four.png
 create mode 100644 bci/browser/interaction/elements/one.png
 create mode 100644 bci/browser/interaction/elements/six.png
 create mode 100644 bci/browser/interaction/elements/three.png
 create mode 100644 bci/browser/interaction/elements/two.png
 create mode 100644 experiments/res/bughog.css

diff --git a/bci/browser/interaction/elements/five.png b/bci/browser/interaction/elements/five.png
new file mode 100644
index 0000000000000000000000000000000000000000..92dfde60004dede0bdc5a2f2505edc5f4fbf4078
GIT binary patch
literal 4220
zcmeHKd2ka|7!MR02vS?5KpB*6ihz{Oo*~JSK%waoFtwDZ9OKB#9_`X>HqCCEv<g}d
zvBeRg6&OIl6O>CqC4fvt4h2P!Q(!C%45A~7&Uh53h;KJNM`xVQ=zmRSv+sTH_x;}Y
z{oeQ9<kb`w<n=cVH5m+s{+@hy5qzh?XJDU1_#2I^+y`F=s(drlA`n4BVObCZJfc>H
zc!ZCN0)$+U3TG!Jr1xpQKG3zG=bq58U8Qx68$Q7-Uu}^_l|`RROZj&D^@L!;>a<v(
z=>4<Y9R83{Ga_%iW7LH6{raQZ=Y2Y?=d7h;SF9?!`sa=7%c_?MGj|U;;#xE>mA)_Q
z7CHHDdgRbsDY-*)7g;{<yY{KfhhO~RkHz~ord}~7zPp9Jb9$|GbM$XX=@m!opQ|}&
z+)y@TQ~&*|KRpX79plf$PS86SH(xx}>%$)34VXD-<UQq&84NuZi`m(Qp6u+_bYK$e
z<}G*SAIKVZ=<(@q6uo`#4P?yfSD2Z}sb#0f418tsrqo&G`0ih_OHRJ<Ud(RV*4sF7
z(B>q|7kikPX~AXVvpW~oy}Go%sp0h#=EDoG&i`=l@3AM0eSRHqYRmIP)~QTkOv+&D
zd@Au%E%MBXg?J(-r0rZ$zvz0?o+Zn+M~IiN&HM5Ams`IXo7m*Ok|wN9y;6SVtz)(O
zE7#jUuH7-^$pYuxlf7$(@7@+`KD_(#J0DRQ0%Oe^mOiMTwB<b_bIs77N0j-_s7D8r
zmd)#S*-PITQ`%V9fY+Z|wJKJiB=;RVCuM+p^d^78{l4Va9v*dQa_w(d@B4H9C)o0-
z>E{xMIKG+oOyc>U7v9-_Gq|<+U~KKc+Ju*|4{j58^{S}OJ@!GvGHGvF&!+X&nftC`
z=c>PZ=wvC1R`pu6t9t48E8o1)W7YVqOLggMcVyfWO}Css&Ck75-@8h#zWnyZ*~;*Z
zcW15HGo{gBT=G$V^^SAS#;?ybUTi2{bNjYIXbQjjPWrSJu<oKFtgji~2N_nDEP#{!
zyd^4yVBr}I&WX_wU`u%w@$<!E&}F{4x512voXh;E)r)&W*?ftZUlr!3R~7i!s#4a$
znI}##Iin0Dka!g!Q7I5qn5fIF^D?llnK3h>L)222d4{(T$(F-BLR)AHj^;$g3er5m
zggC>Tz!bT2TPR@9WiC<G5QAZnNW>DMEONLQBODF~hLadcq7Z>9l|dCm(V&u{QN%gi
zyuyaXkSfYSMB@a0xm<Ob&2SuPjZX@By>0YCr9}m(2Nne(jIiLCBw-yr6g8&;lC%W$
zj~<E-Zd0s?SLE_A%jZ<^K{cZz1;@7ehswhNJspn4_y8|Kr~+ptI!&49@fNmuXc82Q
zQb_lLVs}ESqR=i@r`WU+J)Mq$K=(G@PUzO$budJEy^LFC%eC-4ZkJh`pW$Ry<QTm<
zj-nhk(&|TX+-gH<k|I$5I02w`mbBvlupCa49jH7(MFl~Y*Qg-5MT9(p;HP<q9iS8>
zLup{eQ9xN~bR5Z1oYhVeG$1-qJQNmTRf0grs5B}LQjvCt!v=6W>hR+*8i4?)pR(Ce
zj&NB06wTQwg4L-wmdTaF5`fDoN}!m>LcwBvKogvqROoS;NekX?DGUHrfDSJ6gJQ5e
z+CJbDC4RaJG&Knug;RE#BnZk*SgCm0I^QFFSb>$OaT2(Nvg_{JvM?|iC@j$G6cXrm
zm<y8~=7B1QeX<;InKcST^K2XT!tKNX6}W-QLsFci83Jc0!beycVjN?|GjW2!JK$wb
z6e|A>t!*B}8DH{zQGxSU>ZbUPn$DNSd+}~S)VC6X=$nE8Y&-=8RPdZ0C*+C`u_Yi_
z%)`^8rCeL>;y;vvO|Y>zYqy~`fhJMf%Gyu|E)Xc?Cjig+t+a!Vi`T}k$buRHVSZ9E
z)Ddb0%Tw118LcnW1D#`ul<?XrK*3M~N81G>S_H#d62`PMqcvkE_BTzOI-tWL1O4JQ
zczD6H5NkUOTQr05{-2+gwfH}$K#=Z9x}<NnT-|bYNr5hbyQ`~Pt}ZFiC2)6j{bzES
z+OJc55dI5_z{}EtM~)cbMJv%iHP3ChqkT8*+gJxJeM9*(6@#JQJ=)X5ux8y5Xf&!G
zZ;tU>)9|EZYUyHfE;QZmaZmEuKVzFOaPXFR`01MJq&cToOq-gEroGR_8<P?Z&u8>J
UK0KCp5jq$=IR)-*ljkh>3m-${cmMzZ

literal 0
HcmV?d00001

diff --git a/bci/browser/interaction/elements/four.png b/bci/browser/interaction/elements/four.png
new file mode 100644
index 0000000000000000000000000000000000000000..055ece5b25b3cb0954d9d3f3ca89a44827dd709b
GIT binary patch
literal 4220
zcmeHKeQ?uc7%vDL7Qyi`7#vDgQ29!eG-;d07B<$671lAfs-Sm5n!M@a)}~9+wVT2*
z1|s9m>DU1WLlosh5L84R;2em6%Ap7%2b{RW59GuTJb{4>e3NctJGk?9C;zRzG|BTm
z&+mPn-}`&s>$@Z`cS=9waHGLs=;z6H=fnFjcn$880DoiA<qhz*uiQ6N&1a)XSe68_
z7$9m{7$6`f3ee?LFz<A~eaL;+uMB>m$0X!3{f1|K=*Yz)|3+hQ`q}HWnzHXk`NSq$
z+88AAwXijL@ye9Q^c}sf?Vb5!)0hd?x_{=^^ekKIl50=>b>rm0>hy)b&uU0H?^+ZY
zGGQdrVp-S6`S9?o%d?he<)qdPy4?J@xZ%*HzP~;#6plz4%~ZD(U>`SZz4mC%4pJ@=
zzD=2ba6r?<UIAhD;&YdNp#p3UIs3Cl*P8kE^GUOYTu80CY%ug#BxYpfc``EE%7I1H
z&3)OGy)SLV{@DfV^EW?m0U5XQHD*>)a>+O22ER6WQ}UA$eCOX8g-2i7QBNDU^-i2L
zWJ`bZm%EsH<ASEd7v8H}wr*){bK@IFOb04|nD^1{EA`JN_PI3h*jq0WX~)J3;|2^R
zPbL$8Rw2(Hsl*d_Vc2^$wF}QR?^;~_c9eMKr@6m>_4U@DCnPkx&khq-CZCO*ee-bD
z-m<myCso_0K9}o!=4kIFBX@4AzkXn6(>r^}Q37L~G9q<I-{4z2i1Dk2UmRWHJFb2)
z)MDOJ_aR-heq7O+WsP|4u@x)obCslfC(Icz&^>08|DK0@Nv}Wl$o|Pymw$Nh#=L{r
z%hOU%B&0Zgp7DIb$%~aY_x=&udi~S-)q|_<S%ZDhLfqG@v?A;92aVOi-6cJn*H(;g
z_z62vaen&IA`~s}weZ7=rN1uUaG}SFN7KGvmb!ZTsH>v!s`JlzS>M<8E|)5rHlKb<
z8M*QPv{k#No-rgY-jiLi{e<((cgN41ZVaqyX)y@R^163YXVgI5#YCvDncf_RlY(ZJ
zm;AsS3x=Wa3<l?<SeWICfQtA*Kn%G|f9!5FAtLWGJz@3Y-f#ve6tl}^P*9%h<I0OT
z2XC4*(ddjZ5FiLt7KsImLkbgfnRHwRK5J&ogy=46k;^pGn}=jbGC(LZWyaCWm{@8t
zO*A4-nHQLRcUCI|d~=x!RW;0DSTq_nM@h3J2Qb3naA3Fvvsh5*fhuJom5rexWt0XH
z$8ZCMlf|$qN+CqUWc^Y^b(u_X9BIoh81{PG;X_I*3y=>i#)dJ%jAOwd)-gg+GfN>z
zYeIL7P<*gWv3#IN5t##-r68n^>IlJe?eXD=T&$PFa~LQFLFlT$S&2?lPVsp2+9Nay
z0%9<%M?tbXd8(pti>yw$X(M_$9TS1!?YN!1+iKUHp_kXoxFs&4rRQ<GOxpYmFL5Hz
z=#PYzBCRyfq82MjpcK!MsDrW!C<O#SS~;7aru`kDJRwD8LmbecAh=nCIDQ-FckmR2
zk~q(!l$|3`*2>W+N!fwjM*9IlTRT8Zmqn;bwzy+f8WazqfRz;l8{t50EQzC(0~b&~
z&Dl}PMst9)a0G9$>rgz$WJz+6h07@h*#N-8p@2T15zeINd0Zxo8NX%8D`r&z2DnT)
zVki>3HQ*D2pg?6cHVHe4lQd<)p*}5+)|_>`$APRsC2E)iZYF6xTw4|f76XZ8HJw5L
z-41JEGGxH2lI)YDVwXvSK(xsAVJ~bao>f^ls{#m$TPTLW8Itf3R)(-K6fqtr7`%hO
z#EU}Ne|c-ogE-?$o-HbH{xaPZ@2CP$5`T-o7K?f-A&A}-49mq!P}owy>v=+~_z+jf
zh5`WY9<6e1vx|3-0%^552-@yPY1+!66zjL64glNS<_CbL2&=<ki%X#$U6BMe%E}-;
z0C|L5L3!$2A!GD~8rwOSXd%#60TPB1IC?u_SZl$Uwr8|e?8I*4#Hl-U*koW_+y*x<
zxEEsWn_(+wFyFiRX<duEIR%1rPtqlRyXoqtt4j=Y$+=ry-E?({fi5|BtLuNG%XsTJ
z1w!y&P!t}P`hPy-H+axW@K2lKHr&*{uQqI41}*o7vu7y=L*GH#)x)r=E(IDBRgX6_
z@fYLB{z+p?&mBsFriVQ4bRT_)Q*%GQxVGZJ{3R9r=X_H$V_FtE?0r7|ke*;j-`I2g
U*tzdk!vKRPGuORs@|*?#0yjVDN&o-=

literal 0
HcmV?d00001

diff --git a/bci/browser/interaction/elements/one.png b/bci/browser/interaction/elements/one.png
new file mode 100644
index 0000000000000000000000000000000000000000..c87a2d0cdc24cd8c7f8719bcd209b388273ece85
GIT binary patch
literal 4219
zcmeHKdvFug8c(flF<2-{MG)+Eff2#YKC)>yYa)fDkEp4kL<O7?PIu3yYqQyq-8N~x
zq7^Jsd_fTqidq#xK~!+GfKWh1#o@x`A)}0yTdvQ`6&aOVXW(+rZqnE2jMEwYuSqt$
z`+dLfcfRj;zH?@GNuazm+d9T-u~@QwW!?&O9*vHXLo(5SBEE7LI_;|u&eAJD9E-+O
zNe)9yuZuzqCS(agK34)Kv){w6ybx-7;-S2p`^O#$ymFm>Va%Ce>wkPLd5edgJ~sDI
z>5gnH+PT$te9pFpr*qd%Ts}@=#AnYu(wJF2HEr=jpW;jZwf<P!gXjM#oNAqYm#ri=
z9L&vlJS(vKvBT+)FDY5DXj$Wk)}h}lI@td7rUSJX_jUYaJ=dK1=4Rp2v9-$CN#}><
z*IssDzqd2}FxRp%d++LxJ_U8ptlJCk&6xX9yL(-k`u&92<3`PJPq0|h9+5qsfY0OU
zt_Ic6^1w28*?UD}KbSjneZ{Ne&R~;QxAL=da_1kLJhFBA#@snI)b^h}RY#tBtBtd6
z$;g^|&8A_tzwhANtPh{cdUV^u<`<T(=xTrIFus4`=>_lZ{BPU+Swnug>geVt=%S-j
zq{$;j*+0vre{8}U4=<!LMQQZ5<tr9_)3syqlds3=XTN&jheQ8(?dzK|yS$yFrPaBe
zHJz^<Y}#A*9JjA&>#ct)cind+V@cljEo~R}Z$I_M9{YHSca)CJzh<bi`7L_Nnla}l
z%nu&d4~$}Ln_AxCs@G4h?r3hORvcZms;ykhx#FgIBd+pJ+E|%BF_`n>oi}_iz3Kev
z>;JdlLvq<|`6n{3b$&ggG4r!?3oq?G8+q-*`)zASHl?p4-@Zs+d3kL^$-%eVpHz0v
zzpU%IhAF$gB2P4YcgK-xf~dcI(K`)G|Go0%Gij@CF8Z=Lf9=-szslBMT|X`;`Eo@@
zz1nc<)sy#Vc^ifot=VyFhb3$Ap0b9mCtMw09Pc>U9$ItpqDAV8z3@i<jOD295;AJ9
zS^nF3K~-!(R4bt^p+r&NSuC!pi6{`Np^jC;kQ{O2XLq*am@K;SyB&VYAN9a0xvV}0
zXV#Yoh5Bm2DdJOavAPmGB2b_Xu!IthXnewro4h=_Hq0cBnGn6&jnDE2FpnC87;9r~
z6j7XzYZ?3&E9Q!c5?|phNl~CXH(sUdQJy5@@whE+x2drZNjsfRl43}PArOMl>LNNw
z5D{&>L6PL}LQROtQC(Ien867u)f(N6<H(P7=ch#d{vP^>mQn%fK_)<yq-_+bC}iIV
zO)suRB&mdcH$n@dRZ3PsO|6Luu(%dR^znTuM4=}>S`!PK<%j|a!%#s`4UJ0o8?w~r
z5A;MB5`<(WYDOWk`yq8%>J_VBZidG!r*9%Cyoa|Ry1RB0j8J|*?^T5wBR!wjjT__h
zqAJKDZ(c$mIto}c6O4l^B-qNr0)lhU96<vXP=bRNDVFU+<%?)KhzQW2LgY3X@o<nX
zlmyC7uo6uZtev6=z%UNNNelJ@JLP1gO1cll9WfboB?$M;%AgVv6)kYANHH`4S&2j0
za83fyP#`$AkZ}kMLx}~zq!I<bM2#r`O{c7Y5G13Kkm)c4=cfgHZk(}Ey_P^2=n@KW
z<G0I^nnbT7C@XNL4h%KvLOW&WSh|3AatzJbdqH=>n1(vh;G`*=oijbgwD71HBrGuc
z6cLzqR15EkL7=O#psI%5xRDZOME1D-XgP^M2VS5<L`pF%PgA^|4$=-D{T$pBisq?4
zcvY08x<5i2iwAQhr@Tzo(D-$xDY>F%!uiR&<ZW0smlB4Vi-HG2vIGs(Leb0<aV1?s
z6^Mi&+C5V3+HIGArxgSz3uuvJ2&Vub!Lkx+1<CFtI6$$5tP>U%P*Q((O_lUGh{0(g
zq$AP_wWp~SHp!f*8~dZ<RnV9PBp5+cL`tv%o}yENk*R`7W6$WW*hT(E6PF3-v&o>i
zqz!FeXfGstHp7%=#<uepeo}Ms7mk3a2L~CDzJqcN$~7Pb1_U1Lu0gp5q`-i{gWdI~
z$z|<*oq`ec6ck4<OQVKMH=-A<%*xwJy_QSH_xW8Lnvvy-XxVJdVi|h1aim$+v|Njf
zS-Q_(ob|0WZ&*&6r$YE1nI`(Y(}LV5LU`W1u0ut_p7)EgsQW(NIPt}mkIi9{7g)L_
YIwWnw?#A@pD8S+?F86MkKJVfG11VAHA^-pY

literal 0
HcmV?d00001

diff --git a/bci/browser/interaction/elements/six.png b/bci/browser/interaction/elements/six.png
new file mode 100644
index 0000000000000000000000000000000000000000..4adc77b4003a1af837a070b2424e6520c2c73d2e
GIT binary patch
literal 4219
zcmeHKYj6`)6iz9$5FQpNP#BeM5XItVA9=95CQ@kn2#``rM2h3cX7_HowAoFw+a|42
zsUT=!Kqvx21p!gyJ&4+ZSU|xiDg~z^Lt$VPA1E-Ql#z;eH+_!GIGxddO=e%`e&@UA
zeCOOdyD#KqXZAOaFd7Vo{_ZSSF1!whM`G_dc#ecu9fX%7W!{-;E*nOIAz2U$0iu=$
z0RkeT03qK7a?bSs0O@z@M&dm&;|-Pp;?`6BnR3r%@5W!;)gu=zp_6uBij@X0=&@q^
zsu`2Wtg4K&gFH*i@%!m-77rb<Yi??viXBIXo}P>r-M)O$KW~wg_kO?nr+S}B*%z62
zy&$3VeC>(PZzP@aB~<mjwD4@*OOsBPH6O9p8-I_jay7jD%ujQE{pEBcyXo-9pRCE7
z8?!!h-k*s}UOqnHv)@j<c}yC%Kl%34OpN@4lsD*`bDjo+A?7(TEiK2Lme!gL%%OVj
za%a{DsdpcJa{89sZTDP8Qr2u_@)Cv=ok~gEIC1NcS;ctm^|bu@7x&dr#$CPQ#}C~;
z!2HF2rpEZ}`S|DGnqReP@ye#U*S<0xn}4zVy#qIDo{I1N`=FCMmJ+Ea#|kL}hgnV!
zA+A;;3%{C=$MM4Ow^poNaH(nkqGh|o#QN{%{`%#Ko!>tc*W|h|Tv#*YLh*$+k5?Wp
z-9R0w+&k&%Y{#7XUN4NS-BokzSnc^ehb*H6#+G?^^3cA49s7u}Ye!tWugLqg`spyz
zyuJEes$fe>L1R@NzVhVi)iv2lLZ63b4;<tgv(*<n+MBTX(FcxBto-BRy?>Q|ge{+(
z+z^*Ue?M(u-05rcZy&xQ?Y#A2&AP<O*jKQ3nu&frODZyszf-p?aG<D1(}s$%2fxD_
zDt??=Ux1=zJr}%NvH0gzuV0Q?{c!5Js^oQhN8J>SHyu~YGtRB-RVG)Q-*)D4W#k)o
zrLNsSsnHO>=ulS0-UdhG*{>VV)cMyoHyebe(55}f(^kN?i-@qjW_qSDoE$K-yzB$!
zNFWIN&R}qij|5q+0H}x$_(jQSx^ke-gowP;^q9?qdxB{oU(6~Cf$3%0UaqWwqj}T#
zM~sdL0|^2^WsyjrP*RwP)1>n<@Ln@xCPas*1y0jUPY#kMhX7(VTg^C{9uZ4O(<4U2
z5#j|V*Ok#i0iT?vd{qrH7#0qP&0&jK4*4;HrfCc(F_J_f0#!;Sm5rd1GD@R}a=3uP
zg~XsL$`Yb+vOc+3b(&0Y9BGX&5cGK3=q05^1*iuWVS^Z9#<4&E>*%4V=_Qb)C7?Tc
zC|<Zqv0R|Y#UT!)mjFo})sced+WdpXp+Y?!p2I*P2tcR;XC*pKnd$cAw0UR}_{Bg_
z_kv<~LaL(BE>@@5v=Kd>j)6e;Hr`I?*4%Y4M0q@nOXiBT@Z2t^Nt>VHWlrQ7{gwoP
zvjQKFk{oA4t%P7leH=%j6aaRMjkfbRL3W^WONz=$9MGsBxmkofHlL67kphq6G{K=(
ziegckCwP?N2$}*s4uG{I#ng}pyOJ&J7?noFLn_irkru0uMhOD&kV;@tpKY8Kwb*?Y
z-Xhpp%8u*N@EnsNhXO1trx;-U01HZfeLxeOnULdlnn-gLs#|gjSyg}zPSX@oDvq>6
zy<z}NS6NL>Vw?rHP*##8aLR78TG~M~KuCd|sBsdw*+S_fT3HyF3>22t`V<oAc9;v3
z76Pm)hrF^}=rm~oAev{}um>(Do>f^Fs{%-hlU9bn84KYhYz#4u!SS&;0nhDtnHPo9
ze?x1F2XRD8o+T=9{!-l(T~X6PQS>SLSSadC2|@Hl!LVF31%)jEygo1FiVku4tmFr9
z_h@O?R=aqIR^ae)gpGoY#L*<IpN%9?)<W4(0p~~yZMV{bmF~>0$buSXLtug*>Ik)h
z?Wt>pjL{4AU}tnVA81v8f}sSCwhKnI2!^#JjA?sDYsL=jADTFHK!;5R`bBMU^MZRJ
z*0ve8Xr^sD|KqQv7XRZ6kh*)4F6rAXSGQbUQlLxV?(XWAt4j)W3EbUX|CwCI_RlFG
z!MC6={8+lVw5JPxXvO&^XSxixwg0OJ->8C?KEbR!#bD?=SUX}2Ypau>F<y0h(&K+H
zjvO#BW|lj5J2Z`UyC!(4&p3YenC352xkDeO#^ZB7**bdjswK0?=nWEUNO~fs`lB8z
RYN3O{ou2L5HF5T{e*=mw>$v~`

literal 0
HcmV?d00001

diff --git a/bci/browser/interaction/elements/three.png b/bci/browser/interaction/elements/three.png
new file mode 100644
index 0000000000000000000000000000000000000000..4368269edf52192df25060f30b8029e7640fd91c
GIT binary patch
literal 4220
zcmeHKYj6`)6izL)l2R=4Y8@WSRs>r%`^amvB(%_!HezZmQF;83&F<ZHZL*tYw@q4s
zu@tFP-bGNT4@49npi@x_$P}r9?+56hG8P6!bOZqhwI~JeZrYNL&N!Wse@$kyyXSuA
zyXSo8+&j6;ii-+Tjkg#L21BZ+&|L!W8Sol@V-oz0M%M3xx3(JJY_)`qAfd1<hyj48
z)ggd@s3<_lH^Jh0L)wtRf1Msa<%V}^Z#%TPeB#-pDOaX97J*F@n(EgK^8EGlrNUha
zSxqgb!#m#y%m-VgWqd(S3qSR%l+^7Xb^7Q5%_Aotm?>6VJ+om|ZM`&SU;6v5C6yyB
zW06Y~dQ<wa6$?@_Q!`hmoL=&6>tmZf2zMUXb2{VX#8m$ihpxOh>R9`|XUC+So%G(p
zhdK=D^~3&5Tb?kl{?u<jTDZkapTNF9{#4O(_Ma}#8O?BQD-DK(hsC_SVozRPcRDbM
zriE)<g>AW`-@Sj<){>XToI$c4-^|QO8&UB^*6_{Kw~d%vN$kFuS9W;y&KBCZV_<U5
z$d`whKH0;x7#E*Ne&qGK#%CX0*U|diXXwGY?-sqa_m7qblW+X(<|D7HBy*2s3t2-)
znZF)Eo^L>wepW{$@j}MyYu7C~-La>B)vFQmsZ$Go`SkPcKTJ;QaG%T&9v^YC^5lyj
zH@sc_G<~3<`Hlre&Ib+;TsCg^j+V;@cb|A|zd2K2tOcXTk4y=^vXji-c*})bD|}z7
zAB>_*FE{O?%eQ8gw>P#D>yB*L&{CwN4VpZE=*{j)+x&?Wd}&+mnfUJXhTp#%du7of
ze9g@9N0ZVWKis`E>FW!1SKmG-ZNL0Z%ckKCiBIBhc9Me!RMpP-_|4W;!MznXbUa;~
zz3&u$wDzaF4wqwC&44AlY9IZ1{qtuMHcZJq-Z*|!bLJ({c*%Kw(TwBk2G+>6Ctf->
zPZ{^ZO}QKQ+|h1GuHRo++kDj7{_U6T$68A_c6J(sj_|XujlX*>?7OH4`)juMPKJ|%
zCYG1|z!VLJVB;AK&YWn7<;sDI_(7>CxzKZaTTw*hUFdyQFX0X4fikhMCJbiP6#2NC
za?Zh{Ia7_!C<6(CKxL6=Fd!*R)P?H241CtiIEv^HwcLfy_7)>~au^^Mlf^_}`BAZo
zLZ=!LXP6h568DTQ3i#$i%TzVQ;CLhwF-6QKIb4d94u=CLD4e1&h`^L;NoAv$q-1Io
zF%CCSxUd*fMOi{LPS!71sxB0T<4AXW!I0P6LoX>^DnLE(C>z2_6M+YVc)W+A=2t<I
zu7Lj4L-E07ikARIt_*V^zY0idW;_MY_4tP>!vQ@Vp2I-^1R+#`vy#20Ebw@XdptA=
zO2uGE_kv>gLaL&0O{`wAX(M_%@qs}19^PK)?%Z`SM0vf8TjnaY@H}o8s?E>vGAHtk
z{z#E_KToiBjHU!LX0g*0#?pQo<5`CQ>jXGjFvn4OBt>N<4ro-6+$2ICyOs8H0w-X+
zU?nh%)goZ5*-Bv~Z6Se~ru={x;wbJ4i?A!%Kzvjh6%VOwgq5~ftTv42>@;SfZIFs2
zY#3xAd4~Xiz}s~yo?~Xn;UEjkDF)e6fQO_~eLxeOnO5v^p_GZZW+@J^ssJ5a=$)ce
z8ND{(6N6xu%4%wob~9myF*^tgMcZw0-uHsA0y|OTBngw5*4?$TFfbV?EUWb?B+%_J
z7bY(ZSXB=DWI5nMwU7|avuD@~mlMybteaH<Bqb;dLlTUc^pRGEv@w(;n;;n?4lnbf
zQ2k$MZSf$^Sjh`T1<qfsn_??!7O055#a;uVzLXF|Ula_>#Zpk%D!}V;Lax{lSH?=E
z0PY@L?b>Y@|D_dp8%J6xzkmr22ka)B-;c2@OJjoFMghS_@jM&Ls)t>X1vSEk!L(AS
zBh(7Er>+$;NiWpxy<>@#0j&y9FpMOy9>L5+mtc5T!nn3)bZ6|u|DlOf2gGeM&@X0#
zn-|;*@t)1FOEVbn_55_z;(AVjApMi{N#B0C`sM1A0(}DacUQk$eNv!L;QsFV-{dl0
zJ5B)!{tJr0!;)o7*(rF?O7hPva2u{_-)Hu{&<HJqLWOe_gCS*@b|n}#Hl;&jvg+~X
zC;w<1HzaNBCv&B7&@{p0p5~)J<St(={xv6YMZ&PeG5fbo*s^~4T<U@MV-Jf58wOuY
VM4}aSSD=HzlV9ZCF@65xzX8jg>YM-o

literal 0
HcmV?d00001

diff --git a/bci/browser/interaction/elements/two.png b/bci/browser/interaction/elements/two.png
new file mode 100644
index 0000000000000000000000000000000000000000..7f3c24a9f8e0025be98b28a48ba76ed630ac2c1f
GIT binary patch
literal 4220
zcmeHKYj6`)6b=>=h_-;VKszAYEdo+E`^ctA)<mF59}!bZi3;kBY<BOaYqQxjyKT~<
zV0nlw_@WgmMFkZ>t*D?<idaAq$G3nm%2*jxz!61!U|~SKoAfz4<8(&<HJQ!sp8K8e
zp7Y&v&*at?7ZoHMGmJW&F4<G)E`j%Sc%=^2!{1nR^)7hZTkV^xmatJI9FYYv2oSX@
z3=j|#1qk_4D!wBnDRbz>lc~<dSG0!5Z7-{DU%wwW9&MH;m&aD7k39AA$)r$HV|q)l
z<n_b+e6UZbomkMcZq%%Exvme&CQ!@oJB+UQcH^OzMdyCqzuUE~`OcN!UbFWtW5)3H
z_oQzyWQ=<7;?S(*tlTjl4{KVKeamA%URbtkQ`&KZ{?%sg@}VZ_%#?E}nU(K1tgL;<
zu)cik=Hxw%?;U2V9)E&KfBVDHzsR#DA4@*fIxlt7h4E8#x<SiCm#f(0a&=_`b69up
zqt3#&r;por*PM+dFN{BpOl{o2%o~wberRgyhTP3*w^tB5esh%`c;vMfn{mq!!?e*a
zrkFl%Wm=4jj~O1=wxs^)6%8G2&wh%&yX2dNZ|?k~Wr1Pn?^hpeeu$iYFk6^9a*X+l
zH1g*<<o-{W5PDun-?pk@>B)}P<qy9cC7=B6-k(1C?4=Xe>pR@X(}l*g;}yrB|EO+H
z)l;^;bz5(`r^s>lfg!aMc5G?6`0kEluk1Ew35=y+T;}L2rRLYj?6nzZCzks@SKl8)
znO<DC-Bz}7YFT@I8_{rZ&6<`XWyG-S=a0PFJ!P{$=~~~2XKtCiFSqX8H`n~Ra6kU&
z%*-SDvGx<Q@7I5EcFE;EXF@Mse5a);wJzxi{EbWGRf8*Q@;`c`?O|zW`4t^c)nxDb
z4nI=!{j39J7*;)a>GqlxKdgT4^q@62O#iw*vuSJAdC_>@@$<s`uN#I`%QeSdIC_UN
zVbk#GYg=z>*BO@YF09#l#L@oM=j}(^0&6c_(g_`rr(emOy$Y6HOoZh%*LyR=$&!iX
zWj`>*q%f>Ioz5{W7G}9Jpdx+{5JOJ%%+59x5qT$io5f3b!!A%N7FI{Voa!PUS6#;0
zd34&1Mn{Z+1QJkLBqjwz3KMgpab5;KYi1lp;t;jWiO%&FBQ7}t5ZXkW2rMrqR#NDV
zM#K@}1*XKE-$?=AoM@@4h8Y}>Mx&;v*(65-IBB=rae~4r3WEqtsS2rV3=1h)8byM`
z4HPaShE-7xAsQ#^mn&2!io$WED?Tag^>)*Tlui|(9(arm<D`keB?<5Cp{RM4kfbx9
zfAmm%aF^mGK#?mV9LTE#AvLQv1<!T+hbtn%cse|XgCLL~RDrXSeWon%c#FF|GzkKt
z6pnjAvHKuZQRoq?Pi)#qJe}TwK=*FmKIpF8<6wyLdKtIORcPUP+)h-RpW$Ur<e9{y
z1rQufSuh?DBt}#A9E|0G9WzrLokLi12#PlMqVj|kl?`z~qk`ln5%Lg}P2kKxz|0)M
zVl+<+n9ag-kcTA!P4g6+L-wMW6%k=ovccX_X;eI<5(pdZw~-`fw~~;`#@R4GM*vKq
zc&p7yQ$VoT;#54x<jWC>h07^QYyjZlP#`{_3C_$Y_Bc_>MD$pSgRCk*2Pb;77^;Z%
z4ERI|%u!iQP10&6%r=^|(j-CWSgk#vTR}vDm8fx&gvp!;L0c9ECIf|KwK|0aaSxaa
z<B9-Ql_Ne`4mwdSBt-M<9`?fR#Iq{vW>o-5p~ehJFlN$6S{O2iA*pPFWQbmPnHPns
ze?x1V2XQ2pyiio&{8e#NVn@vZ<%ze%Yfy}DB?O6Y3WntpDJX0u;Nx*ZuEY>m%7y{}
zc8|_-?XrviPzqL#Bst1rhJ{4<F`A@uFn<oLCL3+H2&9$r6RgmeU6BPf%0|G90Mrp`
z1<Nz86*48hP}lWEM@xaW3Q#bNB(NUA=uW}#&V+HTXLM!k!2hO+BM#`*WT0Qd2Adb`
zg?M)}?9@zaJOAUSb1nYI86fq*B>mENK&}C~`lUd>zysAaAXmQ>=ofgPy8bh{j6KIG
z5Q6`LqVTZPxTjV-@9X_D3*5TP+IRh~P4&<+EL=EG(dn)nrCoz`YuAm1MuY0{<{3^I
zC!~x>Ir!tk1<*9f<DTKOeZalB?abvx`i!JWdgi{_GxM?Z*ZD)M5)Xsn4Cez$yN6W_
RPlFyhPhOFGOYZ!|e*qpQ<cR<P

literal 0
HcmV?d00001

diff --git a/bci/browser/interaction/simulation.py b/bci/browser/interaction/simulation.py
index 80ff17e5..c87fa39b 100644
--- a/bci/browser/interaction/simulation.py
+++ b/bci/browser/interaction/simulation.py
@@ -12,7 +12,7 @@
 
 class Simulation:
     browser_config: BrowserConfig
-    public_methods: list[str] = ['navigate', 'click', 'sleep']
+    public_methods: list[str] = ['navigate', 'click', 'click_el', 'sleep', 'screenshot']
 
     def __init__(self, browser_config: BrowserConfig):
         self.browser_config = browser_config
@@ -29,16 +29,20 @@ def navigate(self, url: str):
         self.browser_config.open(url)
 
     def click(self, x: str, y: str):
-        gui.moveTo(100, 540)
+        gui.moveTo(int(x), int(y))
         gui.click()
 
+    def click_el(self, el_id: str):
+        el_image_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), f'elements/{el_id}.png')
+        x, y = gui.locateCenterOnScreen(el_image_path)
+        self.click(str(x), str(y))
+
     def sleep(self, duration: str):
         sleep(float(duration))
 
     def screenshot(self):
-        # TODO
-        # buffered = BytesIO()
-        # print(gui.screenshot().save(buffered, format='JPEG'))
-        # img_str = base64.b64encode(buffered.getvalue())
-        # print(img_str.decode('utf-8'))
-        pass
+        # TODO save to the filesystem
+        buffered = BytesIO()
+        print(gui.screenshot().save(buffered, format='JPEG'))
+        img_str = base64.b64encode(buffered.getvalue())
+        print(img_str.decode('utf-8'))
diff --git a/bci/web/vue/src/interaction_script_mode.js b/bci/web/vue/src/interaction_script_mode.js
index 27fd0d9b..35785b44 100644
--- a/bci/web/vue/src/interaction_script_mode.js
+++ b/bci/web/vue/src/interaction_script_mode.js
@@ -1,4 +1,4 @@
-const KEYWORDS = "NAVIGATE|CLICK|SLEEP";
+const KEYWORDS = "NAVIGATE|CLICK|CLICK_EL|SLEEP|SCREENSHOT";
 
 ace.define("ace/mode/interaction_script_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module){"use strict";
     const oop = require("../lib/oop");
diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index 4c9318e8..fabffa1b 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -1 +1,62 @@
-<a href="https://a.test/report/?leak=AutoGUI" style="position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: red;">CLICK</a>
\ No newline at end of file
+<link rel="stylesheet" href="https://a.test/res/bughog.css">
+<style>
+.fullscreen {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background-color: red;
+    z-index: 10;
+}
+
+#one, #two, #three, #four, #five, #six {
+    position: fixed;
+    width: 25px;
+    height: 20px;
+    font-size: 0;
+}
+
+#one {
+    background-color: #800000;
+    color: #800000;
+    top: 10px;
+    left: 10px;
+}
+
+#two {
+    background-color: #dcbeff;
+    color: #dcbeff;
+    top: 45px;
+    left: 10px;
+}
+
+#three {
+    background-color: #ffe119;
+    color: #ffe119;
+    top: 80px;
+    left: 10px;
+}
+
+#four {
+    background-color: #4363d8;
+    color: #4363d8;
+    top: 10px;
+    left: 45px;
+}
+
+#five {
+    background-color: #f58231;
+    color: #f58231;
+    top: 45px;
+    left: 45px;
+}
+
+#six {
+    background-color: #000075;
+    color: #000075;
+    top: 80px;
+    left: 45px;
+}
+</style>
+<a href="https://a.test/report/?leak=AutoGUI" id="three">CLICK</a>
\ No newline at end of file
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index 43d2d731..8aaa6b70 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -1,6 +1,8 @@
 NAVIGATE https://a.test/Support/AutoGUI/main
-SLEEP 0.5
+SLEEP 1
 
-# test
-CLICK 100 540
+SCREENSHOT
+
+# Comment
+CLICK_EL three
 SLEEP 1
\ No newline at end of file
diff --git a/experiments/res/bughog.css b/experiments/res/bughog.css
new file mode 100644
index 00000000..10be4b2d
--- /dev/null
+++ b/experiments/res/bughog.css
@@ -0,0 +1,58 @@
+.fullscreen {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background-color: red;
+    z-index: 10;
+}
+
+#one, #two, #three, #four, #five, #six {
+    position: fixed;
+    width: 25px;
+    height: 20px;
+    font-size: 0;
+}
+
+#one {
+    background-color: #800000;
+    color: #800000;
+    top: 10px;
+    left: 10px;
+}
+
+#two {
+    background-color: #dcbeff;
+    color: #dcbeff;
+    top: 45px;
+    left: 10px;
+}
+
+#three {
+    background-color: #ffe119;
+    color: #ffe119;
+    top: 80px;
+    left: 10px;
+}
+
+#four {
+    background-color: #4363d8;
+    color: #4363d8;
+    top: 10px;
+    left: 45px;
+}
+
+#five {
+    background-color: #f58231;
+    color: #f58231;
+    top: 45px;
+    left: 45px;
+}
+
+#six {
+    background-color: #000075;
+    color: #000075;
+    top: 80px;
+    left: 45px;
+}
\ No newline at end of file

From 641f122724e6e05ffc28fa3e8cc83402ecfa879c Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 14:48:21 +0000
Subject: [PATCH 20/35] Implement clicking by screen percentage

---
 bci/browser/interaction/simulation.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/bci/browser/interaction/simulation.py b/bci/browser/interaction/simulation.py
index c87fa39b..8548d2cf 100644
--- a/bci/browser/interaction/simulation.py
+++ b/bci/browser/interaction/simulation.py
@@ -23,13 +23,23 @@ def __init__(self, browser_config: BrowserConfig):
     def __del__(self):
         self.browser_config.terminate()
 
+    def parse_position(self, position: str, max_value: int) -> int:
+        # Screen percentage
+        if position[-1] == '%':
+            return round(max_value * (int(position[:-1]) / 100))
+
+        # Absolute value in pixels
+        return int(position)
+
     # --- PUBLIC METHODS ---
     def navigate(self, url: str):
         self.browser_config.terminate()
         self.browser_config.open(url)
 
     def click(self, x: str, y: str):
-        gui.moveTo(int(x), int(y))
+        max_x, max_y = gui.size()
+
+        gui.moveTo(self.parse_position(x, max_x), self.parse_position(y, max_y))
         gui.click()
 
     def click_el(self, el_id: str):

From 3ede69e7248ba241990ad94d730d4c002fe70959 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 14:48:32 +0000
Subject: [PATCH 21/35] Implement clicking by screen percentage

---
 experiments/pages/Support/AutoGUI/a.test/main/index.html | 2 +-
 experiments/pages/Support/AutoGUI/interaction_script.cmd | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index fabffa1b..f7700d53 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -59,4 +59,4 @@
     left: 45px;
 }
 </style>
-<a href="https://a.test/report/?leak=AutoGUI" id="three">CLICK</a>
\ No newline at end of file
+<a href="https://a.test/report/?leak=AutoGUI" class="fullscreen">CLICK</a>
\ No newline at end of file
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index 8aaa6b70..f98842c0 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -3,6 +3,6 @@ SLEEP 1
 
 SCREENSHOT
 
-# Comment
-CLICK_EL three
+# test
+CLICK 200 50%
 SLEEP 1
\ No newline at end of file

From 8e1043a2dc7181de7b842f8c7450953e9a9a1a43 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Thu, 31 Oct 2024 15:08:51 +0000
Subject: [PATCH 22/35] Save screenshots to filesystem

---
 .gitignore                                     | 11 +++++++++++
 bci/browser/interaction/interaction.py         | 11 +++++------
 bci/browser/interaction/simulation.py          | 18 +++++++++---------
 bci/evaluations/custom/custom_evaluation.py    |  2 +-
 .../Support/AutoGUI/a.test/main/index.html     |  2 +-
 .../Support/AutoGUI/interaction_script.cmd     |  7 ++-----
 logs/screenshots/.gitkeep                      |  0
 7 files changed, 29 insertions(+), 22 deletions(-)
 create mode 100644 logs/screenshots/.gitkeep

diff --git a/.gitignore b/.gitignore
index e1bddf03..3732e540 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,17 @@ nginx/ssl/keys/*
 !**/.gitkeep
 **/node_modules
 **/junit.xml
+
+# Screenshots
+logs/screenshots/*
+!logs/screenshots/.gitkeep
+
+# Fish shell
+$HOME
+
+# JetBrains IDEs
+.idea
+
 # Created by https://www.toptal.com/developers/gitignore/api/intellij,python,flask,macos
 # Edit at https://www.toptal.com/developers/gitignore?templates=intellij,python,flask,macos
 
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 4f4e9dd3..5da2716d 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -3,6 +3,7 @@
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
 from bci.browser.interaction.simulation import Simulation
+from bci.evaluations.logic import TestParameters
 
 logger = logging.getLogger(__name__)
 
@@ -10,23 +11,21 @@
 class Interaction:
     browser: BrowserConfig
     script: list[str]
+    params: TestParameters
 
-    def __init__(self, browser: BrowserConfig, script: list[str]) -> None:
+    def __init__(self, browser: BrowserConfig, script: list[str], params: TestParameters) -> None:
         self.browser = browser
         self.script = script
+        self.params = params
 
     def execute(self) -> None:
-        simulation = self._initiate_simulation()
+        simulation = Simulation(self.browser, self.params)
 
         self._interpret(simulation)
 
         simulation.navigate('https://a.test/report/?bughog_sanity_check=OK')
         simulation.sleep('0.5')
 
-    def _initiate_simulation(self) -> Simulation:
-        # TODO - possibly return different browser instances
-        return Simulation(self.browser)
-
     def _interpret(self, simulation: Simulation) -> None:
         for statement in self.script:
             if statement.strip() == '':
diff --git a/bci/browser/interaction/simulation.py b/bci/browser/interaction/simulation.py
index 8548d2cf..da22069f 100644
--- a/bci/browser/interaction/simulation.py
+++ b/bci/browser/interaction/simulation.py
@@ -1,6 +1,4 @@
-import base64
 import os
-from io import BytesIO
 from time import sleep
 
 import pyautogui as gui
@@ -8,14 +6,18 @@
 from pyvirtualdisplay.display import Display
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
+from bci.evaluations.logic import TestParameters
 
 
 class Simulation:
     browser_config: BrowserConfig
+    params: TestParameters
+
     public_methods: list[str] = ['navigate', 'click', 'click_el', 'sleep', 'screenshot']
 
-    def __init__(self, browser_config: BrowserConfig):
+    def __init__(self, browser_config: BrowserConfig, params: TestParameters):
         self.browser_config = browser_config
+        self.params = params
         disp = Display(visible=True, size=(1920, 1080), backend='xvfb', use_xauth=True)
         disp.start()
         gui._pyautogui_x11._display = Xlib.display.Display(os.environ['DISPLAY'])
@@ -50,9 +52,7 @@ def click_el(self, el_id: str):
     def sleep(self, duration: str):
         sleep(float(duration))
 
-    def screenshot(self):
-        # TODO save to the filesystem
-        buffered = BytesIO()
-        print(gui.screenshot().save(buffered, format='JPEG'))
-        img_str = base64.b64encode(buffered.getvalue())
-        print(img_str.decode('utf-8'))
+    def screenshot(self, filename: str):
+        filename = f'{self.params.evaluation_configuration.project}-{self.params.mech_group}-{filename}-{type(self.browser_config).__name__}-{self.browser_config.version}.jpg'
+        filepath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../logs/screenshots', filename)
+        gui.screenshot(filepath)
diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index 8e81ed90..3586d7e7 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -105,7 +105,7 @@ def perform_specific_evaluation(self, browser: Browser, params: TestParameters)
 
             max_tries = 3
             if 'interaction_script' in experiment:
-                interaction = Interaction(browser, experiment['interaction_script'])
+                interaction = Interaction(browser, experiment['interaction_script'], params)
                 tries = 0
                 while tries < max_tries:
                     tries += 1
diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index f7700d53..fabffa1b 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -59,4 +59,4 @@
     left: 45px;
 }
 </style>
-<a href="https://a.test/report/?leak=AutoGUI" class="fullscreen">CLICK</a>
\ No newline at end of file
+<a href="https://a.test/report/?leak=AutoGUI" id="three">CLICK</a>
\ No newline at end of file
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index f98842c0..895572c5 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -1,8 +1,5 @@
 NAVIGATE https://a.test/Support/AutoGUI/main
 SLEEP 1
-
-SCREENSHOT
-
-# test
-CLICK 200 50%
+SCREENSHOT testtest
+CLICK_EL three
 SLEEP 1
\ No newline at end of file
diff --git a/logs/screenshots/.gitkeep b/logs/screenshots/.gitkeep
new file mode 100644
index 00000000..e69de29b

From 63ab107c168d5b482c0e628e440d49de9eb740d8 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 09:10:46 +0000
Subject: [PATCH 23/35] Fix collector notifications status code

---
 nginx/config/experiments.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nginx/config/experiments.conf b/nginx/config/experiments.conf
index 398dac9c..1ecb90be 100644
--- a/nginx/config/experiments.conf
+++ b/nginx/config/experiments.conf
@@ -2,7 +2,7 @@ access_log /logs/nginx-access-poc.log default_format;
 
 location /res/ {
     include /etc/nginx/config/notify_collector.conf;
-    error_page 504 @resources;
+    error_page 504 =200 @resources;
 }
 
 location @resources {
@@ -62,7 +62,7 @@ location ~ ^/(.+)/(.+)/(.+)/$ {
 
 location ~ ^/(.+)/(.+)/(.+)$ {
     include /etc/nginx/config/notify_collector.conf;
-    error_page 504 @experiment;
+    error_page 504 =200 @experiment;
 }
 
 location @experiment {

From dcbc707b4e27c55709b2ed5c40f42254c176ac46 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 09:10:57 +0000
Subject: [PATCH 24/35] Remove redundant styles

---
 .../Support/AutoGUI/a.test/main/index.html    | 60 -------------------
 1 file changed, 60 deletions(-)

diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index fabffa1b..c26ef023 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -1,62 +1,2 @@
 <link rel="stylesheet" href="https://a.test/res/bughog.css">
-<style>
-.fullscreen {
-    position: fixed;
-    top: 0;
-    left: 0;
-    width: 100%;
-    height: 100%;
-    background-color: red;
-    z-index: 10;
-}
-
-#one, #two, #three, #four, #five, #six {
-    position: fixed;
-    width: 25px;
-    height: 20px;
-    font-size: 0;
-}
-
-#one {
-    background-color: #800000;
-    color: #800000;
-    top: 10px;
-    left: 10px;
-}
-
-#two {
-    background-color: #dcbeff;
-    color: #dcbeff;
-    top: 45px;
-    left: 10px;
-}
-
-#three {
-    background-color: #ffe119;
-    color: #ffe119;
-    top: 80px;
-    left: 10px;
-}
-
-#four {
-    background-color: #4363d8;
-    color: #4363d8;
-    top: 10px;
-    left: 45px;
-}
-
-#five {
-    background-color: #f58231;
-    color: #f58231;
-    top: 45px;
-    left: 45px;
-}
-
-#six {
-    background-color: #000075;
-    color: #000075;
-    top: 80px;
-    left: 45px;
-}
-</style>
 <a href="https://a.test/report/?leak=AutoGUI" id="three">CLICK</a>
\ No newline at end of file

From 2fc829b565af2c6e5f607be5526e074a77670c25 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 09:30:10 +0000
Subject: [PATCH 25/35] Add default experiment file values

---
 bci/evaluations/custom/custom_evaluation.py   | 23 +++++--------------
 .../custom/default_files/headers.json         |  6 +++++
 bci/evaluations/custom/default_files/html     | 10 ++++++++
 .../default_files/interaction_script.cmd      |  7 ++++++
 bci/evaluations/custom/default_files/js       |  1 +
 bci/evaluations/custom/default_files/py       | 15 ++++++++++++
 .../custom/default_files/url_queue.txt        |  2 ++
 7 files changed, 47 insertions(+), 17 deletions(-)
 create mode 100644 bci/evaluations/custom/default_files/headers.json
 create mode 100644 bci/evaluations/custom/default_files/html
 create mode 100644 bci/evaluations/custom/default_files/interaction_script.cmd
 create mode 100644 bci/evaluations/custom/default_files/js
 create mode 100644 bci/evaluations/custom/default_files/py
 create mode 100644 bci/evaluations/custom/default_files/url_queue.txt

diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index 3586d7e7..84517c9c 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -202,18 +202,7 @@ def add_page(self, project: str, poc: str, domain: str, path: str, file_type: st
             headers_file_path = os.path.join(page_path, 'headers.json')
             if not os.path.exists(headers_file_path):
                 with open(headers_file_path, 'w') as file:
-                    file.write(
-                        textwrap.dedent(
-                            """\
-                            [
-                                {
-                                    "key": "Header-Name",
-                                    "value": "Header-Value"
-                                }
-                            ]
-                        """
-                        )
-                    )
+                    file.write(self.get_default_file_content('headers.json'))
         self.sync_with_folders()
         # Notify clients of change (an experiment might now be runnable)
         Clients.push_experiments_to_all()
@@ -221,13 +210,13 @@ def add_page(self, project: str, poc: str, domain: str, path: str, file_type: st
 
     @staticmethod
     def get_default_file_content(file_type: str) -> str:
-        if file_type != 'py':
-            return ''
+        path = os.path.join(os.path.dirname(os.path.realpath(__file__)), f'default_files/{file_type}')
 
-        with open('experiments/pages/Support/PythonServer/a.test/py-server/index.py', 'r') as file:
-            template_content = file.read()
+        if not os.path.exists(path):
+            return ''
 
-        return template_content
+        with open(path, 'r') as file:
+            return file.read()
 
     @staticmethod
     def include_file_headers(file_type: str) -> bool:
diff --git a/bci/evaluations/custom/default_files/headers.json b/bci/evaluations/custom/default_files/headers.json
new file mode 100644
index 00000000..81975b3f
--- /dev/null
+++ b/bci/evaluations/custom/default_files/headers.json
@@ -0,0 +1,6 @@
+[
+    {
+        "key": "Header-Name",
+        "value": "Header-Value"
+    }
+]
\ No newline at end of file
diff --git a/bci/evaluations/custom/default_files/html b/bci/evaluations/custom/default_files/html
new file mode 100644
index 00000000..2af82326
--- /dev/null
+++ b/bci/evaluations/custom/default_files/html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/res/bughog.css">
+    <script src="/res/bughog.js"></script>
+</head>
+<body>
+    <!-- TODO - implement your PoC -->
+</body>
+</html>
\ No newline at end of file
diff --git a/bci/evaluations/custom/default_files/interaction_script.cmd b/bci/evaluations/custom/default_files/interaction_script.cmd
new file mode 100644
index 00000000..59f6d22b
--- /dev/null
+++ b/bci/evaluations/custom/default_files/interaction_script.cmd
@@ -0,0 +1,7 @@
+# TODO - add your interaction script
+
+# NAVIGATE https://a.test/Project/Experiment/main
+# SLEEP 1
+# SCREENSHOT file_name
+# CLICK_EL one
+# SLEEP 1
\ No newline at end of file
diff --git a/bci/evaluations/custom/default_files/js b/bci/evaluations/custom/default_files/js
new file mode 100644
index 00000000..664997c1
--- /dev/null
+++ b/bci/evaluations/custom/default_files/js
@@ -0,0 +1 @@
+// TODO - implement your PoC
\ No newline at end of file
diff --git a/bci/evaluations/custom/default_files/py b/bci/evaluations/custom/default_files/py
new file mode 100644
index 00000000..91e823b8
--- /dev/null
+++ b/bci/evaluations/custom/default_files/py
@@ -0,0 +1,15 @@
+from flask import Request
+
+# Make sure that your page directory starts with 'py-'
+
+def main(req: Request):
+    # TODO - implement your functionality and return a Flask response
+    
+    return {
+        "agent": req.headers.get("User-Agent"),
+        "cookies": req.cookies,
+        "host": req.host,
+        "path": req.path,
+        "scheme": req.scheme,
+        "url": req.url
+    }
\ No newline at end of file
diff --git a/bci/evaluations/custom/default_files/url_queue.txt b/bci/evaluations/custom/default_files/url_queue.txt
new file mode 100644
index 00000000..8918d8d1
--- /dev/null
+++ b/bci/evaluations/custom/default_files/url_queue.txt
@@ -0,0 +1,2 @@
+TODO - add your URLs to visit
+https://a.test/report/?bughog_sanity_check=OK
\ No newline at end of file

From 5fbb38b5e48c77d9ddb9896654f723d2c1f0e312 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 09:30:20 +0000
Subject: [PATCH 26/35] Refactor experiment

---
 .../pages/Support/AutoGUI/a.test/main/index.html     | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index c26ef023..70d626e3 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -1,2 +1,10 @@
-<link rel="stylesheet" href="https://a.test/res/bughog.css">
-<a href="https://a.test/report/?leak=AutoGUI" id="three">CLICK</a>
\ No newline at end of file
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/res/bughog.css">
+    <script src="/res/bughog.js"></script>
+</head>
+<body>
+    <a href="/report/?leak=AutoGUI" id="three">CLICK</a>
+</body>
+</html>
\ No newline at end of file

From 6e366bda88527d21aa121bdb32a57c971600d36a Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 09:41:10 +0000
Subject: [PATCH 27/35] Better selectors styling

---
 experiments/res/bughog.css | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/experiments/res/bughog.css b/experiments/res/bughog.css
index 10be4b2d..7cdbfa7d 100644
--- a/experiments/res/bughog.css
+++ b/experiments/res/bughog.css
@@ -1,18 +1,23 @@
-.fullscreen {
+#fullscreen, #one, #two, #three, #four, #five, #six {
     position: fixed;
+    z-index: 10;
+    font-size: 0;
+    border: none;
+    outline: none;
+    resize: none;
+}
+
+#fullscreen {
     top: 0;
     left: 0;
     width: 100%;
     height: 100%;
     background-color: red;
-    z-index: 10;
 }
 
 #one, #two, #three, #four, #five, #six {
-    position: fixed;
     width: 25px;
     height: 20px;
-    font-size: 0;
 }
 
 #one {

From 45710c2ac3ed21536fd293cdf7e22183b059ec5a Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 10:33:16 +0000
Subject: [PATCH 28/35] Adding config files through the frontend - Vue part

---
 bci/web/vue/src/components/poc-editor.vue | 55 ++++++++++++++++++++---
 bci/web/vue/src/main.js                   |  4 +-
 bci/web/vue/src/style.css                 |  8 ++--
 3 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/bci/web/vue/src/components/poc-editor.vue b/bci/web/vue/src/components/poc-editor.vue
index 48253620..78d2f526 100644
--- a/bci/web/vue/src/components/poc-editor.vue
+++ b/bci/web/vue/src/components/poc-editor.vue
@@ -46,6 +46,10 @@
           'js',
           'py',
         ],
+        available_config_types: {
+          'interaction_script': 'Interaction script',
+          'url_queue': 'URL queue'
+        },
         poc_tree_config: {
           domain: 'Config',
           page: '_',
@@ -61,12 +65,16 @@
             type: null,
           }
         },
+        config_type: null,
         editor: null,
         timeout: null,
       }
     },
     computed: {
       active_poc_tree() {
+        const configDomain = this.poc_tree_config.domain;
+        const configPage = this.poc_tree_config.page;
+
         return Object.entries(this.active_poc.tree).reduce((acc, [domain, pages]) => {
           if (domain !== pages) {
             return [
@@ -75,11 +83,8 @@
             ]
           }
 
-          const configDomain = this.poc_tree_config.domain;
-          const configPage = this.poc_tree_config.page;
-
           // A single top-level file -> create a virtual config folder
-          const config_folder = acc.find(([domain]) => domain === configDomain) ?? [configDomain, {[configPage]: {}}];
+          const config_folder = acc.find(([domain]) => domain === configDomain);
           return [
             ...acc.filter(([domain]) => domain !== configDomain),
             [configDomain, {
@@ -89,7 +94,7 @@
               }
             }],
           ]
-        }, []);
+        }, [[configDomain, {[configPage]: {}}]]);
       }
     },
     methods: {
@@ -198,6 +203,19 @@
         })
         .catch(() => {
 
+        });
+      },
+      add_config() {
+        const url = `/api/poc/${this.project}/${this.poc}/config`;
+        axios.post(url, {
+          "type": this.config_type,
+        })
+        .then(() => {
+          this.update_poc_tree();
+          this.config_type = null;
+        })
+        .catch(() => {
+
         });
       }
     },
@@ -248,7 +266,12 @@
                     </a>
                   </template>
                   <template v-else>
-                    {{ domain}}
+                    <div class="w-full">
+                      {{ domain}}
+                    </div>
+                    <button v-if="files && Object.keys(files).length === 0" class="no-style" onclick="create_config_dialog.showModal()">
+                      <v-icon name="fa-plus" class="text-indigo-500"/>
+                    </button>
                   </template>
                 </div>
                 <ul>
@@ -304,4 +327,24 @@
       </div>
     </form>
   </dialog>
+
+  <!-- Create config dialog -->
+  <dialog id="create_config_dialog" class="dialog">
+    <form method="dialog" @submit="add_config">
+      <p>
+        <label>
+          <div class="py-2">Choose config type:</div>
+          <select name="config_type" id="config_type" v-model="config_type" required>
+            <option v-for="(label, value) of available_config_types" :value="value">
+              {{ label }}
+            </option>
+          </select>
+        </label>
+      </p>
+      <div class="flex pt-3">
+        <input class="button m-2 w-full" type="submit" value="Create config">
+        <input class="button m-2" type="button" value="Cancel" onclick="create_config_dialog.close()">
+      </div>
+    </form>
+  </dialog>
 </template>
diff --git a/bci/web/vue/src/main.js b/bci/web/vue/src/main.js
index 70ff98b0..121570e3 100644
--- a/bci/web/vue/src/main.js
+++ b/bci/web/vue/src/main.js
@@ -5,8 +5,8 @@ import 'flowbite'
 import 'axios'
 
 import { OhVueIcon, addIcons } from "oh-vue-icons";
-import { MdInfooutline, FaRegularEdit, FaLink } from "oh-vue-icons/icons";
+import { MdInfooutline, FaRegularEdit, FaLink, FaPlus } from "oh-vue-icons/icons";
 
-addIcons(MdInfooutline, FaRegularEdit, FaLink);
+addIcons(MdInfooutline, FaRegularEdit, FaLink, FaPlus);
 const app = createApp(App);
 app.component("v-icon", OhVueIcon).mount('#app')
diff --git a/bci/web/vue/src/style.css b/bci/web/vue/src/style.css
index 713de6c3..12c7eb4c 100644
--- a/bci/web/vue/src/style.css
+++ b/bci/web/vue/src/style.css
@@ -3,7 +3,7 @@
 @tailwind utilities;
 
 .button {
-  @apply text-white bg-blue-700 hover:bg-blue-800 active:ring-4 focus:outline-none font-medium rounded-lg text-sm px-4 py-2.5 text-center items-center dark:hover:bg-blue-700 dark:focus:ring-blue-800
+  @apply text-white bg-blue-700 hover:bg-blue-800 active:ring-4 focus:outline-none cursor-pointer font-medium rounded-lg text-sm px-4 py-2.5 text-center items-center dark:hover:bg-blue-700 dark:focus:ring-blue-800
 }
 
 .dropdown-head {
@@ -192,7 +192,7 @@ h1 {
   line-height: 1.1;
 }
 
-button {
+button:not(.no-style) {
   border-radius: 8px;
   border: 1px solid transparent;
   padding: 0.6em 1.2em;
@@ -203,8 +203,8 @@ button {
   cursor: pointer;
   transition: border-color 0.25s;
 }
-button:focus,
-button:focus-visible {
+button:not(.no-style):focus,
+button:not(.no-style):focus-visible {
   outline: 4px auto -webkit-focus-ring-color;
 }
 

From acdec3cf51e5c1e4b5f5e2dfc2993b924767789e Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 10:42:29 +0000
Subject: [PATCH 29/35] Adding config files through the frontend - Python part

---
 bci/evaluations/custom/custom_evaluation.py | 16 ++++++++++++++++
 bci/main.py                                 |  4 ++++
 bci/web/blueprints/api.py                   | 14 ++++++++++++++
 bci/web/vue/src/components/poc-editor.vue   |  4 ++--
 4 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index 84517c9c..baa464a3 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -207,6 +207,22 @@ def add_page(self, project: str, poc: str, domain: str, path: str, file_type: st
         # Notify clients of change (an experiment might now be runnable)
         Clients.push_experiments_to_all()
         return True
+    
+    def add_config(self, project: str, poc: str, type: str) -> bool:
+        content = self.get_default_file_content(type)
+
+        if (content == ''):
+            return False
+
+        file_path = os.path.join(Global.custom_page_folder, project, poc, type)
+        with open(file_path, 'w') as file:
+            file.write(content)
+        
+        self.sync_with_folders()
+        # Notify clients of change (an experiment might now be runnable)
+        Clients.push_experiments_to_all()
+
+        return True
 
     @staticmethod
     def get_default_file_content(file_type: str) -> str:
diff --git a/bci/main.py b/bci/main.py
index 127c1c29..fddb53ae 100644
--- a/bci/main.py
+++ b/bci/main.py
@@ -155,6 +155,10 @@ def get_available_domains() -> list[str]:
     @staticmethod
     def add_page(project: str, poc: str, domain: str, path: str, file_type: str) -> bool:
         return Main.master.evaluation_framework.add_page(project, poc, domain, path, file_type)
+    
+    @staticmethod
+    def add_config(project: str, poc: str, type: str) -> bool:
+        return Main.master.evaluation_framework.add_config(project, poc, type)
 
     @staticmethod
     def sigint_handler(signum, frame):
diff --git a/bci/web/blueprints/api.py b/bci/web/blueprints/api.py
index 592fbcab..8c1094b8 100644
--- a/bci/web/blueprints/api.py
+++ b/bci/web/blueprints/api.py
@@ -208,6 +208,20 @@ def add_page(project: str, poc: str):
         }
 
 
+@api.route('/poc/<string:project>/<string:poc>/config', methods=['POST'])
+def add_config(project: str, poc: str):
+    data = request.json.copy()
+    success = bci_api.add_config(project, poc, data['type'])
+    if success:
+        return {
+            'status': 'OK'
+        }
+    else:
+        return {
+            'status': 'NOK'
+        }
+
+
 @api.route('/poc/domain/', methods=['GET'])
 def get_available_domains():
     return {
diff --git a/bci/web/vue/src/components/poc-editor.vue b/bci/web/vue/src/components/poc-editor.vue
index 78d2f526..45cffbed 100644
--- a/bci/web/vue/src/components/poc-editor.vue
+++ b/bci/web/vue/src/components/poc-editor.vue
@@ -47,8 +47,8 @@
           'py',
         ],
         available_config_types: {
-          'interaction_script': 'Interaction script',
-          'url_queue': 'URL queue'
+          'interaction_script.cmd': 'Interaction script',
+          'url_queue.txt': 'URL queue'
         },
         poc_tree_config: {
           domain: 'Config',

From 3ee4d20e19d748bcd2210b846966e8ea1deb6516 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 12:06:47 +0000
Subject: [PATCH 30/35] More interaction commands

---
 bci/browser/interaction/interaction.py        | 14 ++++----
 bci/browser/interaction/simulation.py         | 35 ++++++++++++++++---
 .../default_files/interaction_script.cmd      | 17 +++++----
 bci/web/vue/src/interaction_script_mode.js    |  2 +-
 .../Support/AutoGUI/a.test/main/index.html    |  6 +++-
 .../Support/AutoGUI/interaction_script.cmd    | 19 ++++++++--
 experiments/res/bughog.css                    |  2 +-
 7 files changed, 71 insertions(+), 24 deletions(-)

diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 5da2716d..4561fff5 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -22,32 +22,30 @@ def execute(self) -> None:
         simulation = Simulation(self.browser, self.params)
 
         self._interpret(simulation)
+        simulation.sleep('0.5')
 
         simulation.navigate('https://a.test/report/?bughog_sanity_check=OK')
-        simulation.sleep('0.5')
 
     def _interpret(self, simulation: Simulation) -> None:
         for statement in self.script:
-            if statement.strip() == '':
+            if statement.strip() == '' or statement[0] == '#':
                 continue
 
             cmd, *args = statement.split()
             method_name = cmd.lower()
 
-            if cmd == '#':
-                continue
-
             if method_name not in Simulation.public_methods:
                 raise Exception(
                     f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Simulation.public_methods))}.'
                 )
 
             method = getattr(simulation, method_name)
-            method_params_len = len(signature(method).parameters)
+            method_params = list(signature(method).parameters.values())
 
-            if method_params_len != len(args):
+            # Allow different number of arguments only for variable argument number (*)
+            if len(method_params) != len(args) and (len(method_params) < 1 or str(method_params[0])[0] != '*'):
                 raise Exception(
-                    f'Invalid number of arguments for command `{cmd}`. Expected {method_params_len}, got {len(args)}.'
+                    f'Invalid number of arguments for command `{cmd}`. Expected {len(method_params)}, got {len(args)}.'
                 )
 
             logger.debug(f'Executing interaction method `{method_name}` with the arguments {args}')
diff --git a/bci/browser/interaction/simulation.py b/bci/browser/interaction/simulation.py
index da22069f..68da367e 100644
--- a/bci/browser/interaction/simulation.py
+++ b/bci/browser/interaction/simulation.py
@@ -13,7 +13,18 @@ class Simulation:
     browser_config: BrowserConfig
     params: TestParameters
 
-    public_methods: list[str] = ['navigate', 'click', 'click_el', 'sleep', 'screenshot']
+    public_methods: list[str] = [
+        'navigate',
+        'click_position',
+        'click',
+        'write',
+        'press',
+        'hold',
+        'release',
+        'hotkey',
+        'sleep',
+        'screenshot',
+    ]
 
     def __init__(self, browser_config: BrowserConfig, params: TestParameters):
         self.browser_config = browser_config
@@ -37,17 +48,33 @@ def parse_position(self, position: str, max_value: int) -> int:
     def navigate(self, url: str):
         self.browser_config.terminate()
         self.browser_config.open(url)
+        self.sleep('0.5')
 
-    def click(self, x: str, y: str):
+    def click_position(self, x: str, y: str):
         max_x, max_y = gui.size()
 
         gui.moveTo(self.parse_position(x, max_x), self.parse_position(y, max_y))
         gui.click()
 
-    def click_el(self, el_id: str):
+    def click(self, el_id: str):
         el_image_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), f'elements/{el_id}.png')
         x, y = gui.locateCenterOnScreen(el_image_path)
-        self.click(str(x), str(y))
+        self.click_position(str(x), str(y))
+
+    def write(self, text: str):
+        gui.write(text, interval=0.1)
+
+    def press(self, key: str):
+        gui.press(key)
+
+    def hold(self, key: str):
+        gui.keyDown(key)
+
+    def release(self, key: str):
+        gui.keyUp(key)
+
+    def hotkey(self, *keys: str):
+        gui.hotkey(*keys)
 
     def sleep(self, duration: str):
         sleep(float(duration))
diff --git a/bci/evaluations/custom/default_files/interaction_script.cmd b/bci/evaluations/custom/default_files/interaction_script.cmd
index 59f6d22b..63ad6d9b 100644
--- a/bci/evaluations/custom/default_files/interaction_script.cmd
+++ b/bci/evaluations/custom/default_files/interaction_script.cmd
@@ -1,7 +1,12 @@
-# TODO - add your interaction script
+# TODO - add your interaction script using the commands 
 
-# NAVIGATE https://a.test/Project/Experiment/main
-# SLEEP 1
-# SCREENSHOT file_name
-# CLICK_EL one
-# SLEEP 1
\ No newline at end of file
+# NAVIGATE url
+# CLICK_POSITION x y    where x and y are absolute numbers or screen percentages
+# CLICK element_id      where element_id is one of one, two, three, four, five, six
+# WRITE text
+# PRESS key
+# HOLD key
+# RELEASE key
+# HOTKEY key1 key2 ...
+# SLEEP seconds         where seconds is a float or an int
+# SCREENSHOT file_name
\ No newline at end of file
diff --git a/bci/web/vue/src/interaction_script_mode.js b/bci/web/vue/src/interaction_script_mode.js
index 35785b44..4413d521 100644
--- a/bci/web/vue/src/interaction_script_mode.js
+++ b/bci/web/vue/src/interaction_script_mode.js
@@ -1,4 +1,4 @@
-const KEYWORDS = "NAVIGATE|CLICK|CLICK_EL|SLEEP|SCREENSHOT";
+const KEYWORDS = "NAVIGATE|CLICK_POSITION|CLICK|WRITE|PRESS|HOLD|RELEASE|HOTKEY|SLEEP|SCREENSHOT";
 
 ace.define("ace/mode/interaction_script_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module){"use strict";
     const oop = require("../lib/oop");
diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index 70d626e3..e2b6d3c4 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -5,6 +5,10 @@
     <script src="/res/bughog.js"></script>
 </head>
 <body>
-    <a href="/report/?leak=AutoGUI" id="three">CLICK</a>
+    <form action="/report" method="GET">
+        <input id="one">
+        <input name="leak" id="two">
+        <input type="submit">
+    </form>
 </body>
 </html>
\ No newline at end of file
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index 895572c5..a9ed5d45 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -1,5 +1,18 @@
 NAVIGATE https://a.test/Support/AutoGUI/main
 SLEEP 1
-SCREENSHOT testtest
-CLICK_EL three
-SLEEP 1
\ No newline at end of file
+SCREENSHOT test
+
+CLICK one
+WRITE AutoGUI
+HOTKEY ctrl a
+HOTKEY ctrl c
+
+CLICK two
+
+# Equivalent to HOTKEY ctrl v
+HOLD ctrl
+HOLD v
+RELEASE v
+RELEASE ctrl
+
+PRESS Enter
\ No newline at end of file
diff --git a/experiments/res/bughog.css b/experiments/res/bughog.css
index 7cdbfa7d..fd01d4cd 100644
--- a/experiments/res/bughog.css
+++ b/experiments/res/bughog.css
@@ -1,7 +1,7 @@
 #fullscreen, #one, #two, #three, #four, #five, #six {
     position: fixed;
     z-index: 10;
-    font-size: 0;
+    font-size: 1px;
     border: none;
     outline: none;
     resize: none;

From 52211d03ee147b2f08f5a2f9a1ef7f0aae177fff Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 12:11:14 +0000
Subject: [PATCH 31/35] Styling fix

---
 bci/web/vue/src/style.css | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/bci/web/vue/src/style.css b/bci/web/vue/src/style.css
index 12c7eb4c..c52649c6 100644
--- a/bci/web/vue/src/style.css
+++ b/bci/web/vue/src/style.css
@@ -192,7 +192,7 @@ h1 {
   line-height: 1.1;
 }
 
-button:not(.no-style) {
+button {
   border-radius: 8px;
   border: 1px solid transparent;
   padding: 0.6em 1.2em;
@@ -203,11 +203,17 @@ button:not(.no-style) {
   cursor: pointer;
   transition: border-color 0.25s;
 }
-button:not(.no-style):focus,
-button:not(.no-style):focus-visible {
+button:focus,
+button:focus-visible {
   outline: 4px auto -webkit-focus-ring-color;
 }
 
+button.no-style {
+  padding: 0;
+  background: transparent;
+  border: none;
+}
+
 .card {
   padding: 2em;
 }

From 639441c6c4d7144609a2568ed759751f25d3ccb8 Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 13:25:22 +0000
Subject: [PATCH 32/35] Final interaction improvements

---
 bci/browser/configuration/browser.py            |  4 ++++
 bci/browser/configuration/chromium.py           |  3 +++
 bci/browser/configuration/firefox.py            |  3 +++
 bci/browser/interaction/interaction.py          | 17 +++++++++++------
 bci/browser/interaction/simulation.py           |  2 +-
 .../Support/AutoGUI/a.test/main/index.html      |  2 +-
 .../Support/AutoGUI/interaction_script.cmd      |  4 ++--
 experiments/res/bughog.css                      | 14 +++++++-------
 8 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/bci/browser/configuration/browser.py b/bci/browser/configuration/browser.py
index 0247f28e..f6a9f2e7 100644
--- a/bci/browser/configuration/browser.py
+++ b/bci/browser/configuration/browser.py
@@ -100,6 +100,10 @@ def _get_executable_file_path(self) -> str:
     def _get_terminal_args(self) -> list[str]:
         pass
 
+    @abstractmethod
+    def get_navigation_sleep_duration(self) -> int:
+        pass
+
     @staticmethod
     def get_browser(
         browser_config: BrowserConfiguration, eval_config: EvaluationConfiguration, state: State
diff --git a/bci/browser/configuration/chromium.py b/bci/browser/configuration/chromium.py
index 6108328d..f2cd16cb 100644
--- a/bci/browser/configuration/chromium.py
+++ b/bci/browser/configuration/chromium.py
@@ -32,6 +32,9 @@
 
 class Chromium(Browser):
 
+    def get_navigation_sleep_duration(self) -> int:
+        return 1
+
     def _get_terminal_args(self) -> list[str]:
         assert self._profile_path is not None
 
diff --git a/bci/browser/configuration/firefox.py b/bci/browser/configuration/firefox.py
index 4310bf20..cff1d7f1 100644
--- a/bci/browser/configuration/firefox.py
+++ b/bci/browser/configuration/firefox.py
@@ -20,6 +20,9 @@
 
 class Firefox(Browser):
 
+    def get_navigation_sleep_duration(self) -> int:
+        return 2
+
     def _get_terminal_args(self) -> list[str]:
         assert self._profile_path is not None
 
diff --git a/bci/browser/interaction/interaction.py b/bci/browser/interaction/interaction.py
index 4561fff5..14fb8d1b 100644
--- a/bci/browser/interaction/interaction.py
+++ b/bci/browser/interaction/interaction.py
@@ -21,12 +21,11 @@ def __init__(self, browser: BrowserConfig, script: list[str], params: TestParame
     def execute(self) -> None:
         simulation = Simulation(self.browser, self.params)
 
-        self._interpret(simulation)
-        simulation.sleep('0.5')
+        if self._interpret(simulation):
+            simulation.sleep(str(self.browser.get_navigation_sleep_duration()))
+            simulation.navigate('https://a.test/report/?bughog_sanity_check=OK')
 
-        simulation.navigate('https://a.test/report/?bughog_sanity_check=OK')
-
-    def _interpret(self, simulation: Simulation) -> None:
+    def _interpret(self, simulation: Simulation) -> bool:
         for statement in self.script:
             if statement.strip() == '' or statement[0] == '#':
                 continue
@@ -49,4 +48,10 @@ def _interpret(self, simulation: Simulation) -> None:
                 )
 
             logger.debug(f'Executing interaction method `{method_name}` with the arguments {args}')
-            method(*args)
+
+            try:
+                method(*args)
+            except:
+                return False
+
+        return True
diff --git a/bci/browser/interaction/simulation.py b/bci/browser/interaction/simulation.py
index 68da367e..d4ba899c 100644
--- a/bci/browser/interaction/simulation.py
+++ b/bci/browser/interaction/simulation.py
@@ -48,7 +48,7 @@ def parse_position(self, position: str, max_value: int) -> int:
     def navigate(self, url: str):
         self.browser_config.terminate()
         self.browser_config.open(url)
-        self.sleep('0.5')
+        self.sleep(str(self.browser_config.get_navigation_sleep_duration()))
 
     def click_position(self, x: str, y: str):
         max_x, max_y = gui.size()
diff --git a/experiments/pages/Support/AutoGUI/a.test/main/index.html b/experiments/pages/Support/AutoGUI/a.test/main/index.html
index e2b6d3c4..93eb4fc7 100644
--- a/experiments/pages/Support/AutoGUI/a.test/main/index.html
+++ b/experiments/pages/Support/AutoGUI/a.test/main/index.html
@@ -5,7 +5,7 @@
     <script src="/res/bughog.js"></script>
 </head>
 <body>
-    <form action="/report" method="GET">
+    <form action="/report/" method="GET">
         <input id="one">
         <input name="leak" id="two">
         <input type="submit">
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/interaction_script.cmd
index a9ed5d45..91d6f21f 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/interaction_script.cmd
@@ -1,12 +1,12 @@
 NAVIGATE https://a.test/Support/AutoGUI/main
-SLEEP 1
-SCREENSHOT test
 
+SCREENSHOT click1
 CLICK one
 WRITE AutoGUI
 HOTKEY ctrl a
 HOTKEY ctrl c
 
+SCREENSHOT click2
 CLICK two
 
 # Equivalent to HOTKEY ctrl v
diff --git a/experiments/res/bughog.css b/experiments/res/bughog.css
index fd01d4cd..8fe70435 100644
--- a/experiments/res/bughog.css
+++ b/experiments/res/bughog.css
@@ -30,14 +30,14 @@
 #two {
     background-color: #dcbeff;
     color: #dcbeff;
-    top: 45px;
+    top: 70px;
     left: 10px;
 }
 
 #three {
     background-color: #ffe119;
     color: #ffe119;
-    top: 80px;
+    top: 130px;
     left: 10px;
 }
 
@@ -45,19 +45,19 @@
     background-color: #4363d8;
     color: #4363d8;
     top: 10px;
-    left: 45px;
+    left: 85px;
 }
 
 #five {
     background-color: #f58231;
     color: #f58231;
-    top: 45px;
-    left: 45px;
+    top: 70px;
+    left: 85px;
 }
 
 #six {
     background-color: #000075;
     color: #000075;
-    top: 80px;
-    left: 45px;
+    top: 130px;
+    left: 85px;
 }
\ No newline at end of file

From 8e4af952e4915460c239a95e450b4060087f255d Mon Sep 17 00:00:00 2001
From: Jakub Szymsza <jakub@szymsza.cz>
Date: Fri, 1 Nov 2024 13:29:05 +0000
Subject: [PATCH 33/35] Remove old experiment

---
 .../pages/Support/UserInteraction/a.test/main/headers.json  | 1 -
 .../pages/Support/UserInteraction/a.test/main/index.html    | 6 ------
 .../pages/Support/UserInteraction/interaction_script.cmd    | 1 -
 3 files changed, 8 deletions(-)
 delete mode 100644 experiments/pages/Support/UserInteraction/a.test/main/headers.json
 delete mode 100644 experiments/pages/Support/UserInteraction/a.test/main/index.html
 delete mode 100644 experiments/pages/Support/UserInteraction/interaction_script.cmd

diff --git a/experiments/pages/Support/UserInteraction/a.test/main/headers.json b/experiments/pages/Support/UserInteraction/a.test/main/headers.json
deleted file mode 100644
index fe51488c..00000000
--- a/experiments/pages/Support/UserInteraction/a.test/main/headers.json
+++ /dev/null
@@ -1 +0,0 @@
-[]
diff --git a/experiments/pages/Support/UserInteraction/a.test/main/index.html b/experiments/pages/Support/UserInteraction/a.test/main/index.html
deleted file mode 100644
index ad482c63..00000000
--- a/experiments/pages/Support/UserInteraction/a.test/main/index.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<a href="http://a.test/report/?leak=UserInteraction" style="position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: red;" id="leak">
-    CLICK
-</a>
-<script>
-    document.getElementById('leak').click();
-</script>
\ No newline at end of file
diff --git a/experiments/pages/Support/UserInteraction/interaction_script.cmd b/experiments/pages/Support/UserInteraction/interaction_script.cmd
deleted file mode 100644
index 5daa56cf..00000000
--- a/experiments/pages/Support/UserInteraction/interaction_script.cmd
+++ /dev/null
@@ -1 +0,0 @@
-NAVIGATE https://a.test/Support/UserInteraction/main

From a2e61c2b4b1fd2e5de940159b2762fff2929278d Mon Sep 17 00:00:00 2001
From: Gertjan <gertjan.franken@kuleuven.be>
Date: Wed, 13 Nov 2024 11:17:54 +0000
Subject: [PATCH 34/35] Fix button to add script

---
 bci/web/vue/src/components/poc-editor.vue | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/bci/web/vue/src/components/poc-editor.vue b/bci/web/vue/src/components/poc-editor.vue
index ff9afb5b..a750fb01 100644
--- a/bci/web/vue/src/components/poc-editor.vue
+++ b/bci/web/vue/src/components/poc-editor.vue
@@ -232,6 +232,14 @@
         .catch(() => {
 
         });
+      },
+      has_config() {
+        if (this.active_poc.tree === undefined) {
+          console.error("Could not check config in undefined poc tree");
+          return false;
+        }
+        const arr = Object.keys(this.active_poc.tree).filter((domain) => ['url_queue.txt', 'interaction_script.cmd'].includes(domain));
+        return arr.length > 0;
       }
     },
     mounted() {
@@ -292,9 +300,6 @@
                     <a :href="'https://'+domain+'/'+this.project+'/'+this.active_poc.name+'/'+path" target="_blank">
                       <v-icon name="fa-link" class=""/>
                     </a>
-                    <button v-if="files && Object.keys(files).length === 0" class="no-style" onclick="create_config_dialog.showModal()">
-                      <v-icon name="fa-plus" class="text-indigo-500"/>
-                    </button>
                   </div>
                   <ul>
                     <li v-for="([file, _]) in Object.entries(files)"
@@ -328,9 +333,14 @@
               </li>
             </ul>
           </li>
-          <li role="button" class="button text-center" onclick="create_domain_dialog.showModal()">
-            Add page
-          </li>
+          <div class="flex flex-row justify-between">
+            <li role="button" class="button text-center w-full mr-1" onclick="create_domain_dialog.showModal()">
+              Add page
+            </li>
+            <li v-if="!this.has_config()" role="button" class="button text-center w-full mr-1" onclick="create_config_dialog.showModal()">
+              Add script
+            </li>
+          </div>
         </nav>
       </ul>
     </div>

From 792447e4384be3164169f85bc12f35c9b75efe14 Mon Sep 17 00:00:00 2001
From: Gertjan <gertjan.franken@kuleuven.be>
Date: Wed, 13 Nov 2024 12:10:55 +0000
Subject: [PATCH 35/35] Rename interaction_script.cmd to script.md and small
 button fix

---
 bci/evaluations/custom/custom_evaluation.py   | 10 ++---
 .../custom/default_files/script.cmd           | 12 ++++++
 bci/web/vue/src/components/poc-editor.vue     | 41 +++++++++++--------
 .../{interaction_script.cmd => script.cmd}    |  2 +-
 .../Support/test}/interaction_script.cmd      |  4 +-
 5 files changed, 43 insertions(+), 26 deletions(-)
 create mode 100644 bci/evaluations/custom/default_files/script.cmd
 rename experiments/pages/Support/AutoGUI/{interaction_script.cmd => script.cmd} (94%)
 rename {bci/evaluations/custom/default_files => experiments/pages/Support/test}/interaction_script.cmd (79%)

diff --git a/bci/evaluations/custom/custom_evaluation.py b/bci/evaluations/custom/custom_evaluation.py
index 085abf41..e4e9ff5c 100644
--- a/bci/evaluations/custom/custom_evaluation.py
+++ b/bci/evaluations/custom/custom_evaluation.py
@@ -57,7 +57,7 @@ def initialize_tests_and_interactions(dir_tree: dict) -> dict:
                 data = {}
 
                 if interaction_script := CustomEvaluationFramework.__get_interaction_script(project_path, experiment):
-                    data['interaction_script'] = interaction_script
+                    data['script'] = interaction_script
                 elif url_queue := CustomEvaluationFramework.__get_url_queue(project, project_path, experiment):
                     data['url_queue'] = url_queue
 
@@ -68,7 +68,7 @@ def initialize_tests_and_interactions(dir_tree: dict) -> dict:
 
     @staticmethod
     def __get_interaction_script(project_path: str, experiment: str) -> list[str] | None:
-        interaction_file_path = os.path.join(project_path, experiment, 'interaction_script.cmd')
+        interaction_file_path = os.path.join(project_path, experiment, 'script.cmd')
         if os.path.isfile(interaction_file_path):
             # If an interaction script is specified, it is parsed and used
             with open(interaction_file_path) as file:
@@ -97,7 +97,7 @@ def __get_url_queue(project: str, project_path: str, experiment: str) -> Optiona
     @staticmethod
     def is_runnable_experiment(project: str, poc: str, dir_tree: dict[str,dict], data: dict[str,str]) -> bool:
         # Always runnable if there is either an interaction script or url_queue present
-        if 'interaction_script' in data or 'url_queue' in data:
+        if 'script' in data or 'url_queue' in data:
             return True
 
         # Should have exactly one main folder otherwise
@@ -123,8 +123,8 @@ def perform_specific_evaluation(self, browser: Browser, params: TestParameters)
             experiment = self.tests_per_project[params.evaluation_configuration.project][params.mech_group]
 
             max_tries = 3
-            if 'interaction_script' in experiment:
-                interaction = Interaction(browser, experiment['interaction_script'], params)
+            if 'script' in experiment:
+                interaction = Interaction(browser, experiment['script'], params)
                 tries = 0
                 while tries < max_tries:
                     tries += 1
diff --git a/bci/evaluations/custom/default_files/script.cmd b/bci/evaluations/custom/default_files/script.cmd
new file mode 100644
index 00000000..749e7a21
--- /dev/null
+++ b/bci/evaluations/custom/default_files/script.cmd
@@ -0,0 +1,12 @@
+# TODO - add your interaction script using the commands
+
+# NAVIGATE url
+# CLICK_POSITION x y    where x and y are absolute numbers or screen percentages
+# CLICK element_id      where element_id is one of one, two, three, four, five, six
+# WRITE text
+# PRESS key
+# HOLD key
+# RELEASE key
+# HOTKEY key1 key2 ...
+# SLEEP seconds         where seconds is a float or an int
+# SCREENSHOT file_name
diff --git a/bci/web/vue/src/components/poc-editor.vue b/bci/web/vue/src/components/poc-editor.vue
index a750fb01..167e0175 100644
--- a/bci/web/vue/src/components/poc-editor.vue
+++ b/bci/web/vue/src/components/poc-editor.vue
@@ -31,14 +31,24 @@
             // Example tree:
             // {
             //   'test.com': {
-            //     "path1": ["file1", "file2"],
-            //     "path2": ["file1"],
+            //     "path1": {
+            //        "file1": null,
+            //        "file2": null
+            //     },
+            //     "path2": {
+            //        "file1": null
+            //     },
             //   },
             //   'not.test': {
-            //     "path1": ["file1", "file2"],
-            //     "path2": ["file1"],
+            //     "path1": {
+            //        "file1": null,
+            //        "file2": null
+            //     },
+            //     "path2": {
+            //        "file1": null
+            //     },
             //   },
-            //  'interaction_script.cmd': 'interaction_script.cmd',
+            //  'script.cmd': null,
             // },
         },
         available_file_types: [
@@ -47,7 +57,7 @@
           'py',
         ],
         available_config_types: {
-          'interaction_script.cmd': 'Interaction script',
+          'script.cmd': 'Interaction script',
           'url_queue.txt': 'URL queue'
         },
         poc_tree_config: {
@@ -238,7 +248,7 @@
           console.error("Could not check config in undefined poc tree");
           return false;
         }
-        const arr = Object.keys(this.active_poc.tree).filter((domain) => ['url_queue.txt', 'interaction_script.cmd'].includes(domain));
+        const arr = Object.keys(this.active_poc.tree).filter((domain) => ['url_queue.txt', 'script.cmd'].includes(domain));
         return arr.length > 0;
       }
     },
@@ -277,17 +287,12 @@
       <ul class="relative flex flex-col text-gray-700 bg-white shadow-md w-96 rounded-md bg-clip-border dark:bg-dark-3 dark:text-white">
         <nav class="flex min-w-[240px] h-full flex-col gap-1 p-2 font-sans text-base font-normal text-blue-gray-700">
           <li v-for="([domain, pages]) in Object.entries(active_poc.tree).sort()">
-            <div v-if="domain === 'url_queue.txt' || domain === 'interaction_script.cmd'">
-              <div class="flex border-b-2 p-2 font-bold mb-2 hover:bg-gray-100 hover:bg-opacity-80 hover:text-blue-gray-900 hover:cursor-pointer focus:bg-blue-gray-50 focus:bg-opacity-80 focus:text-blue-gray-900" >
-                <div
-                    role="button"
-                    class="rounded-lg"
-                    @click="set_active_file(null, null, domain)"
-                  >
-                  <div class="w-full">
-                    {{ domain }}
-                  </div>
-                </div>
+            <div v-if="domain === 'url_queue.txt' || domain === 'script.cmd'">
+              <div
+                class="flex border-b-2 p-2 font-bold mb-2 hover:bg-gray-100 hover:bg-opacity-80 hover:text-blue-gray-900 hover:cursor-pointer focus:bg-blue-gray-50 focus:bg-opacity-80 focus:text-blue-gray-900"
+                role="button"
+                @click="set_active_file(null, null, domain)" >
+                {{ domain }}
               </div>
             </div>
             <ul v-else>
diff --git a/experiments/pages/Support/AutoGUI/interaction_script.cmd b/experiments/pages/Support/AutoGUI/script.cmd
similarity index 94%
rename from experiments/pages/Support/AutoGUI/interaction_script.cmd
rename to experiments/pages/Support/AutoGUI/script.cmd
index 91d6f21f..f72b0eff 100644
--- a/experiments/pages/Support/AutoGUI/interaction_script.cmd
+++ b/experiments/pages/Support/AutoGUI/script.cmd
@@ -15,4 +15,4 @@ HOLD v
 RELEASE v
 RELEASE ctrl
 
-PRESS Enter
\ No newline at end of file
+PRESS Enter
diff --git a/bci/evaluations/custom/default_files/interaction_script.cmd b/experiments/pages/Support/test/interaction_script.cmd
similarity index 79%
rename from bci/evaluations/custom/default_files/interaction_script.cmd
rename to experiments/pages/Support/test/interaction_script.cmd
index 63ad6d9b..749e7a21 100644
--- a/bci/evaluations/custom/default_files/interaction_script.cmd
+++ b/experiments/pages/Support/test/interaction_script.cmd
@@ -1,4 +1,4 @@
-# TODO - add your interaction script using the commands 
+# TODO - add your interaction script using the commands
 
 # NAVIGATE url
 # CLICK_POSITION x y    where x and y are absolute numbers or screen percentages
@@ -9,4 +9,4 @@
 # RELEASE key
 # HOTKEY key1 key2 ...
 # SLEEP seconds         where seconds is a float or an int
-# SCREENSHOT file_name
\ No newline at end of file
+# SCREENSHOT file_name