Asserting downloaded file content and more changes

bci/browser/interaction/interaction.py

@@ -1,9 +1,11 @@
 import logging
 from inspect import signature
+from urllib.parse import quote_plus
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
 from bci.browser.interaction.simulation import Simulation
 from bci.evaluations.logic import TestParameters
+from bci.browser.interaction.simulation_exception import SimulationException
 
 logger = logging.getLogger(__name__)
 
@@ -26,32 +28,38 @@ def execute(self) -> None:
             simulation.navigate('https://a.test/report/?bughog_sanity_check=OK')
 
     def _interpret(self, simulation: Simulation) -> bool:
-        for statement in self.script:
-            if statement.strip() == '' or statement[0] == '#':
-                continue
+        try: 
+            for statement in self.script:
+                if statement.strip() == '' or statement[0] == '#':
+                    continue
 
-            cmd, *args = statement.split()
-            method_name = cmd.lower()
+                cmd, *args = statement.split()
+                method_name = cmd.lower()
 
-            if method_name not in Simulation.public_methods:
-                raise Exception(
-                    f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Simulation.public_methods))}.'
-                )
+                if method_name not in Simulation.public_methods:
+                    raise Exception(
+                        f'Invalid command `{cmd}`. Expected one of {", ".join(map(lambda m: m.upper(), Simulation.public_methods))}.'
+                    )
 
-            method = getattr(simulation, method_name)
-            method_params = list(signature(method).parameters.values())
+                method = getattr(simulation, method_name)
+                method_params = list(signature(method).parameters.values())
 
-            # Allow different number of arguments only for variable argument number (*)
-            if len(method_params) != len(args) and (len(method_params) < 1 or str(method_params[0])[0] != '*'):
-                raise Exception(
-                    f'Invalid number of arguments for command `{cmd}`. Expected {len(method_params)}, got {len(args)}.'
-                )
+                # Allow different number of arguments only for variable argument number (*)
+                if len(method_params) != len(args) and (len(method_params) < 1 or str(method_params[0])[0] != '*'):
+                    raise Exception(
+                        f'Invalid number of arguments for command `{cmd}`. Expected {len(method_params)}, got {len(args)}.'
+                    )
 
-            logger.debug(f'Executing interaction method `{method_name}` with the arguments {args}')
+                logger.debug(f'Executing interaction method `{method_name}` with the arguments {args}')
 
-            try:
                 method(*args)
-            except:
-                return False
 
-        return True
+            return True
+        except SimulationException as e:
+            # Simulation exception - sane behaviour, but do not continue interpreting
+            simulation.navigate(f'https://a.test/report/?exception={quote_plus(str(e))}')
+            return True
+        except Exception as e:
+            # Unexpected exception type - not sane, report the exception
+            simulation.navigate(f'https://a.test/report/?uncaught-exception={quote_plus(type(e).__name__)}&message={quote_plus(str(e))}')
+            return False

bci/browser/interaction/simulation.py

@@ -6,6 +6,7 @@
 from pyvirtualdisplay.display import Display
 
 from bci.browser.configuration.browser import Browser as BrowserConfig
+from bci.browser.interaction.simulation_exception import SimulationException
 from bci.evaluations.logic import TestParameters
 
 
@@ -15,6 +16,7 @@ class Simulation:
 
     public_methods: list[str] = [
         'navigate',
+        'new_tab',
         'click_position',
         'click',
         'write',
@@ -24,6 +26,9 @@ class Simulation:
         'hotkey',
         'sleep',
         'screenshot',
+        'report_leak',
+        'assert_file_contains',
+        'open_file',
     ]
 
     def __init__(self, browser_config: BrowserConfig, params: TestParameters):
@@ -50,6 +55,14 @@ def navigate(self, url: str):
         self.browser_config.open(url)
         self.sleep(str(self.browser_config.get_navigation_sleep_duration()))
 
+    def new_tab(self, url: str):
+        self.click_position("100", "50%")   # focus the browser window
+        self.hotkey("ctrl", "t")
+        self.sleep("0.5")
+        self.write(url)
+        self.press("enter")
+        self.sleep(str(self.browser_config.get_navigation_sleep_duration()))
+
     def click_position(self, x: str, y: str):
         max_x, max_y = gui.size()
 
@@ -83,3 +96,19 @@ def screenshot(self, filename: str):
         filename = f'{self.params.evaluation_configuration.project}-{self.params.mech_group}-{filename}-{type(self.browser_config).__name__}-{self.browser_config.version}.jpg'
         filepath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../logs/screenshots', filename)
         gui.screenshot(filepath)
+
+    def report_leak(self):
+        self.navigate(f'https://a.test/report/?leak={self.params.mech_group}')
+
+    def assert_file_contains(self, filename: str, content: str):
+        filepath = os.path.join('/root/Downloads', filename)
+
+        if not os.path.isfile(filepath):
+            raise SimulationException(f'file-{filename}-does-not-exist')
+
+        with open(filepath, 'r') as f:
+            if content not in f.read():
+                raise SimulationException(f'file-{filename}-does-not-contain-{content}')
+
+    def open_file(self, filename: str):
+        self.navigate(f'file:///root/Downloads/{filename}')

bci/browser/interaction/simulation_exception.py

@@ -0,0 +1,5 @@
+class SimulationException(Exception):
+    """
+    Common class for exceptions thrown upon failed experiment assertions defined by script.cmd.
+    """
+    pass

bci/evaluations/custom/default_files/script.cmd

@@ -1,6 +1,8 @@
 # TODO - add your interaction script using the commands
 
+### Production commands
 # NAVIGATE url
+# NEW_TAB url
 # CLICK_POSITION x y    where x and y are absolute numbers or screen percentages
 # CLICK element_id      where element_id is one of one, two, three, four, five, six
 # WRITE text
@@ -9,4 +11,10 @@
 # RELEASE key
 # HOTKEY key1 key2 ...
 # SLEEP seconds         where seconds is a float or an int
+# REPORT_LEAK
+# ASSERT_FILE_CONTAINS file content     if the downloaded file exists and contains the given content as a substring, the evaluation continues
+#                                       otherwise the evaluation terminates and the exact reason is reported
+
+### Debugging commands
 # SCREENSHOT file_name
+# OPEN_FILE file

bci/web/blueprints/experiments.py

@@ -140,7 +140,7 @@ def report_leak_if_contains(expected_header_name: str, expected_header_value: st
     )
 
 
-@exp.route("/<string:project>/<string:experiment>/<string:file_name>.py")
+@exp.route("/<string:project>/<string:experiment>/<string:file_name>.py", methods=["GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"])
 def python_evaluation(project: str, experiment: str, file_name: str):
     """
     Evaluates the python script and returns its result.

bci/web/vue/src/interaction_script_mode.js

@@ -1,4 +1,4 @@
-const KEYWORDS = "NAVIGATE|CLICK_POSITION|CLICK|WRITE|PRESS|HOLD|RELEASE|HOTKEY|SLEEP|SCREENSHOT";
+const KEYWORDS = "NAVIGATE|NEW_TAB|CLICK_POSITION|CLICK|WRITE|PRESS|HOLD|RELEASE|HOTKEY|SLEEP|SCREENSHOT|REPORT_LEAK|ASSERT_FILE_CONTAINS|OPEN_FILE";
 
 ace.define("ace/mode/interaction_script_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module){"use strict";
     const oop = require("../lib/oop");

bci/web/vue/tailwind.config.js

@@ -1,5 +1,7 @@
+import flowbite from 'flowbite/plugin';
+
 /** @type {import('tailwindcss').Config} */
-module.exports = {
+export default {
   mode: "jit",
   content: [
     "./index.html",
@@ -8,7 +10,7 @@ module.exports = {
   ],
   darkMode: ['selector'],
   plugins: [
-    require('flowbite/plugin'),
+    flowbite,
   ],
   theme: {
     extend: {

docker-compose.yml

@@ -101,7 +101,7 @@ services:
       - ./experiments/pages/:/www/data/pages/:ro
       - ./experiments/res/:/www/data/res/:ro
       - ./logs/:/logs/:rw
-      - ./logs/:/logs/:rw
+      - ./logs/screenshots/:/www/data/screenshots/:ro
     ports:
       - "80:80"
       - "443:443"

experiments/pages/Support/Downloading/a.test/main/headers.json

@@ -0,0 +1,6 @@
+[
+    {
+        "key": "Header-Name",
+        "value": "Header-Value"
+    }
+]

experiments/pages/Support/Downloading/a.test/main/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<body>
+    <a href="https://a.test/res/short-text.txt" download>Download</a>
+    <script>
+        document.querySelector("a").click();
+    </script>
+</body>
+</html>

experiments/pages/Support/Downloading/script.cmd

@@ -0,0 +1,13 @@
+# Download file short-text.txt with content 123456789
+NAVIGATE https://a.test/Support/Downloading/main
+
+SLEEP 1
+
+# These commands would stop the evaluation and not report a leak
+# ASSERT_FILE_CONTAINS i-dont-exist.txt 234
+# ASSERT_FILE_CONTAINS short-text.txt i-am-not-in-the-file
+
+# This command will continue the evaluation and report a leak
+ASSERT_FILE_CONTAINS short-text.txt 234
+
+REPORT_LEAK

experiments/res/short-text.txt

@@ -0,0 +1 @@
+123456789

nginx/config/core_dev.conf

@@ -25,3 +25,8 @@ location /api/ {
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 }
+
+location /screenshots/ {
+    autoindex on;
+    alias /www/data/screenshots/;
+}

nginx/config/core_prod.conf

@@ -16,3 +16,8 @@ location /api/ {
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 }
+
+location /screenshots/ {
+    autoindex on;
+    alias /www/data/screenshots/;
+}