Merge branch 'ArchipelagoMW:main' into main

Author: EverydaySimpleDev

.gitignore

@@ -63,6 +63,7 @@ Output Logs/
 /installdelete.iss
 /data/user.kv
 /datapackage
+/datapackage_export.json
 /custom_worlds
 
 # Byte-compiled / optimized / DLL files

.run/Build APWorld.run.xml .run/Build APWorlds.run.xml.run/Build APWorld.run.xml renamed to .run/Build APWorlds.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build APWorld" type="PythonConfigurationType" factoryName="Python">
+  <configuration default="false" name="Build APWorlds" type="PythonConfigurationType" factoryName="Python">
     <module name="Archipelago" />
     <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />

Generate.py

@@ -119,9 +119,9 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
     else:
         meta_weights = None
 
-
-    player_id = 1
-    player_files = {}
+    player_id: int = 1
+    player_files: dict[int, str] = {}
+    player_errors: list[str] = []
     for file in os.scandir(args.player_files_path):
         fname = file.name
         if file.is_file() and not fname.startswith(".") and not fname.lower().endswith(".ini") and \
@@ -137,7 +137,11 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
                 weights_cache[fname] = tuple(weights_for_file)
 
             except Exception as e:
-                raise ValueError(f"File {fname} is invalid. Please fix your yaml.") from e
+                logging.exception(f"Exception reading weights in file {fname}")
+                player_errors.append(
+                    f"{len(player_errors) + 1}. "
+                    f"File {fname} is invalid. Please fix your yaml.\n{Utils.get_all_causes(e)}"
+                )
 
     # sort dict for consistent results across platforms:
     weights_cache = {key: value for key, value in sorted(weights_cache.items(), key=lambda k: k[0].casefold())}
@@ -152,6 +156,10 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
     args.multi = max(player_id - 1, args.multi)
 
     if args.multi == 0:
+        if player_errors:
+            errors = "\n\n".join(player_errors)
+            raise ValueError(f"Encountered {len(player_errors)} error(s) in player files. "
+                             f"See logs for full tracebacks.\n\n{errors}")
         raise ValueError(
             "No individual player files found and number of players is 0. "
             "Provide individual player files or specify the number of players via host.yaml or --multi."
@@ -161,6 +169,10 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
                  f"{seed_name} Seed {seed} with plando: {args.plando}")
 
     if not weights_cache:
+        if player_errors:
+            errors = "\n\n".join(player_errors)
+            raise ValueError(f"Encountered {len(player_errors)} error(s) in player files. "
+                             f"See logs for full tracebacks.\n\n{errors}")
         raise Exception(f"No weights found. "
                         f"Provide a general weights file ({args.weights_file_path}) or individual player files. "
                         f"A mix is also permitted.")
@@ -171,10 +183,6 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
     args.sprite_pool = dict.fromkeys(range(1, args.multi+1), None)
     args.name = {}
 
-    settings_cache: dict[str, tuple[argparse.Namespace, ...]] = \
-        {fname: (tuple(roll_settings(yaml, args.plando) for yaml in yamls) if args.sameoptions else None)
-         for fname, yamls in weights_cache.items()}
-
     if meta_weights:
         for category_name, category_dict in meta_weights.items():
             for key in category_dict:
@@ -197,7 +205,24 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
                             else:
                                 yaml[category_name][key] = option
 
-    player_path_cache = {}
+    settings_cache: dict[str, tuple[argparse.Namespace, ...]] = {fname: None for fname in weights_cache}
+    if args.sameoptions:
+        for fname, yamls in weights_cache.items():
+            try:
+                settings_cache[fname] = tuple(roll_settings(yaml, args.plando) for yaml in yamls)
+            except Exception as e:
+                logging.exception(f"Exception reading settings in file {fname}")
+                player_errors.append(
+                    f"{len(player_errors) + 1}. "
+                    f"File {fname} is invalid. Please fix your yaml.\n{Utils.get_all_causes(e)}"
+                )
+        # Exit early here to avoid throwing the same errors again later
+        if player_errors:
+            errors = "\n\n".join(player_errors)
+            raise ValueError(f"Encountered {len(player_errors)} error(s) in player files. "
+                             f"See logs for full tracebacks.\n\n{errors}")
+
+    player_path_cache: dict[int, str] = {}
     for player in range(1, args.multi + 1):
         player_path_cache[player] = player_files.get(player, args.weights_file_path)
     name_counter = Counter()
@@ -206,38 +231,62 @@ def main(args=None) -> tuple[argparse.Namespace, int]:
     player = 1
     while player <= args.multi:
         path = player_path_cache[player]
-        if path:
+        if not path:
+            player_errors.append(f'No weights specified for player {player}')
+            player += 1
+            continue
+
+        for doc_index, yaml in enumerate(weights_cache[path]):
+            name = yaml.get("name")
             try:
-                settings: tuple[argparse.Namespace, ...] = settings_cache[path] if settings_cache[path] else \
-                    tuple(roll_settings(yaml, args.plando) for yaml in weights_cache[path])
-                for settingsObject in settings:
-                    for k, v in vars(settingsObject).items():
-                        if v is not None:
-                            try:
-                                getattr(args, k)[player] = v
-                            except AttributeError:
-                                setattr(args, k, {player: v})
-                            except Exception as e:
-                                raise Exception(f"Error setting {k} to {v} for player {player}") from e
-
-                    # name was not specified
-                    if player not in args.name:
-                        if path == args.weights_file_path:
-                            # weights file, so we need to make the name unique
-                            args.name[player] = f"Player{player}"
-                        else:
-                            # use the filename
-                            args.name[player] = os.path.splitext(os.path.split(path)[-1])[0]
-                    args.name[player] = handle_name(args.name[player], player, name_counter)
-
-                    player += 1
+                # Use the cached settings object if it exists, otherwise roll settings within the try-catch
+                # Invariant: settings_cache[path] and weights_cache[path] have the same length
+                settingsObject: argparse.Namespace = (
+                    settings_cache[path][doc_index]
+                    if settings_cache[path]
+                    else roll_settings(yaml, args.plando)
+                )
+                
+                for k, v in vars(settingsObject).items():
+                    if v is not None:
+                        try:
+                            getattr(args, k)[player] = v
+                        except AttributeError:
+                            setattr(args, k, {player: v})
+                        except Exception as e:
+                            raise Exception(f"Error setting {k} to {v} for player {player}") from e
+
+                # name was not specified
+                if player not in args.name:
+                    if path == args.weights_file_path:
+                        # weights file, so we need to make the name unique
+                        args.name[player] = f"Player{player}"
+                    else:
+                        # use the filename
+                        args.name[player] = os.path.splitext(os.path.split(path)[-1])[0]
+                args.name[player] = handle_name(args.name[player], player, name_counter)
+
             except Exception as e:
-                raise ValueError(f"File {path} is invalid. Please fix your yaml.") from e
-        else:
-            raise RuntimeError(f'No weights specified for player {player}')
+                logging.exception(f"Exception reading settings in file {path} document #{doc_index + 1} "
+                                  f"(name: {args.name.get(player, name)})")
+                player_errors.append(
+                    f"{len(player_errors) + 1}. "
+                    f"File {path} document #{doc_index + 1} (name: {args.name.get(player, name)}) is invalid. "
+                    f"Please fix your yaml.\n{Utils.get_all_causes(e)}")
+
+            # increment for each yaml document in the file
+            player += 1
 
     if len(set(name.lower() for name in args.name.values())) != len(args.name):
-        raise Exception(f"Names have to be unique. Names: {Counter(name.lower() for name in args.name.values())}")
+        player_errors.append(
+            f"{len(player_errors) + 1}. "
+            f"Names have to be unique. Names: {Counter(name.lower() for name in args.name.values())}"
+        )
+
+    if player_errors:
+        errors = "\n\n".join(player_errors)
+        raise ValueError(f"Encountered {len(player_errors)} error(s) in player files. "
+                         f"See logs for full tracebacks.\n\n{errors}")
 
     return args, seed
 

MultiServer.py

@@ -69,6 +69,12 @@ def remove_from_list(container, value):
 
 
 def pop_from_container(container, value):
+    if isinstance(container, list) and isinstance(value, int) and len(container) <= value:
+        return container
+
+    if isinstance(container, dict) and value not in container:
+        return container
+
     try:
         container.pop(value)
     except ValueError:
@@ -911,12 +917,6 @@ async def server(websocket: "ServerConnection", path: str = "/", ctx: Context =
 
 
 async def on_client_connected(ctx: Context, client: Client):
-    players = []
-    for team, clients in ctx.clients.items():
-        for slot, connected_clients in clients.items():
-            if connected_clients:
-                name = ctx.player_names[team, slot]
-                players.append(NetworkPlayer(team, slot, ctx.name_aliases.get((team, slot), name), name))
     games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)}
     games.add("Archipelago")
     await ctx.send_msgs(client, [{
@@ -1364,7 +1364,10 @@ def get_help_text(self) -> str:
                         argname += "=" + parameter.default
                 argtext += argname
                 argtext += " "
-            doctext = '\n    '.join(inspect.getdoc(method).split('\n'))
+            method_doc = inspect.getdoc(method)
+            if method_doc is None:
+                method_doc = "(missing help text)"
+            doctext = "\n    ".join(method_doc.split("\n"))
             s += f"{self.marker}{command} {argtext}\n    {doctext}\n"
         return s
 

OptionsCreator.py

@@ -509,8 +509,10 @@ def randomize_option(instance: Widget, value: str):
                     self.options[name] = "random-" + str(self.options[name])
                 else:
                     self.options[name] = self.options[name].replace("random-", "")
-                    if self.options[name].isnumeric() or self.options[name] in ("True", "False"):
-                        self.options[name] = eval(self.options[name])
+                    if self.options[name].isnumeric():
+                        self.options[name] = int(self.options[name])
+                    elif self.options[name] in ("True", "False"):
+                        self.options[name] = self.options[name] == "True"
 
                 base_object = instance.parent.parent
                 label_object = instance.parent
@@ -632,7 +634,7 @@ def world_button_action(world_btn: WorldButton):
             self.create_options_panel(world_btn)
 
         for world, cls in sorted(AutoWorldRegister.world_types.items(), key=lambda x: x[0]):
-            if world == "Archipelago":
+            if cls.hidden:
                 continue
             world_text = MDButtonText(text=world, size_hint_y=None, width=dp(150),
                                       pos_hint={"x": 0.03, "center_y": 0.5})

README.md

@@ -83,6 +83,8 @@ Currently, the following games are supported:
 * Celeste (Open World)
 * Choo-Choo Charles
 * APQuest
+* Satisfactory
+* EarthBound
 
 For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/).
 Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled

Utils.py

@@ -22,6 +22,7 @@
 from time import sleep
 from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union, TypeGuard
 from yaml import load, load_all, dump
+from pathspec import PathSpec, GitIgnoreSpec
 
 try:
     from yaml import CLoader as UnsafeLoader, CSafeLoader as SafeLoader, CDumper as Dumper
@@ -48,7 +49,7 @@ def as_simple_string(self) -> str:
         return ".".join(str(item) for item in self)
 
 
-__version__ = "0.6.5"
+__version__ = "0.6.7"
 version_tuple = tuplize_version(__version__)
 
 is_linux = sys.platform.startswith("linux")
@@ -387,6 +388,14 @@ def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> N
             logging.debug(f"Could not store data package: {e}")
 
 
+def read_apignore(filename: str | pathlib.Path) -> PathSpec | None:
+    try:
+        with open(filename) as ignore_file:
+            return GitIgnoreSpec.from_lines(ignore_file)
+    except FileNotFoundError:
+        return None
+
+
 def get_default_adjuster_settings(game_name: str) -> Namespace:
     import LttPAdjuster
     adjuster_settings = Namespace()
@@ -1222,3 +1231,35 @@ def weakref_cb(_, q=self._work_queue):
             t.start()
             self._threads.add(t)
             # NOTE: don't add to _threads_queues so we don't block on shutdown
+
+
+def get_full_typename(t: type) -> str:
+    """Returns the full qualified name of a type, including its module (if not builtins)."""
+    module = t.__module__
+    if module and module != "builtins":
+        return f"{module}.{t.__qualname__}"
+    return t.__qualname__
+
+
+def get_all_causes(ex: Exception) -> str:
+    """Return a string describing the recursive causes of this exception.
+
+    :param ex: The exception to be described.
+    :return A multiline string starting with the initial exception on the first line and each resulting exception
+            on subsequent lines with progressive indentation.
+
+            For example:
+
+            ```
+            Exception: Invalid value 'bad'.
+             Which caused: Options.OptionError: Error generating option
+              Which caused: ValueError: File bad.yaml is invalid.
+            ```
+    """
+    cause = ex
+    causes = [f"{get_full_typename(type(ex))}: {ex}"]
+    while cause := cause.__cause__:
+        causes.append(f"{get_full_typename(type(cause))}: {cause}")
+    top = causes[-1]
+    others = "".join(f"\n{' ' * (i + 1)}Which caused: {c}" for i, c in enumerate(reversed(causes[:-1])))
+    return f"{top}{others}"

WebHostLib/__init__.py

@@ -23,26 +23,30 @@
 app.jinja_env.filters['all'] = all
 app.jinja_env.filters['get_file_safe_name'] = get_file_safe_name
 
+# overwrites of flask default config
+app.config["DEBUG"] = False
+app.config["PORT"] = 80
+app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
+app.config["MAX_CONTENT_LENGTH"] = 64 * 1024 * 1024  # 64 megabyte limit
+# if you want to deploy, make sure you have a non-guessable secret key
+app.config["SECRET_KEY"] = bytes(socket.gethostname(), encoding="utf-8")
+app.config["SESSION_PERMANENT"] = True
+app.config["MAX_FORM_MEMORY_SIZE"] = 2 * 1024 * 1024  # 2 MB, needed for large option pages such as SC2
+
+# custom config
 app.config["SELFHOST"] = True  # application process is in charge of running the websites
 app.config["GENERATORS"] = 8  # maximum concurrent world gens
 app.config["HOSTERS"] = 8  # maximum concurrent room hosters
 app.config["SELFLAUNCH"] = True  # application process is in charge of launching Rooms.
 app.config["SELFLAUNCHCERT"] = None  # can point to a SSL Certificate to encrypt Room websocket connections
 app.config["SELFLAUNCHKEY"] = None  # can point to a SSL Certificate Key to encrypt Room websocket connections
 app.config["SELFGEN"] = True  # application process is in charge of scheduling Generations.
-app.config["DEBUG"] = False
-app.config["PORT"] = 80
-app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
-app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024  # 64 megabyte limit
-# if you want to deploy, make sure you have a non-guessable secret key
-app.config["SECRET_KEY"] = bytes(socket.gethostname(), encoding="utf-8")
 # at what amount of worlds should scheduling be used, instead of rolling in the web-thread
 app.config["JOB_THRESHOLD"] = 1
 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable.
 app.config["JOB_TIME"] = 600
 # memory limit for generator processes in bytes
 app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296
-app.config['SESSION_PERMANENT'] = True
 
 # waitress uses one thread for I/O, these are for processing of views that then get sent
 # archipelago.gg uses gunicorn + nginx; ignoring this option

WebHostLib/customserver.py

@@ -325,7 +325,7 @@ async def start_room(room_id):
 
             except (KeyboardInterrupt, SystemExit):
                 if ctx.saving:
-                    ctx._save()
+                    ctx._save(True)
                     setattr(asyncio.current_task(), "save", None)
             except Exception as e:
                 with db_session:
@@ -336,7 +336,7 @@ async def start_room(room_id):
                 raise
             else:
                 if ctx.saving:
-                    ctx._save()
+                    ctx._save(True)
                     setattr(asyncio.current_task(), "save", None)
             finally:
                 try:

WebHostLib/misc.py

@@ -128,8 +128,13 @@ def tutorial_landing():
                 "authors": tutorial.authors,
                 "language": tutorial.language
             }
-    tutorials = {world_name: tutorials for world_name, tutorials in title_sorted(
-        tutorials.items(), key=lambda element: "\x00" if element[0] == "Archipelago" else worlds[element[0]].game)}
+
+    worlds = dict(
+        title_sorted(
+            worlds.items(), key=lambda element: "\x00" if element[0] == "Archipelago" else worlds[element[0]].game
+        )
+    )
+
     return render_template("tutorialLanding.html", worlds=worlds, tutorials=tutorials)