From 09f69532cd054bbb03c16e17b741628701ab46d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:09:51 +0000 Subject: [PATCH 1/3] Initial plan From 13fdc1fbc22d1fc611dc352e2abec34f21f56988 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:19:41 +0000 Subject: [PATCH 2/3] feat: add Moodle compatibility layer Co-authored-by: erseco <1876752+erseco@users.noreply.github.com> --- README.md | 13 ++ src/py_moodle/auth.py | 33 +++-- src/py_moodle/compat.py | 276 +++++++++++++++++++++++++++++++++++++++ src/py_moodle/folder.py | 19 +-- src/py_moodle/module.py | 22 ++-- src/py_moodle/session.py | 26 ++-- tests/test_compat.py | 161 +++++++++++++++++++++++ 7 files changed, 506 insertions(+), 44 deletions(-) create mode 100644 src/py_moodle/compat.py create mode 100644 tests/test_compat.py diff --git a/README.md b/README.md index 1298ce6..d11cb7d 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,19 @@ --- +## Moodle Compatibility + +`py-moodle` interacts with Moodle through authenticated web sessions, HTML forms, and page parsing. To make those flows more resilient across Moodle releases, the library now centralizes version-sensitive logic in `py_moodle.compat`. + +- **Version detection** happens during login/session initialization. The library first tries `core_webservice_get_site_info` when a webservice token is available, then falls back to probing the dashboard HTML (`/my/`) for Moodle release metadata. +- **Version-aware strategies** are grouped into compatibility ranges instead of scattering selectors throughout the codebase. The current built-in strategies cover legacy Moodle 3.x layouts and modern Moodle 4.x/5.x layouts. +- **Feature probing remains in place** when version detection is not enough. Each strategy can try multiple selectors or form patterns before failing. +- **Fragile flows should read selectors from the compatibility layer** so future Moodle HTML changes are isolated to one module. + +At the moment, representative compatibility handling has been wired into login/session bootstrap, generic module form parsing, and folder page scraping. When a new Moodle release changes one of these flows, the recommended fix is to update `py_moodle.compat` and add a regression test for the new selector or workflow. + +--- + ## Installation You will need Python 3.8+ and `pip`. diff --git a/src/py_moodle/auth.py b/src/py_moodle/auth.py index fa2b693..1585308 100644 --- a/src/py_moodle/auth.py +++ b/src/py_moodle/auth.py @@ -9,7 +9,8 @@ from typing import Optional import requests -from bs4 import BeautifulSoup + +from .compat import DEFAULT_COMPATIBILITY, detect_moodle_compatibility class LoginError(Exception): @@ -50,6 +51,8 @@ def __init__( self.pre_configured_token = pre_configured_token self.debug = debug self.webservice_token = None + self.compatibility = DEFAULT_COMPATIBILITY + self.moodle_version = None def login(self) -> requests.Session: """Authenticate the user and return a Moodle session. @@ -86,6 +89,18 @@ def login(self) -> requests.Session: if self.debug: print(f"[DEBUG] Could not obtain webservice token: {e}") self.webservice_token = None + compatibility_context = detect_moodle_compatibility( + self.session, self.base_url, token=self.webservice_token + ) + self.compatibility = compatibility_context.strategy + self.moodle_version = compatibility_context.version + if self.debug: + print( + "[DEBUG] Moodle compatibility:" + f" version={self.moodle_version.raw}" + f" source={self.moodle_version.source}" + f" strategy={self.compatibility.version_range}" + ) return self.session def _standard_login(self): @@ -96,9 +111,7 @@ def _standard_login(self): resp = self.session.get(login_url) if self.debug: print(f"[DEBUG] Response {resp.status_code} {resp.url}") - soup = BeautifulSoup(resp.text, "lxml") - logintoken_input = soup.find("input", {"name": "logintoken"}) - logintoken = logintoken_input["value"] if logintoken_input else "" + logintoken = self.compatibility.extract_login_token(resp.text) payload = { "username": self.username, @@ -123,8 +136,6 @@ def _cas_login(self): Perform CAS login flow programmatically (no browser interaction). Maintains cookies and follows the CAS ticket flow. """ - import re - # Step 1: Get CAS login page to extract execution token service_url = f"{self.base_url}/login/index.php" from urllib.parse import quote @@ -219,12 +230,10 @@ def _get_sesskey(self) -> str: """ dashboard_url = f"{self.base_url}/my/" resp = self.session.get(dashboard_url) - match = re.search(r'"sesskey":"([^"]+)"', resp.text) - if not match: - match = re.search(r"M\.cfg\.sesskey\s*=\s*[\"']([^\"']+)[\"']", resp.text) - if not match: + sesskey = self.compatibility.extract_sesskey(resp.text) + if not sesskey: raise LoginError("Could not extract sesskey after login.") - return match.group(1) + return sesskey def _get_webservice_token(self) -> Optional[str]: """ @@ -345,6 +354,8 @@ def login( # Attach tokens to session for convenience session.sesskey = getattr(auth, "sesskey", None) session.webservice_token = getattr(auth, "webservice_token", None) + session.moodle_version = getattr(auth, "moodle_version", None) + session.moodle_compat = getattr(auth, "compatibility", DEFAULT_COMPATIBILITY) return session diff --git a/src/py_moodle/compat.py b/src/py_moodle/compat.py new file mode 100644 index 0000000..201cfb6 --- /dev/null +++ b/src/py_moodle/compat.py @@ -0,0 +1,276 @@ +"""Compatibility helpers for Moodle version-specific HTML parsing.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Optional + +import requests +from bs4 import BeautifulSoup, Tag + + +@dataclass(frozen=True) +class MoodleVersion: + """Normalized Moodle version information.""" + + raw: str + major: Optional[int] = None + minor: Optional[int] = None + patch: Optional[int] = None + source: str = "unknown" + + +@dataclass(frozen=True) +class MoodleCompatibilityContext: + """Detected Moodle compatibility information.""" + + version: MoodleVersion + strategy: "BaseCompatibilityStrategy" + + +class BaseCompatibilityStrategy: + """Base strategy for Moodle HTML compatibility helpers.""" + + version_range = "generic" + login_token_selectors = ( + 'input[name="logintoken"]', + 'input[name="login_token"]', + ) + sesskey_patterns = ( + r'"sesskey":"([^"]+)"', + r"M\.cfg\.sesskey\s*=\s*[\"']([^\"']+)[\"']", + r'["\']sesskey["\']\s*:\s*["\']([^"\']+)["\']', + ) + error_selectors = ( + ".error", + ".errormessage", + ".alert-danger", + 'div[data-fieldtype="error"]', + ".notifyproblem", + ) + modedit_form_selectors = ( + 'form[action*="modedit.php"]', + "form#mform1", + 'form[id^="mform"]', + ) + user_fullname_selectors = ( + ".usermenu .usertext", + '[data-region="usermenu"] .usertext', + '[data-region="usermenu"] .logininfo', + ".logininfo a:last-of-type", + ) + folder_file_selectors = ( + '.folder_tree a[href*="/pluginfile.php/"]', + '.foldertree a[href*="/pluginfile.php/"]', + 'a[href*="/pluginfile.php/"]', + ) + + def extract_login_token(self, html: str) -> str: + """Extract the login token from a Moodle login page.""" + soup = BeautifulSoup(html, "lxml") + for selector in self.login_token_selectors: + token_input = soup.select_one(selector) + if token_input and token_input.get("value"): + return token_input["value"] + return "" + + def extract_sesskey(self, html: str) -> Optional[str]: + """Extract a sesskey from Moodle HTML or embedded JavaScript.""" + for pattern in self.sesskey_patterns: + match = re.search(pattern, html) + if match: + return match.group(1) + return None + + def extract_error_message(self, soup: BeautifulSoup) -> Optional[str]: + """Extract a visible Moodle error message from a response page.""" + for selector in self.error_selectors: + error_node = soup.select_one(selector) + if error_node: + message = error_node.get_text(strip=True) + if message: + return message + return None + + def find_modedit_form(self, soup: BeautifulSoup) -> Optional[Tag]: + """Locate the module edit form in a Moodle page.""" + for selector in self.modedit_form_selectors: + form = soup.select_one(selector) + if form is not None: + return form + return None + + def extract_user_fullname(self, soup: BeautifulSoup) -> Optional[str]: + """Extract the current user's visible fullname from the dashboard.""" + for selector in self.user_fullname_selectors: + node = soup.select_one(selector) + if node: + fullname = node.get_text(" ", strip=True) + if fullname: + return fullname + return None + + def extract_folder_filenames(self, soup: BeautifulSoup) -> list[str]: + """Extract filenames from a folder activity page.""" + filenames: list[str] = [] + for selector in self.folder_file_selectors: + for link in soup.select(selector): + href = link.get("href", "") + if "pluginfile.php" not in href: + continue + filename = link.get_text(strip=True) + if filename: + filenames.append(filename) + if filenames: + break + return sorted(set(filenames)) + + +class LegacyCompatibilityStrategy(BaseCompatibilityStrategy): + """Compatibility strategy for older Moodle 3.x layouts.""" + + version_range = "3.x" + user_fullname_selectors = ( + ".logininfo a:last-of-type", + '[data-region="usermenu"] .logininfo', + ".usermenu .usertext", + ) + folder_file_selectors = ( + '.foldertree a[href*="/pluginfile.php/"]', + '.folder_tree a[href*="/pluginfile.php/"]', + 'a[href*="/pluginfile.php/"]', + ) + + +class ModernCompatibilityStrategy(BaseCompatibilityStrategy): + """Compatibility strategy for Moodle 4.x and newer layouts.""" + + version_range = "4.x+" + user_fullname_selectors = ( + '[data-region="usermenu"] .usertext', + ".usermenu .usertext", + '[data-region="usermenu"] .logininfo', + ".logininfo a:last-of-type", + ) + + +DEFAULT_COMPATIBILITY = ModernCompatibilityStrategy() +LEGACY_COMPATIBILITY = LegacyCompatibilityStrategy() + + +def parse_moodle_version( + raw_version: Optional[str], source: str = "unknown" +) -> MoodleVersion: + """Parse Moodle version text into a normalized structure.""" + cleaned = (raw_version or "").strip() + match = re.search(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", cleaned) + if not match: + return MoodleVersion(raw=cleaned or "unknown", source=source) + major, minor, patch = match.groups() + return MoodleVersion( + raw=cleaned, + major=int(major), + minor=int(minor) if minor is not None else None, + patch=int(patch) if patch is not None else None, + source=source, + ) + + +def extract_version_from_dashboard(html: str) -> MoodleVersion: + """Extract Moodle version information from dashboard HTML.""" + soup = BeautifulSoup(html, "lxml") + generator = soup.find("meta", attrs={"name": re.compile("^generator$", re.I)}) + if generator and generator.get("content"): + version = parse_moodle_version(generator["content"], source="html-meta") + if version.major is not None: + return version + + patterns = ( + (r'"release"\s*:\s*"([^"]+)"', "html-js"), + (r"M\.cfg\.release\s*=\s*[\"']([^\"']+)[\"']", "html-js"), + ) + for pattern, source in patterns: + match = re.search(pattern, html) + if match: + version = parse_moodle_version(match.group(1), source=source) + if version.major is not None: + return version + + return MoodleVersion(raw="unknown", source="html") + + +def get_strategy_for_version(version: MoodleVersion) -> BaseCompatibilityStrategy: + """Return the selector strategy matching a Moodle version.""" + if version.major is not None and version.major < 4: + return LEGACY_COMPATIBILITY + return DEFAULT_COMPATIBILITY + + +def detect_moodle_compatibility( + session: requests.Session, base_url: str, token: Optional[str] = None +) -> MoodleCompatibilityContext: + """Detect the Moodle version and return the matching compatibility strategy.""" + if token is None: + token = getattr(session, "webservice_token", None) + + version = MoodleVersion(raw="unknown") + if token: + request_params = { + "moodlewsrestformat": "json", + "wsfunction": "core_webservice_get_site_info", + "wstoken": token, + } + try: + response = session.post( + f"{base_url}/webservice/rest/server.php", + params=request_params, + timeout=30, + ) + response.raise_for_status() + data = response.json() + if isinstance(data, dict) and "exception" not in data: + release = data.get("release") or data.get("version") + version = parse_moodle_version(release, source="webservice") + except (requests.RequestException, ValueError, TypeError, json.JSONDecodeError): + pass + + if version.major is None: + try: + response = session.get(f"{base_url}/my/") + response.raise_for_status() + version = extract_version_from_dashboard(response.text) + except requests.RequestException: + pass + + return MoodleCompatibilityContext( + version=version, + strategy=get_strategy_for_version(version), + ) + + +def get_session_compatibility(session: requests.Session) -> BaseCompatibilityStrategy: + """Return the compatibility strategy attached to a session.""" + compatibility = getattr(session, "moodle_compat", None) + if compatibility is not None: + return compatibility + version = getattr(session, "moodle_version", None) + if isinstance(version, MoodleVersion): + return get_strategy_for_version(version) + return DEFAULT_COMPATIBILITY + + +__all__ = [ + "BaseCompatibilityStrategy", + "DEFAULT_COMPATIBILITY", + "LegacyCompatibilityStrategy", + "ModernCompatibilityStrategy", + "MoodleCompatibilityContext", + "MoodleVersion", + "detect_moodle_compatibility", + "extract_version_from_dashboard", + "get_session_compatibility", + "get_strategy_for_version", + "parse_moodle_version", +] diff --git a/src/py_moodle/folder.py b/src/py_moodle/folder.py index c09a41d..9b79d71 100644 --- a/src/py_moodle/folder.py +++ b/src/py_moodle/folder.py @@ -17,6 +17,7 @@ import requests from bs4 import BeautifulSoup +from .compat import get_session_compatibility from .course import get_course_context_id from .draftfile import upload_file_to_draft_area @@ -100,9 +101,10 @@ def _get_current_user_fullname(session: requests.Session, base_url: str) -> str: my_page_resp = session.get(f"{base_url}/my/") my_page_resp.raise_for_status() soup = BeautifulSoup(my_page_resp.text, "lxml") - user_menu = soup.select_one(".usermenu .usertext") - if user_menu: - return user_menu.get_text(strip=True) + compatibility = get_session_compatibility(session) + fullname = compatibility.extract_user_fullname(soup) + if fullname: + return fullname except Exception: # Fallback if scraping fails return "Admin User" @@ -280,15 +282,8 @@ def list_folder_content( resp = session.get(view_url) resp.raise_for_status() soup = BeautifulSoup(resp.text, "lxml") - file_links = soup.select( - '.folder_tree a[href*="/pluginfile.php/"], .foldertree a[href*="/pluginfile.php/"]' - ) - filenames = [ - link.text.strip() - for link in file_links - if "pluginfile.php" in link.get("href", "") - ] - return sorted(list(set(filenames))) + compatibility = get_session_compatibility(session) + return compatibility.extract_folder_filenames(soup) except requests.RequestException as e: raise MoodleFolderError( f"Failed to load folder content page (cmid={cmid}): {e}" diff --git a/src/py_moodle/module.py b/src/py_moodle/module.py index ae22e63..8665ebc 100644 --- a/src/py_moodle/module.py +++ b/src/py_moodle/module.py @@ -13,6 +13,7 @@ import requests from bs4 import BeautifulSoup +from py_moodle.compat import get_session_compatibility from py_moodle.course import MoodleCourseError, get_course_with_sections_and_modules # --- Cache for module IDs --- @@ -118,6 +119,7 @@ def add_generic_module( url = f"{base_url}/course/modedit.php" headers = {"Content-Type": "application/x-www-form-urlencoded"} encoded_payload = urllib.parse.urlencode(full_payload) + compatibility = get_session_compatibility(session) resp = session.post( url, data=encoded_payload, headers=headers, allow_redirects=False ) @@ -127,12 +129,9 @@ def add_generic_module( # A 200 OK status almost always means a silent failure. # Parse the HTML to find Moodle's error message. soup = BeautifulSoup(resp.text, "lxml") - error_div = soup.select_one( - ".error, .errormessage, .alert-danger, div[data-fieldtype=error]" - ) - if error_div: + error_message = compatibility.extract_error_message(soup) + if error_message: # Found a specific error message. - error_message = error_div.get_text(strip=True) raise MoodleModuleError( f"Form submission failed. Moodle error: {error_message}" ) @@ -192,7 +191,8 @@ def update_generic_module( raise MoodleModuleError(f"Failed to load module edit page for cmid {cmid}: {e}") # 2. Parse the form and extract all input, textarea, and select fields - form = soup.select_one('form[action*="modedit.php"]') + compatibility = get_session_compatibility(session) + form = compatibility.find_modedit_form(soup) if not form: raise MoodleModuleError("Could not find the edit form on the page.") @@ -235,13 +235,9 @@ def update_generic_module( else: # If we get a 200, it's likely an error page. Check for Moodle error notifications. error_soup = BeautifulSoup(resp.text, "lxml") - error_div = error_soup.select_one( - ".error, .errormessage, .alert-danger, div[data-fieldtype=error]" - ) - if error_div: - raise MoodleModuleError( - f"Failed to update module: {error_div.get_text(strip=True)}" - ) + error_message = compatibility.extract_error_message(error_soup) + if error_message: + raise MoodleModuleError(f"Failed to update module: {error_message}") raise MoodleModuleError( f"Failed to update module. Status: {resp.status_code}. Response: {resp.text[:500]}" ) diff --git a/src/py_moodle/session.py b/src/py_moodle/session.py index d38b7c5..04ea624 100644 --- a/src/py_moodle/session.py +++ b/src/py_moodle/session.py @@ -15,6 +15,7 @@ from typing import Any, Dict, Optional from .auth import LoginError, login +from .compat import DEFAULT_COMPATIBILITY, get_session_compatibility class MoodleSessionError(RuntimeError): @@ -33,6 +34,8 @@ def __init__(self, settings: "Settings") -> None: self._session: requests.Session | None = None self._sesskey: str | None = None self._token: str | None = None + self._compatibility = DEFAULT_COMPATIBILITY + self._moodle_version = None # ------------- internal helpers ------------- def _login(self) -> None: @@ -55,18 +58,13 @@ def _login(self) -> None: ) self._token = getattr(session, "webservice_token", None) self._sesskey = getattr(session, "sesskey", None) + self._compatibility = get_session_compatibility(session) + self._moodle_version = getattr(session, "moodle_version", None) # Fallback extraction if sesskey was not attached by login() if not self._sesskey: - import re - resp = session.get(f"{self.settings.url}/my/") - m = re.search(r'"sesskey":"([a-zA-Z0-9]+)"', resp.text) - if not m: - m = re.search( - r"M\.cfg\.sesskey\s*=\s*['\"]([a-zA-Z0-9]+)['\"]", resp.text - ) - self._sesskey = m.group(1) if m else None + self._sesskey = self._compatibility.extract_sesskey(resp.text) # Validate we have at least one usable token if not self._token and not self._sesskey: @@ -98,6 +96,18 @@ def token(self) -> str | None: self._login() return self._token + @property + def compatibility(self): + """Return the compatibility strategy selected for the current session.""" + self._login() + return self._compatibility + + @property + def moodle_version(self): + """Return detected Moodle version information when available.""" + self._login() + return self._moodle_version + def call( self, wsfunction: str, diff --git a/tests/test_compat.py b/tests/test_compat.py new file mode 100644 index 0000000..f2f38e6 --- /dev/null +++ b/tests/test_compat.py @@ -0,0 +1,161 @@ +"""Unit tests for Moodle compatibility helpers.""" + +from py_moodle.compat import ( + LegacyCompatibilityStrategy, + ModernCompatibilityStrategy, + detect_moodle_compatibility, +) +from py_moodle.folder import _get_current_user_fullname, list_folder_content +from py_moodle.module import update_generic_module + + +class FakeResponse: + """Minimal response object for unit tests.""" + + def __init__(self, text="", status_code=200, json_data=None): + self.text = text + self.status_code = status_code + self._json_data = json_data + + def raise_for_status(self): + """Raise for failing HTTP statuses.""" + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def json(self): + """Return the configured JSON payload.""" + return self._json_data + + +class FakeSession: + """Minimal requests.Session-like object for compatibility tests.""" + + def __init__(self, get_responses=None, post_responses=None): + self.get_responses = list(get_responses or []) + self.post_responses = list(post_responses or []) + self.post_calls = [] + self.moodle_compat = ModernCompatibilityStrategy() + + def get(self, url, *args, **kwargs): + """Return a queued GET response.""" + if not self.get_responses: + raise AssertionError(f"Unexpected GET request to {url}") + return self.get_responses.pop(0) + + def post(self, url, *args, **kwargs): + """Return a queued POST response and store the call.""" + self.post_calls.append({"url": url, "args": args, "kwargs": kwargs}) + if not self.post_responses: + raise AssertionError(f"Unexpected POST request to {url}") + return self.post_responses.pop(0) + + +def test_detect_moodle_compatibility_prefers_webservice_version(): + """Compatibility detection should use site info when a token is available.""" + session = FakeSession( + post_responses=[ + FakeResponse( + json_data={ + "release": "4.5.2+ (Build: 20241001)", + "version": "2024100100", + } + ) + ] + ) + + compatibility = detect_moodle_compatibility( + session, "https://moodle.example.test", token="token" + ) + + assert compatibility.version.major == 4 + assert compatibility.version.minor == 5 + assert compatibility.version.source == "webservice" + assert isinstance(compatibility.strategy, ModernCompatibilityStrategy) + + +def test_detect_moodle_compatibility_falls_back_to_dashboard_html(): + """Compatibility detection should parse dashboard metadata without a token.""" + html = """ + +
+ + + + """ + session = FakeSession(get_responses=[FakeResponse(text=html)]) + + compatibility = detect_moodle_compatibility(session, "https://moodle.example.test") + + assert compatibility.version.major == 3 + assert compatibility.version.minor == 11 + assert compatibility.version.source == "html-meta" + assert isinstance(compatibility.strategy, LegacyCompatibilityStrategy) + + +def test_update_generic_module_uses_compatibility_form_fallback(): + """Module updates should accept compatibility fallback selectors for modedit.""" + html = """ + + + + + + """ + session = FakeSession( + get_responses=[FakeResponse(text=html)], + post_responses=[FakeResponse(status_code=303)], + ) + + assert ( + update_generic_module( + session, + "https://moodle.example.test", + 17, + specific_payload={"name": "New name"}, + ) + is True + ) + assert session.post_calls[0]["kwargs"]["data"]["name"] == "New name" + + +def test_folder_content_uses_compatibility_selector_fallback(): + """Folder listing should work even when legacy folder tree classes are missing.""" + html = """ + + + + + + """ + session = FakeSession(get_responses=[FakeResponse(text=html)]) + + assert list_folder_content(session, "https://moodle.example.test", 24) == [ + "test.pdf" + ] + + +def test_current_user_fullname_uses_data_region_selector(): + """Folder file renames should support modern user menu markup.""" + html = """ + + +