From 4c15e7f893eed2d7b8e2c0490945e1d5adad8fec Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 17 May 2025 09:46:45 +0200 Subject: [PATCH 1/5] more bugfix --- CHANGELOG.md | 2 ++ .../compatibility_hints.py | 19 +++++++++++++++++++ docs/source/index.rst | 2 +- tests/conf.py | 6 +++--- tests/conf_private.py.EXAMPLE | 6 +++--- tests/test_caldav.py | 8 ++++---- 6 files changed, 32 insertions(+), 11 deletions(-) rename tests/compatibility_issues.py => caldav/compatibility_hints.py (95%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7141b482..931c336d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,10 +21,12 @@ Python 3.7 is no longer tested - but it should work. Please file a bug report i ### Changed * Some exotic servers may return object URLs on search, but it does not work out to fetch the calendar data. Now it will log an error instead of raising an error in such cases. +* The `tests/compatibility_issues.py` has been moved to `caldav/compatibility_hints.py`, this to make it available for a caldav-server-tester-tool that I'm splitting off to a separate project/repository, and also to make https://github.com/python-caldav/caldav/issues/402 possible. #### Refactoring * Minor code cleanups by github user @ArtemIsmagilov in https://github.com/python-caldav/caldav/pull/456 +* The very much overgrown `objects.py`-file has been split into three. #### Test framework diff --git a/tests/compatibility_issues.py b/caldav/compatibility_hints.py similarity index 95% rename from tests/compatibility_issues.py rename to caldav/compatibility_hints.py index 75464b30..f9f32c99 100644 --- a/tests/compatibility_issues.py +++ b/caldav/compatibility_hints.py @@ -1,4 +1,23 @@ # fmt: off +"""This text was updated 2025-05-17. The plan is to reorganize this +file a lot over the next few months, see +https://github.com/python-caldav/caldav/issues/402 + +This file serves as a database of different compatibility issues we've +encountered while working on the caldav library, and descriptions on +how the well-known servers behave. + +As for now, this is a list of binary "flags" that could be turned on +or off. My experience is that there are often neuances, so the +compatibility matrix will be changed from being a list of flags to a +key=value store in the near future (at least, that's the plan). + +The issues may be grouped together, maybe even organized +hierarchically. I did consider organizing the compatibility issues in +some more advanced way, but I don't want to overcomplicate things - I +will try out the key-value-approach first. +""" + ## The lists below are specifying what tests should be skipped or ## modified to accept non-conforming resultsets from the different ## calendar servers. In addition there are some hacks in the library diff --git a/docs/source/index.rst b/docs/source/index.rst index 853f4807..c435f8ec 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -307,7 +307,7 @@ Compatibility tend to be a moving target, and I rarely recheck if things works in newer versions of the software after I find an incompatibility) -The test suite is regularly run against several calendar servers, see https://github.com/python-caldav/caldav/issues/45 for the latest updates. See ``tests/compatibility_issues.py`` for the most up-to-date list of compatibility issues. In early versions of this library test breakages was often an indication that the library did not conform well enough to the standards, but as of today it mostly indicates that the servers does not support the standard well enough. It may be an option to add tweaks to the library code to cover some of the missing functionality. +The test suite is regularly run against several calendar servers, see https://github.com/python-caldav/caldav/issues/45 for the latest updates. See ``compatibility_hints.py`` for the most up-to-date list of compatibility issues. In early versions of this library test breakages was often an indication that the library did not conform well enough to the standards, but as of today it mostly indicates that the servers does not support the standard well enough. It may be an option to add tweaks to the library code to cover some of the missing functionality. Here are some known issues: diff --git a/tests/conf.py b/tests/conf.py index 8217ecfa..39015dc1 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -10,7 +10,7 @@ import niquests -from . import compatibility_issues +from caldav import compatibility_hints from caldav.davclient import DAVClient #################################### @@ -140,7 +140,7 @@ def teardown_radicale(self): "username": "user1", "password": "", "backwards_compatibility_url": url + "user1", - "incompatibilities": compatibility_issues.radicale, + "incompatibilities": compatibility_hints.radicale, "setup": setup_radicale, "teardown": teardown_radicale, } @@ -225,7 +225,7 @@ def silly_request(): "name": "LocalXandikos", "url": url, "backwards_compatibility_url": url + "sometestuser", - "incompatibilities": compatibility_issues.xandikos, + "incompatibilities": compatibility_hints.xandikos, "setup": setup_xandikos, "teardown": teardown_xandikos, } diff --git a/tests/conf_private.py.EXAMPLE b/tests/conf_private.py.EXAMPLE index 4253ad39..030af76a 100644 --- a/tests/conf_private.py.EXAMPLE +++ b/tests/conf_private.py.EXAMPLE @@ -1,4 +1,4 @@ -from tests import compatibility_issues +from caldav import compatibility_hints ## PRIVATE CALDAV SERVER(S) TO RUN TESTS TOWARDS ## Make a list of your own servers/accounts that you'd like to run the @@ -29,8 +29,8 @@ caldav_servers = [ ## incompatibilities is a list of flags that can be set for ## skipping (parts) of certain tests. See - ## tests/compatibility_issues.py for premade lists - #'incompatibilities': compatibility_issues.nextcloud + ## compatibility_hints.py for premade lists + #'incompatibilities': compatibility_hints.nextcloud 'incompatibilities': [], ## You may even add setup and teardown methods to set up diff --git a/tests/test_caldav.py b/tests/test_caldav.py index e7906e00..dc805e92 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -25,7 +25,7 @@ import pytest import vobject -from . import compatibility_issues +from caldav import compatibility_hints from .conf import caldav_servers from .conf import client from .conf import proxy @@ -581,12 +581,12 @@ class RepeatedFunctionalTestsBaseClass: def check_compatibility_flag(self, flag): ## yield an assertion error if checking for the wrong thig - assert flag in compatibility_issues.incompatibility_description + assert flag in compatibility_hints.incompatibility_description return flag in self.incompatibilities def skip_on_compatibility_flag(self, flag): if self.check_compatibility_flag(flag): - msg = compatibility_issues.incompatibility_description[flag] + msg = compatibility_hints.incompatibility_description[flag] pytest.skip("Test skipped due to server incompatibility issue: " + msg) def setup_method(self): @@ -596,7 +596,7 @@ def setup_method(self): self.calendars_used = [] for flag in self.server_params.get("incompatibilities", []): - assert flag in compatibility_issues.incompatibility_description + assert flag in compatibility_hints.incompatibility_description self.incompatibilities.add(flag) if self.check_compatibility_flag("unique_calendar_ids"): From e50afba650e1bb845d8109231651d4020f1641d9 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 17 May 2025 09:47:12 +0200 Subject: [PATCH 2/5] more bugfix --- caldav/compatibility_hints.py | 1 - caldav/davobject.py | 3 ++- tests/test_caldav.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/caldav/compatibility_hints.py b/caldav/compatibility_hints.py index f9f32c99..9b427b6c 100644 --- a/caldav/compatibility_hints.py +++ b/caldav/compatibility_hints.py @@ -17,7 +17,6 @@ some more advanced way, but I don't want to overcomplicate things - I will try out the key-value-approach first. """ - ## The lists below are specifying what tests should be skipped or ## modified to accept non-conforming resultsets from the different ## calendar servers. In addition there are some hacks in the library diff --git a/caldav/davobject.py b/caldav/davobject.py index a858a053..01cca546 100644 --- a/caldav/davobject.py +++ b/caldav/davobject.py @@ -263,7 +263,8 @@ def get_properties( * {proptag: value, ...} """ - from .collection import Principal ## late import to avoid cyclic dependencies + from .collection import Principal ## late import to avoid cyclic dependencies + rc = None response = self._query_properties(props, depth) if not parse_response_xml: diff --git a/tests/test_caldav.py b/tests/test_caldav.py index dc805e92..b7f55569 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -25,7 +25,6 @@ import pytest import vobject -from caldav import compatibility_hints from .conf import caldav_servers from .conf import client from .conf import proxy @@ -39,6 +38,7 @@ from .conf import xandikos_port from .proxy import NonThreadingHTTPServer from .proxy import ProxyHandler +from caldav import compatibility_hints from caldav.davclient import DAVClient from caldav.davclient import DAVResponse from caldav.elements import cdav From cb28611cb65c7cc8fc8bf98a2222a85d0d3816f1 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 17 May 2025 11:48:34 +0200 Subject: [PATCH 3/5] started moving/consolidating code from plann and the test code over to the caldav library --- CHANGELOG.md | 1 + caldav/davclient.py | 96 +++++++++++++++++++++++++++++++++++++++++++++ caldav/lib/debug.py | 5 ++- caldav/lib/error.py | 2 + tests/conf.py | 26 ++---------- 5 files changed, 107 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 931c336d..308ae834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Python 3.7 is no longer tested - but it should work. Please file a bug report i ### Added +* Work in progress: `auto_conn`, `auto_calendar` and `auto_calendars` may read caldav connection and calendar configuration from a config file, environmental variables or other sources. Currently I've made the minimal possible work to be able to test the caldav-server-tester script. * By now `calendar.search(..., sort_keys=("DTSTART")` will work. Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake. https://github.com/python-caldav/caldav/issues/448 https://github.com/python-caldav/caldav/pull/449 * Compatibility workaround: If `event.load()` fails, it will retry the load by doing a multiget - https://github.com/python-caldav/caldav/pull/475 - https://github.com/python-caldav/caldav/issues/459 * The `class_`-parameter now works when sending data to `save_event()` etc. diff --git a/caldav/davclient.py b/caldav/davclient.py index 22fa898a..8f2e05d7 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import logging +import os import sys from types import TracebackType from typing import Any @@ -44,6 +45,24 @@ else: from typing import Self +## TODO: this is also declared in davclient.DAVClient.__init__(...) +## TODO: it should be consolidated, duplication is a bad thing +## TODO: and it's almost certain that we'll forget to update this list +CONNKEYS = set( + ( + "url", + "proxy", + "username", + "password", + "timeout", + "headers", + "huge_tree", + "ssl_verify_cert", + "ssl_cert", + "auth", + ) +) + class DAVResponse: """ @@ -819,3 +838,80 @@ def request( commlog.write(b"\n") return response + + +def auto_calendars( + configfile: str = f"{os.environ.get('HOME')}/.config/calendar.conf", + testconfig=False, + environment: bool = True, + config_data: dict = None, + config_name: str = None, +) -> Iterable["Calendar"]: + """ + This will replace plann.lib.findcalendars() + """ + raise NotImplementedError("auto_calendars not implemented yet") + + +def auto_calendar(*largs, **kwargs) -> Iterable["Calendar"]: + """ + Alternative to auto_calendars - in most use cases, one calendar suffices + """ + return next(auto_calendars(*largs, **kwargs), None) + + +def auto_conn( + configfile: str = f"{os.environ.get('HOME')}/.config/calendar.conf", + testconfig=False, + environment: bool = True, + config_data: dict = None, + name: str = None, +) -> "DAVClient": + """ + Normally you would like to look into auto_calendars or + auto_calendar instead. However, in some cases it's needed + with a DAVClient object rather than a Calendar object. + + This function will yield a DAVClient object. It will not try to + connect (see auto_calendars for that). It will read configuration + from various sources, dependent on the parameters given, in this + order: + + * Data from the given dict + * Environment variables prepended with "CALDAV_" + * Data from `./tests/conf.py` or `./conf.py` (this includes the possibility to spin up a test server) + * Configuration file. Documented in the plann project as for now. (TODO - move it) + + """ + if config_data: + return DAVClient(**config_data) + + if testconfig: + sys.path.insert(0, "tests") + sys.path.insert(1, ".") + ## TODO: move the code from client into here + try: + from conf import client + + try: + idx = int(name) + except ValueError: + idx = None + try: + conn = client(idx, name, **config_data) + if conn: + return conn + except: + error.weirdness("traceback from client()") + except ImportError: + pass + + if environment: + raise NotImplementedError( + "Not possible to configure the caldav server through environmental variables yet" + ) + + if configfile: + raise NotImplementedError( + "Support for configuration file not made yet (TODO: copy the code from the plann tool)" + ) diff --git a/caldav/lib/debug.py b/caldav/lib/debug.py index cdb4951e..7c233bbc 100644 --- a/caldav/lib/debug.py +++ b/caldav/lib/debug.py @@ -6,7 +6,10 @@ def xmlstring(root): return root if hasattr(root, "xmlelement"): root = root.xmlelement() - return etree.tostring(root, pretty_print=True).decode("utf-8") + try: + return etree.tostring(root, pretty_print=True).decode("utf-8") + except: + return root def printxml(root) -> None: diff --git a/caldav/lib/error.py b/caldav/lib/error.py index aac83a71..0fb1d078 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -10,6 +10,8 @@ try: import os + ## Environmental variables prepended with "PYTHON_CALDAV" are used for debug purposes, + ## environmental variables prepended with "CALDAV_" are for connection parameters debug_dump_communication = os.environ.get("PYTHON_CALDAV_COMMDUMP", False) ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"] diff --git a/tests/conf.py b/tests/conf.py index 39015dc1..bcfba3ae 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -11,6 +11,7 @@ import niquests from caldav import compatibility_hints +from caldav.davclient import CONNKEYS from caldav.davclient import DAVClient #################################### @@ -231,29 +232,10 @@ def silly_request(): } ) + ################################################################### # Convenience - get a DAVClient object from the caldav_servers list ################################################################### -## TODO: this is already declared in davclient.DAVClient.__init__(...) -## TODO: is it possible to reuse the declaration here instead of -## duplicating the list? -## TODO: If not, it's needed to look through and ensure the list is uptodate -CONNKEYS = set( - ( - "url", - "proxy", - "username", - "password", - "timeout", - "headers", - "huge_tree", - "ssl_verify_cert", - "ssl_cert", - "auth", - ) -) - - def client( idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs ): @@ -266,8 +248,8 @@ def client( return client(**caldav_servers[idx]) elif name is not None and no_args and caldav_servers: for s in caldav_servers: - if caldav_servers["name"] == s: - return s + if s["name"] == name: + return client(**s) return None elif no_args: return None From 43f7c48e5295d346c3acf4818dc046a4e979a95e Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 17 May 2025 12:39:57 +0200 Subject: [PATCH 4/5] piggybacking in a fix for https://github.com/python-caldav/caldav/pull/465#issuecomment-2888247449 --- caldav/davclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caldav/davclient.py b/caldav/davclient.py index 8f2e05d7..b20a81a7 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -96,7 +96,7 @@ def __init__( xml = ["text/xml", "application/xml"] no_xml = ["text/plain", "text/calendar"] expect_xml = any((content_type.startswith(x) for x in xml)) - expect_no_xml = any((content_type.startswith(x) for x in no_xml)) + expect_no_xml = any((content_type.startswith(x) for x in no_xml)) or response.status_code>399 try: content_length = int(self.headers["Content-Length"]) except: From a36a662b14a40ad8086fa692c103c19d6e9a33a1 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 17 May 2025 12:58:31 +0200 Subject: [PATCH 5/5] bugfix --- caldav/davclient.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/caldav/davclient.py b/caldav/davclient.py index b20a81a7..0f5d60b2 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -94,9 +94,11 @@ def __init__( content_type = self.headers.get("Content-Type", "") xml = ["text/xml", "application/xml"] - no_xml = ["text/plain", "text/calendar"] + no_xml = ["text/plain", "text/calendar", "application/octet-stream"] expect_xml = any((content_type.startswith(x) for x in xml)) - expect_no_xml = any((content_type.startswith(x) for x in no_xml)) or response.status_code>399 + expect_no_xml = any((content_type.startswith(x) for x in no_xml)) + if content_type and not expect_xml and not expect_no_xml: + error.weirdness(f"Unexpected content type: {content_type}") try: content_length = int(self.headers["Content-Length"]) except: @@ -905,6 +907,8 @@ def auto_conn( error.weirdness("traceback from client()") except ImportError: pass + finally: + sys.path = sys.path[2:] if environment: raise NotImplementedError(