Refactor compatibility issues

CHANGELOG.md

@@ -21,10 +21,12 @@ Python 3.7 is no longer tested - but it should work.  Please file a bug report i
 ### Changed
 
 * Some exotic servers may return object URLs on search, but it does not work out to fetch the calendar data.  Now it will log an error instead of raising an error in such cases.
+* The `tests/compatibility_issues.py` has been moved to `caldav/compatibility_hints.py`, this to make it available for a caldav-server-tester-tool that I'm splitting off to a separate project/repository, and also to make https://github.com/python-caldav/caldav/issues/402 possible.
 
 #### Refactoring
 
 * Minor code cleanups by github user @ArtemIsmagilov in https://github.com/python-caldav/caldav/pull/456
+* The very much overgrown `objects.py`-file has been split into three.
 
 #### Test framework
 
@@ -40,6 +42,7 @@ Python 3.7 is no longer tested - but it should work.  Please file a bug report i
 
 ### Added
 
+* Work in progress: `auto_conn`, `auto_calendar` and `auto_calendars` may read caldav connection and calendar configuration from a config file, environmental variables or other sources.  Currently I've made the minimal possible work to be able to test the caldav-server-tester script.
 * By now `calendar.search(..., sort_keys=("DTSTART")` will work.  Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake.  https://github.com/python-caldav/caldav/issues/448 https://github.com/python-caldav/caldav/pull/449
 * Compatibility workaround: If `event.load()` fails, it will retry the load by doing a multiget - https://github.com/python-caldav/caldav/pull/475 - https://github.com/python-caldav/caldav/issues/459
 * The `class_`-parameter now works when sending data to `save_event()` etc.

tests/compatibility_issues.py → caldav/compatibility_hints.py

@@ -1,4 +1,22 @@
 # fmt: off
+"""This text was updated 2025-05-17.  The plan is to reorganize this
+file a lot over the next few months, see
+https://github.com/python-caldav/caldav/issues/402
+
+This file serves as a database of different compatibility issues we've
+encountered while working on the caldav library, and descriptions on
+how the well-known servers behave.
+
+As for now, this is a list of binary "flags" that could be turned on
+or off.  My experience is that there are often neuances, so the
+compatibility matrix will be changed from being a list of flags to a
+key=value store in the near future (at least, that's the plan).
+
+The issues may be grouped together, maybe even organized
+hierarchically.  I did consider organizing the compatibility issues in
+some more advanced way, but I don't want to overcomplicate things - I
+will try out the key-value-approach first.
+"""
 ## The lists below are specifying what tests should be skipped or
 ## modified to accept non-conforming resultsets from the different
 ## calendar servers.  In addition there are some hacks in the library

caldav/davclient.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import logging
+import os
 import sys
 from types import TracebackType
 from typing import Any
@@ -44,6 +45,24 @@
 else:
     from typing import Self
 
+## TODO: this is also declared in davclient.DAVClient.__init__(...)
+## TODO: it should be consolidated, duplication is a bad thing
+## TODO: and it's almost certain that we'll forget to update this list
+CONNKEYS = set(
+    (
+        "url",
+        "proxy",
+        "username",
+        "password",
+        "timeout",
+        "headers",
+        "huge_tree",
+        "ssl_verify_cert",
+        "ssl_cert",
+        "auth",
+    )
+)
+
 
 class DAVResponse:
     """
@@ -75,9 +94,11 @@ def __init__(
 
         content_type = self.headers.get("Content-Type", "")
         xml = ["text/xml", "application/xml"]
-        no_xml = ["text/plain", "text/calendar"]
+        no_xml = ["text/plain", "text/calendar", "application/octet-stream"]
         expect_xml = any((content_type.startswith(x) for x in xml))
         expect_no_xml = any((content_type.startswith(x) for x in no_xml))
+        if content_type and not expect_xml and not expect_no_xml:
+            error.weirdness(f"Unexpected content type: {content_type}")
         try:
             content_length = int(self.headers["Content-Length"])
         except:
@@ -819,3 +840,82 @@ def request(
                 commlog.write(b"\n")
 
         return response
+
+
+def auto_calendars(
+    configfile: str = f"{os.environ.get('HOME')}/.config/calendar.conf",
+    testconfig=False,
+    environment: bool = True,
+    config_data: dict = None,
+    config_name: str = None,
+) -> Iterable["Calendar"]:
+    """
+    This will replace plann.lib.findcalendars()
+    """
+    raise NotImplementedError("auto_calendars not implemented yet")
+
+
+def auto_calendar(*largs, **kwargs) -> Iterable["Calendar"]:
+    """
+    Alternative to auto_calendars - in most use cases, one calendar suffices
+    """
+    return next(auto_calendars(*largs, **kwargs), None)
+
+
+def auto_conn(
+    configfile: str = f"{os.environ.get('HOME')}/.config/calendar.conf",
+    testconfig=False,
+    environment: bool = True,
+    config_data: dict = None,
+    name: str = None,
+) -> "DAVClient":
+    """
+    Normally you would like to look into auto_calendars or
+    auto_calendar instead.  However, in some cases it's needed
+    with a DAVClient object rather than a Calendar object.
+
+    This function will yield a DAVClient object.  It will not try to
+    connect (see auto_calendars for that).  It will read configuration
+    from various sources, dependent on the parameters given, in this
+    order:
+
+    * Data from the given dict
+    * Environment variables prepended with "CALDAV_"
+    * Data from `./tests/conf.py` or `./conf.py` (this includes the possibility to spin up a test server)
+    * Configuration file.  Documented in the plann project as for now.  (TODO - move it)
+
+    """
+    if config_data:
+        return DAVClient(**config_data)
+
+    if testconfig:
+        sys.path.insert(0, "tests")
+        sys.path.insert(1, ".")
+        ## TODO: move the code from client into here
+        try:
+            from conf import client
+
+            try:
+                idx = int(name)
+            except ValueError:
+                idx = None
+            try:
+                conn = client(idx, name, **config_data)
+                if conn:
+                    return conn
+            except:
+                error.weirdness("traceback from client()")
+        except ImportError:
+            pass
+        finally:
+            sys.path = sys.path[2:]
+
+    if environment:
+        raise NotImplementedError(
+            "Not possible to configure the caldav server through environmental variables yet"
+        )
+
+    if configfile:
+        raise NotImplementedError(
+            "Support for configuration file not made yet (TODO: copy the code from the plann tool)"
+        )

caldav/davobject.py

@@ -263,7 +263,8 @@ def get_properties(
          * {proptag: value, ...}
 
         """
-        from .collection import Principal ## late import to avoid cyclic dependencies
+        from .collection import Principal  ## late import to avoid cyclic dependencies
+
         rc = None
         response = self._query_properties(props, depth)
         if not parse_response_xml:

caldav/lib/debug.py

@@ -6,7 +6,10 @@ def xmlstring(root):
         return root
     if hasattr(root, "xmlelement"):
         root = root.xmlelement()
-    return etree.tostring(root, pretty_print=True).decode("utf-8")
+    try:
+        return etree.tostring(root, pretty_print=True).decode("utf-8")
+    except:
+        return root
 
 
 def printxml(root) -> None:

caldav/lib/error.py

@@ -10,6 +10,8 @@
 try:
     import os
 
+    ## Environmental variables prepended with "PYTHON_CALDAV" are used for debug purposes,
+    ## environmental variables prepended with "CALDAV_" are for connection parameters
     debug_dump_communication = os.environ.get("PYTHON_CALDAV_COMMDUMP", False)
     ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION
     debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"]

docs/source/index.rst

@@ -307,7 +307,7 @@ Compatibility
 tend to be a moving target, and I rarely recheck if things works in
 newer versions of the software after I find an incompatibility)
 
-The test suite is regularly run against several calendar servers, see https://github.com/python-caldav/caldav/issues/45 for the latest updates.  See ``tests/compatibility_issues.py`` for the most up-to-date list of compatibility issues.  In early versions of this library test breakages was often an indication that the library did not conform well enough to the standards, but as of today it mostly indicates that the servers does not support the standard well enough.  It may be an option to add tweaks to the library code to cover some of the missing functionality.
+The test suite is regularly run against several calendar servers, see https://github.com/python-caldav/caldav/issues/45 for the latest updates.  See ``compatibility_hints.py`` for the most up-to-date list of compatibility issues.  In early versions of this library test breakages was often an indication that the library did not conform well enough to the standards, but as of today it mostly indicates that the servers does not support the standard well enough.  It may be an option to add tweaks to the library code to cover some of the missing functionality.
 
 Here are some known issues:
 

tests/conf.py

@@ -10,7 +10,8 @@
 
 import niquests
 
-from . import compatibility_issues
+from caldav import compatibility_hints
+from caldav.davclient import CONNKEYS
 from caldav.davclient import DAVClient
 
 ####################################
@@ -140,7 +141,7 @@ def teardown_radicale(self):
             "username": "user1",
             "password": "",
             "backwards_compatibility_url": url + "user1",
-            "incompatibilities": compatibility_issues.radicale,
+            "incompatibilities": compatibility_hints.radicale,
             "setup": setup_radicale,
             "teardown": teardown_radicale,
         }
@@ -225,35 +226,16 @@ def silly_request():
             "name": "LocalXandikos",
             "url": url,
             "backwards_compatibility_url": url + "sometestuser",
-            "incompatibilities": compatibility_issues.xandikos,
+            "incompatibilities": compatibility_hints.xandikos,
             "setup": setup_xandikos,
             "teardown": teardown_xandikos,
         }
     )
 
+
 ###################################################################
 # Convenience - get a DAVClient object from the caldav_servers list
 ###################################################################
-## TODO: this is already declared in davclient.DAVClient.__init__(...)
-## TODO: is it possible to reuse the declaration here instead of
-## duplicating the list?
-## TODO: If not, it's needed to look through and ensure the list is uptodate
-CONNKEYS = set(
-    (
-        "url",
-        "proxy",
-        "username",
-        "password",
-        "timeout",
-        "headers",
-        "huge_tree",
-        "ssl_verify_cert",
-        "ssl_cert",
-        "auth",
-    )
-)
-
-
 def client(
     idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs
 ):
@@ -266,8 +248,8 @@ def client(
         return client(**caldav_servers[idx])
     elif name is not None and no_args and caldav_servers:
         for s in caldav_servers:
-            if caldav_servers["name"] == s:
-                return s
+            if s["name"] == name:
+                return client(**s)
         return None
     elif no_args:
         return None

tests/conf_private.py.EXAMPLE

@@ -1,4 +1,4 @@
-from tests import compatibility_issues
+from caldav import compatibility_hints
 
 ## PRIVATE CALDAV SERVER(S) TO RUN TESTS TOWARDS
 ## Make a list of your own servers/accounts that you'd like to run the
@@ -29,8 +29,8 @@ caldav_servers = [
 
         ## incompatibilities is a list of flags that can be set for
         ## skipping (parts) of certain tests.  See
-        ## tests/compatibility_issues.py for premade lists
-        #'incompatibilities': compatibility_issues.nextcloud
+        ## compatibility_hints.py for premade lists
+        #'incompatibilities': compatibility_hints.nextcloud
       	'incompatibilities': [],
 
         ## You may even add setup and teardown methods to set up

tests/test_caldav.py

@@ -25,7 +25,6 @@
 import pytest
 import vobject
 
-from . import compatibility_issues
 from .conf import caldav_servers
 from .conf import client
 from .conf import proxy
@@ -39,6 +38,7 @@
 from .conf import xandikos_port
 from .proxy import NonThreadingHTTPServer
 from .proxy import ProxyHandler
+from caldav import compatibility_hints
 from caldav.davclient import DAVClient
 from caldav.davclient import DAVResponse
 from caldav.elements import cdav
@@ -581,12 +581,12 @@ class RepeatedFunctionalTestsBaseClass:
 
     def check_compatibility_flag(self, flag):
         ## yield an assertion error if checking for the wrong thig
-        assert flag in compatibility_issues.incompatibility_description
+        assert flag in compatibility_hints.incompatibility_description
         return flag in self.incompatibilities
 
     def skip_on_compatibility_flag(self, flag):
         if self.check_compatibility_flag(flag):
-            msg = compatibility_issues.incompatibility_description[flag]
+            msg = compatibility_hints.incompatibility_description[flag]
             pytest.skip("Test skipped due to server incompatibility issue: " + msg)
 
     def setup_method(self):
@@ -596,7 +596,7 @@ def setup_method(self):
         self.calendars_used = []
 
         for flag in self.server_params.get("incompatibilities", []):
-            assert flag in compatibility_issues.incompatibility_description
+            assert flag in compatibility_hints.incompatibility_description
             self.incompatibilities.add(flag)
 
         if self.check_compatibility_flag("unique_calendar_ids"):