[17.0][FW] fs_storage: forward port v16 changes

fs_storage/__init__.py

@@ -4,3 +4,4 @@
 
 # then add normal imports
 from . import models
+from . import wizards

fs_storage/__manifest__.py

@@ -16,6 +16,7 @@
     "data": [
         "views/fs_storage_view.xml",
         "security/ir.model.access.csv",
+        "wizards/fs_test_connection.xml",
     ],
     "demo": ["demo/fs_storage_demo.xml"],
     "external_dependencies": {"python": ["fsspec>=2024.5.0"]},

fs_storage/models/fs_storage.py

@@ -146,6 +146,15 @@ def __init__(self, env, ids=(), prefetch_ids=()):
         compute="_compute_options_properties",
         store=False,
     )
+    check_connection_method = fields.Selection(
+        selection="_get_check_connection_method_selection",
+        default="marker_file",
+        help="Set a method if you want the connection to remote to be checked every "
+        "time the storage is used, in order to remove the obsolete connection from"
+        " the cache.\n"
+        "* Create Marker file : Create a file on remote and check it exists\n"
+        "* List File : List all files from root directory",
+    )
 
     _sql_constraints = [
         (
@@ -157,6 +166,13 @@ def __init__(self, env, ids=(), prefetch_ids=()):
 
     _server_env_section_name_field = "code"
 
+    @api.model
+    def _get_check_connection_method_selection(self):
+        return [
+            ("marker_file", _("Create Marker file")),
+            ("ls", _("List File")),
+        ]
+
     @property
     def _server_env_fields(self):
         return {"protocol": {}, "options": {}, "directory_path": {}}
@@ -264,12 +280,41 @@ def _compute_options_properties(self) -> None:
             doc = inspect.getdoc(cls.__init__)
             rec.options_properties = f"__init__{signature}\n{doc}"
 
+    def _get_marker_file_name(self):
+        return ".odoo_fs_storage_%s.marker" % self.id
+
+    def _marker_file_check_connection(self, fs):
+        marker_file_name = self._get_marker_file_name()
+        try:
+            fs.info(marker_file_name)
+        except FileNotFoundError:
+            fs.touch(marker_file_name)
+
+    def _ls_check_connection(self, fs):
+        fs.ls("", detail=False)
+
+    def _check_connection(self, fs, check_connection_method):
+        if check_connection_method == "marker_file":
+            self._marker_file_check_connection(fs)
+        elif check_connection_method == "ls":
+            self._ls_check_connection(fs)
+        return True
+
     @property
     def fs(self) -> fsspec.AbstractFileSystem:
         """Get the fsspec filesystem for this backend."""
         self.ensure_one()
         if not self.__fs:
-            self.__fs = self._get_filesystem()
+            self.__fs = self.sudo()._get_filesystem()
+        if not tools.config["test_enable"]:
+            # Check whether we need to invalidate FS cache or not.
+            # Use a marker file to limit the scope of the LS command for performance.
+            try:
+                self._check_connection(self.__fs, self.check_connection_method)
+            except Exception as e:
+                self.__fs.clear_instance_cache()
+                self.__fs = None
+                raise e
         return self.__fs
 
     def _get_filesystem_storage_path(self) -> str:
@@ -406,7 +451,8 @@ def find_files(self, pattern, relative_path="", **kw) -> list[str]:
             return []
         regex = re.compile(pattern)
         for file_path in self.fs.ls(relative_path, detail=False):
-            if regex.match(file_path):
+            # fs.ls returns a relative path
+            if regex.match(os.path.basename(file_path)):
                 result.append(file_path)
         return result
 
@@ -429,9 +475,20 @@ def move_files(self, files, destination_path, **kw) -> None:
     def delete(self, relative_path) -> None:
         self.fs.rm_file(relative_path)
 
-    def action_test_config(self) -> None:
+    def action_test_config(self):
+        self.ensure_one()
+        if self.check_connection_method:
+            return self._test_config(self.check_connection_method)
+        else:
+            action = self.env["ir.actions.actions"]._for_xml_id(
+                "fs_storage.act_open_fs_test_connection_view"
+            )
+            action["context"] = {"active_model": "fs.storage", "active_id": self.id}
+            return action
+
+    def _test_config(self, connection_method):
         try:
-            self.fs.ls("", detail=False)
+            self._check_connection(self.fs, connection_method)
             title = _("Connection Test Succeeded!")
             message = _("Everything seems properly set up!")
             msg_type = "success"

fs_storage/readme/USAGE.md

@@ -18,6 +18,16 @@ When you create a new backend, you must specify the following:
   documentation.
 - Resolve env vars. This options resolves the protocol options values
   starting with \$ from environment variables
+- Check Connection Method. If set, Odoo will always check the connection before
+  using a storage and it will remove the fs connection from the cache if the
+  check fails.
+
+  - `Create Marker file`: create a hidden file on remote and then check it
+    exists with Use it if you have write access to the remote and if it is not
+    an issue to leave the marker file in the root directory.
+  - `List file`: list all files from the root directory. You can use it if the
+    directory path does not contain a big list of files (for performance
+    reasons)
 
 Some protocols defined in the fsspec package are wrappers around other
 protocols. For example, the SimpleCacheFileSystem protocol is a wrapper

fs_storage/readme/newsfragments/320.feature

@@ -0,0 +1 @@
+Invalidate FS filesystem object cache when the connection fails, forcing a reconnection.

fs_storage/security/ir.model.access.csv

@@ -1,2 +1,3 @@
 id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
 access_fs_storage_edit,fs_storage edit,model_fs_storage,base.group_system,1,1,1,1
+access_fs_test_connection,fs.test.connection.access,model_fs_test_connection,base.group_system,1,1,1,1

fs_storage/views/fs_storage_view.xml

@@ -42,6 +42,7 @@
                                 options="{'mode': 'python'}"
                                 placeholder="Enter you fsspec options here."
                             />
+                            <field name="check_connection_method" />
                         </group>
                         <group>
                             <notebook colspan="2">

fs_storage/wizards/__init__.py

@@ -0,0 +1 @@
+from . import fs_test_connection

fs_storage/wizards/fs_test_connection.py

@@ -0,0 +1,26 @@
+# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
+
+from odoo import api, fields, models
+
+
+class FSTestConnection(models.TransientModel):
+    _name = "fs.test.connection"
+    _description = "FS Test Connection Wizard"
+
+    def _get_check_connection_method_selection(self):
+        return self.env["fs.storage"]._get_check_connection_method_selection()
+
+    storage_id = fields.Many2one("fs.storage")
+    check_connection_method = fields.Selection(
+        selection="_get_check_connection_method_selection",
+        required=True,
+    )
+
+    @api.model
+    def default_get(self, field_list):
+        res = super().default_get(field_list)
+        res["storage_id"] = self.env.context.get("active_id", False)
+        return res
+
+    def action_test_config(self):
+        return self.storage_id._test_config(self.check_connection_method)

fs_storage/wizards/fs_test_connection.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+    <record id="fs_test_connection_form_view" model="ir.ui.view">
+        <field name="name">fs.test.connection.form</field>
+        <field name="model">fs.test.connection</field>
+        <field name="arch" type="xml">
+            <form string="Test Connection">
+                <group>
+                    <field name="storage_id" readonly="1" />
+                    <field name="check_connection_method" />
+                </group>
+                <footer>
+                    <button
+                        type="object"
+                        name="action_test_config"
+                        string="Test connection"
+                    />
+                    <button string="Close" class="btn-secondary" special="cancel" />
+
+                </footer>
+            </form>
+        </field>
+    </record>
+    <record model="ir.actions.act_window" id="act_open_fs_test_connection_view">
+        <field name="name">FS Test Connection</field>
+        <field name="type">ir.actions.act_window</field>
+        <field name="res_model">fs.test.connection</field>
+        <field name="view_mode">form</field>
+        <field name="target">new</field>
+    </record>
+</odoo>