diff --git a/requirements.txt b/requirements.txt index 48e0ddf7be..19fa2e6dac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ paramiko +pyftpdlib python-slugify>=3.0.2 vcrpy>=2.1.1 cerberus diff --git a/setup/storage_backend_ftp/odoo/addons/storage_backend_ftp b/setup/storage_backend_ftp/odoo/addons/storage_backend_ftp new file mode 120000 index 0000000000..18973453c2 --- /dev/null +++ b/setup/storage_backend_ftp/odoo/addons/storage_backend_ftp @@ -0,0 +1 @@ +../../../../storage_backend_ftp \ No newline at end of file diff --git a/setup/storage_backend_ftp/setup.py b/setup/storage_backend_ftp/setup.py new file mode 100644 index 0000000000..28c57bb640 --- /dev/null +++ b/setup/storage_backend_ftp/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) diff --git a/storage_backend_ftp/README.rst b/storage_backend_ftp/README.rst new file mode 100644 index 0000000000..5ff3bb30bc --- /dev/null +++ b/storage_backend_ftp/README.rst @@ -0,0 +1,73 @@ +=================== +Storage Backend FTP +=================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fstorage-lightgray.png?logo=github + :target: https://github.com/OCA/storage/tree/13.0/storage_backend_ftp + :alt: OCA/storage +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/storage-13-0/storage-13-0-storage_backend_ftp + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/275/13.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Add FTP as storage backend + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Acsone SA/NV + +Contributors +~~~~~~~~~~~~ + +* François Honoré + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/storage `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/storage_backend_ftp/__init__.py b/storage_backend_ftp/__init__.py new file mode 100644 index 0000000000..0f00a6730d --- /dev/null +++ b/storage_backend_ftp/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import components diff --git a/storage_backend_ftp/__manifest__.py b/storage_backend_ftp/__manifest__.py new file mode 100644 index 0000000000..48108c4442 --- /dev/null +++ b/storage_backend_ftp/__manifest__.py @@ -0,0 +1,14 @@ +# Copyright 2021 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +{ + "name": "Storage Backend FTP", + "summary": "Implement FTP Storage", + "version": "13.0.1.0.0", + "category": "Storage", + "website": "https://github.com/OCA/storage", + "author": " Acsone SA/NV,Odoo Community Association (OCA)", + "license": "LGPL-3", + "external_dependencies": {"python": ["pyftpdlib"]}, + "depends": ["storage_backend"], + "data": ["views/backend_storage_view.xml"], +} diff --git a/storage_backend_ftp/components/__init__.py b/storage_backend_ftp/components/__init__.py new file mode 100644 index 0000000000..72dbc3a308 --- /dev/null +++ b/storage_backend_ftp/components/__init__.py @@ -0,0 +1 @@ +from . import ftp_adapter diff --git a/storage_backend_ftp/components/ftp_adapter.py b/storage_backend_ftp/components/ftp_adapter.py new file mode 100644 index 0000000000..2c3f7bab05 --- /dev/null +++ b/storage_backend_ftp/components/ftp_adapter.py @@ -0,0 +1,148 @@ +# Copyright 2021 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import errno +import io +import logging +import os +import ssl +from contextlib import contextmanager +from io import BytesIO + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + +try: + import ftplib +except ImportError as err: # pragma: no cover + _logger.debug(err) + + +def ftp_mkdirs(client, path): + try: + client.mkd(path) + except IOError as e: + if e.errno == errno.ENOENT and path: + ftp_mkdirs(client, os.path.dirname(path)) + client.mkd(path) + else: + raise # pragma: no cover + + +@contextmanager +def ftp(backend): + params = {} + security = None + if backend.ftp_encryption == "ftp": + ftp = ftplib.FTP + elif backend.ftp_encryption == "tls": + ftp = ftplib.FTP_TLS + # Due to a bug into between ftplib and ssl, this part (about ssl) might not work! + # https://bugs.python.org/issue31727 + security = None + if backend.ftp_security == "tls": + security = ssl.PROTOCOL_TLS + elif backend.ftp_security == "tlsv1": + security = ssl.PROTOCOL_TLSv1 + elif backend.ftp_security == "tlsv1_1": + security = ssl.PROTOCOL_TLSv1_1 + elif backend.ftp_security == "tlsv1_2": + security = ssl.PROTOCOL_TLSv1_2 + elif backend.ftp_security == "sslv2": + security = ssl.PROTOCOL_SSLv2 + elif backend.ftp_security == "sslv23": + security = ssl.PROTOCOL_SSLv23 + elif backend.ftp_security == "sslv3": + security = ssl.PROTOCOL_SSLv3 + if security: + ctx = ssl._create_stdlib_context(security) + params.update({"context": ctx}) + else: + raise NotImplementedError() + with ftp(**params) as client: + client.connect(host=backend.ftp_server, port=backend.ftp_port) + if security: + client.auth() + client.login(backend.ftp_login, backend.ftp_password) + if security: + client.ssl_version = security + client.prot_p() + if backend.ftp_passive: + client.set_pasv(True) + yield client + + +class FTPStorageBackendAdapter(Component): + _name = "ftp.adapter" + _inherit = "base.storage.adapter" + _usage = "ftp" + + def add(self, relative_path, data, **kwargs): + with ftp(self.collection) as client: + full_path = self._fullpath(relative_path) + dirname = os.path.dirname(full_path) + if dirname: + try: + client.cwd(dirname) + except IOError as e: + if e.errno == errno.ENOENT: + ftp_mkdirs(client, dirname) + else: + raise # pragma: no cover + with io.BytesIO(data) as tmp_file: + try: + client.storbinary("STOR " + full_path, tmp_file) + except ftplib.Error as e: + raise ValueError(repr(e)) + except OSError as e: + raise ValueError(repr(e)) + + def get(self, relative_path, **kwargs): + full_path = self._fullpath(relative_path) + with ftp(self.collection) as client, BytesIO() as buff: + try: + client.retrbinary("RETR " + full_path, buff.write) + data = buff.getvalue() + except ftplib.Error as e: + raise FileNotFoundError(repr(e)) + return data + + def list(self, relative_path): + full_path = self._fullpath(relative_path) + with ftp(self.collection) as client: + try: + return client.nlst(full_path) + except IOError as e: + if e.errno == errno.ENOENT: + # The path do not exist return an empty list + return [] + else: + raise # pragma: no cover + + def move_files(self, files, destination_path): + _logger.debug("mv %s %s", files, destination_path) + with ftp(self.collection) as client: + for ftp_file in files: + dest_file_path = os.path.join( + destination_path, os.path.basename(ftp_file) + ) + # Remove existing file at the destination path (an error is raised + # otherwise) + result = [] + try: + result = client.nlst(dest_file_path) + except ftplib.Error: + _logger.debug("destination %s is free", dest_file_path) + if result: + client.delete(dest_file_path) + # Move the file + client.rename(ftp_file, dest_file_path) + + def delete(self, relative_path): + full_path = self._fullpath(relative_path) + with ftp(self.collection) as client: + return client.delete(full_path) + + def validate_config(self): + with ftp(self.collection) as client: + client.getwelcome() diff --git a/storage_backend_ftp/models/__init__.py b/storage_backend_ftp/models/__init__.py new file mode 100644 index 0000000000..f45f402268 --- /dev/null +++ b/storage_backend_ftp/models/__init__.py @@ -0,0 +1 @@ +from . import storage_backend diff --git a/storage_backend_ftp/models/storage_backend.py b/storage_backend_ftp/models/storage_backend.py new file mode 100644 index 0000000000..9cb939ea5d --- /dev/null +++ b/storage_backend_ftp/models/storage_backend.py @@ -0,0 +1,49 @@ +# Copyright 2021 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from odoo import fields, models + + +class StorageBackend(models.Model): + _inherit = "storage.backend" + + backend_type = fields.Selection(selection_add=[("ftp", "FTP")]) + ftp_server = fields.Char(string="FTP Host") + ftp_port = fields.Integer(string="FTP Port", default=21) + ftp_encryption = fields.Selection( + string="FTP Encryption method", + selection=[("ftp", "FTP"), ("tls", "FTP over TLS")], + default="ftp", + required=True, + ) + ftp_security = fields.Selection( + string="FTP security option", + selection=[ + ("none", "None"), + ("tlsv1", "TLS"), + ("tlsv1_1", "TLSv1_1"), + ("tlsv1_2", "TLSv1_2"), + ("sslv2", "SSLv2"), + ("sslv23", "SSLv23"), + ("sslv3", "SSLv3"), + ], + required=True, + ) + ftp_login = fields.Char(string="FTP Login", help="Login to connect to ftp server") + ftp_password = fields.Char(string="FTP Password") + ftp_passive = fields.Boolean(string="FTP Passive", default=False) + + @property + def _server_env_fields(self): + env_fields = super()._server_env_fields + env_fields.update( + { + "ftp_password": {}, + "ftp_login": {}, + "ftp_server": {}, + "ftp_port": {}, + "ftp_encryption": {}, + "ftp_security": {}, + "ftp_passive": {}, + } + ) + return env_fields diff --git a/storage_backend_ftp/readme/CONTRIBUTORS.rst b/storage_backend_ftp/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000000..51b8da248c --- /dev/null +++ b/storage_backend_ftp/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* François Honoré +* Lois Rilo diff --git a/storage_backend_ftp/readme/DESCRIPTION.rst b/storage_backend_ftp/readme/DESCRIPTION.rst new file mode 100644 index 0000000000..b0b32036fc --- /dev/null +++ b/storage_backend_ftp/readme/DESCRIPTION.rst @@ -0,0 +1 @@ +Add FTP as storage backend diff --git a/storage_backend_ftp/static/description/index.html b/storage_backend_ftp/static/description/index.html new file mode 100644 index 0000000000..a3df4e58dc --- /dev/null +++ b/storage_backend_ftp/static/description/index.html @@ -0,0 +1,419 @@ + + + + + + +Storage Backend FTP + + + +
+

Storage Backend FTP

+ + +

Beta License: LGPL-3 OCA/storage Translate me on Weblate Try me on Runbot

+

Add FTP as storage backend

+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Acsone SA/NV
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/storage project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/storage_backend_ftp/tests/__init__.py b/storage_backend_ftp/tests/__init__.py new file mode 100644 index 0000000000..9ab6aa0f49 --- /dev/null +++ b/storage_backend_ftp/tests/__init__.py @@ -0,0 +1 @@ +from . import test_ftp diff --git a/storage_backend_ftp/tests/test_ftp.py b/storage_backend_ftp/tests/test_ftp.py new file mode 100644 index 0000000000..c3bd4a48d7 --- /dev/null +++ b/storage_backend_ftp/tests/test_ftp.py @@ -0,0 +1,159 @@ +# Copyright 2021 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +# pylint: disable=missing-manifest-dependency +# disable warning on 'vcr' missing in manifest: this is only a dependency for +# dev/tests + +import errno +import ftplib +import logging +import os + +import mock + +from odoo.addons.storage_backend.tests.common import BackendStorageTestMixin, CommonCase + +_logger = logging.getLogger(__name__) + +MOD_PATH = "odoo.addons.storage_backend_ftp.components.ftp_adapter" +ADAPTER_PATH = MOD_PATH + ".FTPStorageBackendAdapter" +FTP_LIB_PATH = MOD_PATH + ".ftplib" + + +class FtpCase(CommonCase, BackendStorageTestMixin): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend.write( + { + "backend_type": "ftp", + "ftp_login": os.environ.get("FTP_LOGIN", "foo"), + "ftp_password": os.environ.get("FTP_PWD", "pass"), + "ftp_server": os.environ.get("FTP_HOST", "localhost"), + "ftp_port": os.environ.get("FTP_PORT", "21"), + "directory_path": "upload", + "ftp_encryption": "ftp", + } + ) + cls.case_with_subdirectory = "upload/subdirectory/here" + + @mock.patch(MOD_PATH + ".ftp_mkdirs") + @mock.patch(FTP_LIB_PATH) + def test_add(self, mocked_ftplib, mocked_mkdirs): + client = mocked_ftplib.FTP().__enter__() + # simulate errors + exc = IOError() + # general + client.cwd.side_effect = exc + with self.assertRaises(IOError): + self.backend.add("fake/path", b"fake data") + # not found + exc.errno = errno.ENOENT + client.cwd.side_effect = exc + file_data = b"fake data" + with mock.patch("io.BytesIO") as tmp_file: + self.backend.add("fake/path", file_data) + # mkdirs has been called + mocked_mkdirs.assert_called() + client.storbinary.assert_called() + tmp_file.assert_called() + tmp_file.assert_called_with(file_data) + client.storbinary.assert_called_with( + "STOR upload/fake/path", tmp_file().__enter__() + ) + + @mock.patch(FTP_LIB_PATH) + def test_get(self, mocked_ftplib): + client = mocked_ftplib.FTP().__enter__() + content = b"filecontent" + with open("/tmp/fakefile2.txt", "w+b") as fakefile: + fakefile.write(content) + + def side_effect_retrlines(*args, **kwargs): + """ + Mock function to read tmp file. + """ + cmd, buff_write = args + with open("/tmp/fakefile2.txt", "r") as tmp_file: + buff_write(tmp_file.read()) + + client.retrlines.side_effect = side_effect_retrlines + self.assertEqual(self.backend.get("fake/path"), content) + + @mock.patch(FTP_LIB_PATH) + def test_list(self, mocked_ftplib): + client = mocked_ftplib.FTP().__enter__() + # simulate errors + exc = IOError() + # general + client.retrlines.side_effect = exc + with self.assertRaises(IOError): + self.backend.list_files() + # not found + exc.errno = errno.ENOENT + client.retrlines.side_effect = exc + self.assertEqual(self.backend.list_files(), []) + + def test_find_files(self): + good_filepaths = ["somepath/file%d.good" % x for x in range(1, 10)] + bad_filepaths = ["somepath/file%d.bad" % x for x in range(1, 10)] + mocked_filepaths = bad_filepaths + good_filepaths + backend = self.backend.sudo() + expected = good_filepaths[:] + expected = [backend.directory_path + "/" + path for path in good_filepaths] + self._test_find_files( + backend, ADAPTER_PATH, mocked_filepaths, r".*\.good$", expected + ) + + # Do not patch the entire ftplib otherwise the error_perm Exception + # become also a mock and then a traceback is genrated on the "except ftplib.error_perm" + # because this ftplib.error_perm is not really an Exception (but a mock)! + @mock.patch(FTP_LIB_PATH + ".FTP") + def test_move_files(self, mocked_ftplib): + client = mocked_ftplib().__enter__() + # simulate file is not already there + client.nlst.side_effect = ftplib.error_perm() + to_move = "move/from/path/myfile.txt" + to_path = "move/to/path" + self.backend.move_files([to_move], to_path) + # no need to delete it + client.delete.assert_not_called() + # rename gets called + client.rename.assert_called_with(to_move, to_move.replace("from", "to")) + # now try to override destination + client.nlst.side_effect = None + client.nlst.return_value = True + self.backend.move_files([to_move], to_path) + # client will delete it first + client.delete.assert_called_with(to_move.replace("from", "to")) + # then move it + client.rename.assert_called_with(to_move, to_move.replace("from", "to")) + + @mock.patch(FTP_LIB_PATH) + def test_delete(self, mocked_ftplib): + client = mocked_ftplib.FTP().__enter__() + path = "delete/a/path" + self.backend.delete(path) + client.delete.assert_called_with( + os.path.join(self.backend.directory_path, path) + ) + + @mock.patch(FTP_LIB_PATH) + def test_validate_config(self, mocked_ftplib): + client = mocked_ftplib.FTP().__enter__() + self.backend.action_test_config() + client.getwelcome.assert_called() + + @mock.patch(FTP_LIB_PATH) + def test_mkd(self, mocked_ftplib): + client = mocked_ftplib.FTP().__enter__() + # simulate errors + exc = IOError() + exc.errno = errno.ENOENT + # general + client.cwd.side_effect = exc + client.mkd.side_effect = exc + with self.assertRaises(OSError): + self.backend.add("fake/path", b"fake data") + client.mkd.assert_called() diff --git a/storage_backend_ftp/views/backend_storage_view.xml b/storage_backend_ftp/views/backend_storage_view.xml new file mode 100644 index 0000000000..b20d0f87be --- /dev/null +++ b/storage_backend_ftp/views/backend_storage_view.xml @@ -0,0 +1,28 @@ + + + + storage.backend + + + + + + + + + + + + + + + + + diff --git a/storage_backend_sftp/models/storage_backend.py b/storage_backend_sftp/models/storage_backend.py index eea928af91..850232f0ad 100644 --- a/storage_backend_sftp/models/storage_backend.py +++ b/storage_backend_sftp/models/storage_backend.py @@ -12,15 +12,17 @@ class StorageBackend(models.Model): backend_type = fields.Selection(selection_add=[("sftp", "SFTP")]) sftp_server = fields.Char(string="SFTP Host") - sftp_port = fields.Integer(string="Port", default=22) + sftp_port = fields.Integer(string="SFTP Port", default=22) sftp_auth_method = fields.Selection( - string="Authentification Method", + string="SFTP Authentification Method", selection=[("pwd", "Password"), ("ssh_key", "Private key")], default="pwd", required=True, ) - sftp_login = fields.Char(string="Login", help="Login to connect to sftp server") - sftp_password = fields.Char(string="Password") + sftp_login = fields.Char( + string="SFTP Login", help="Login to connect to sftp server" + ) + sftp_password = fields.Char(string="SFTP Password") sftp_ssh_private_key = fields.Text( string="SSH private key", help="It's recommended to not store the key here "