From 289a77e906db6995123793dd9c1546ef4349287f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 19 Jul 2023 15:31:17 +0200 Subject: [PATCH 01/32] add notification --- config.py | 39 +++++++++++++++++++++++++++++++++++++++ dbsync_daemon.py | 5 ++++- log_functions.py | 24 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index 1098b7a..8106679 100644 --- a/config.py +++ b/config.py @@ -13,6 +13,8 @@ import tempfile import pathlib import subprocess +import smtplib +import socket config = Dynaconf( envvar_prefix=False, @@ -100,6 +102,43 @@ def validate_config(config): ): raise ConfigError("Config error: Ignored tables parameter should be a list") + if "notification" in config: + settings = [ + "smtp_server", + "smtp_username", + "smtp_password", + "email_sender", + "email_subject", + "email_recipients", + ] + + for setting in settings: + if setting not in config.notification: + raise ConfigError(f"Config error: `{setting}` is missing from `notification`.") + + if not isinstance(config.notification.email_recipients, list): + raise ConfigError("Config error: `email_recipients` should be list of address.") + + smtp_conn: smtplib.SMTP = None + + try: + smtp_conn = smtplib.SMTP(config.notification.smtp_server) + except socket.gaierror: + raise ConfigError(f"Config error: Cannot connect to SMTP server: `{config.notification.smtp_server}`.") + + try: + smtp_conn.login(config.notification.smtp_username, config.notification.smtp_password) + except smtplib.SMTPAuthenticationError: + raise ConfigError( + f"Config error: Cannot login to SMTP server with user: `{config.notification.smtp_username}` and a specified password." + ) + + +def can_send_email(config: Dynaconf) -> bool: + if "notification" in config: + return True + return False + def get_ignored_tables( connection, diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 785d050..bdf9f47 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -21,9 +21,10 @@ validate_config, ConfigError, update_config_path, + can_send_email, ) -from log_functions import setup_logger, handle_error_and_exit +from log_functions import setup_logger, handle_error_and_exit, send_email def is_pyinstaller() -> bool: @@ -171,6 +172,8 @@ def main(): except dbsync.DbSyncError as e: logging.error(str(e)) + if can_send_email(config): + send_email(str(e), datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")) logging.debug("Going to sleep") time.sleep(sleep_time) diff --git a/log_functions.py b/log_functions.py index 40125d7..42ea935 100644 --- a/log_functions.py +++ b/log_functions.py @@ -2,6 +2,11 @@ import pathlib import sys import typing +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from config import config def filter_below_error(record): @@ -47,3 +52,22 @@ def log_verbosity_to_logging(verbosity: str): def handle_error_and_exit(error: typing.Union[str, Exception]): logging.error(str(error)) sys.exit(1) + + +def send_email(error: str, date_time: str = None) -> None: + smtp_conn = smtplib.SMTP(config.notification.smtp_server) + smtp_conn.login(config.notification.smtp_username, config.notification.smtp_password) + + msg = MIMEMultipart() + msg["Subject"] = config.notification.email_subject + msg["From"] = config.notification.email_sender + msg["To"] = ", ".join(config.notification.email_recipients) + msg.preamble = error + + if date_time: + error = f"{date_time}: {error}" + + msg.attach(MIMEText(error, "plain")) + + smtp_conn.sendmail(config.notification.smtp_username, config.notification.email_recipients, msg.as_string()) + smtp_conn.quit() From 99377869e08b5eca4f9d959e9fde97dcf8d60400 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 21 Jul 2023 11:47:10 +0200 Subject: [PATCH 02/32] move function --- config.py | 6 ------ dbsync_daemon.py | 8 ++++++-- log_functions.py | 8 ++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index 8106679..767d134 100644 --- a/config.py +++ b/config.py @@ -134,12 +134,6 @@ def validate_config(config): ) -def can_send_email(config: Dynaconf) -> bool: - if "notification" in config: - return True - return False - - def get_ignored_tables( connection, ): diff --git a/dbsync_daemon.py b/dbsync_daemon.py index bdf9f47..c46bdea 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -21,10 +21,14 @@ validate_config, ConfigError, update_config_path, - can_send_email, ) -from log_functions import setup_logger, handle_error_and_exit, send_email +from log_functions import ( + setup_logger, + handle_error_and_exit, + send_email, + can_send_email, +) def is_pyinstaller() -> bool: diff --git a/log_functions.py b/log_functions.py index 42ea935..dbfec20 100644 --- a/log_functions.py +++ b/log_functions.py @@ -6,6 +6,8 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from dynaconf import Dynaconf + from config import config @@ -71,3 +73,9 @@ def send_email(error: str, date_time: str = None) -> None: smtp_conn.sendmail(config.notification.smtp_username, config.notification.email_recipients, msg.as_string()) smtp_conn.quit() + + +def can_send_email(config: Dynaconf) -> bool: + if "notification" in config: + return True + return False From c16102befcdf144e1ca5506be9e1552a431bc9ac Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 21 Jul 2023 12:10:19 +0200 Subject: [PATCH 03/32] add tests --- test/test_config.py | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/test/test_config.py b/test/test_config.py index c8badfc..2b91a4f 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -13,6 +13,7 @@ validate_config, get_ignored_tables, ) +from log_functions import can_send_email from .conftest import ( _reset_config, @@ -256,3 +257,60 @@ def test_get_ignored_tables(): validate_config(config) ignored_tables = get_ignored_tables(config.connections[0]) assert ignored_tables == ["table"] + + +def test_config_notification_setup(): + _reset_config() + + # no NOTIFICATIONS set should pass but cannot send email + validate_config(config) + assert can_send_email(config) is False + + # incomplete setting + config.update( + { + "NOTIFICATION": { + "smtp_server": "server", + "smtp_username": "user", + } + } + ) + + with pytest.raises(ConfigError, match="Config error: `smtp_password`"): + validate_config(config) + + # another incomplete setting + _reset_config() + + config.update( + { + "NOTIFICATION": { + "smtp_server": "server", + "smtp_username": "user", + "smtp_password": "pass", + } + } + ) + + with pytest.raises(ConfigError, match="Config error: `email_sender`"): + validate_config(config) + + # complete setting but does not work + config.update( + { + "NOTIFICATION": { + "smtp_server": "server", + "smtp_username": "user", + "smtp_password": "pass", + "email_sender": "dbsync@sync.com", + "email_subject": "DB Sync Error", + "email_recipients": ["recipient1@test.com", "recipient2@test.com"], + } + } + ) + + with pytest.raises(ConfigError, match="Config error: Cannot connect to SMTP server"): + validate_config(config) + + # notifications are set, emails can be send - but this config was not validated, as it would be in real run + assert can_send_email(config) From a6360ce113395202817a4ec46fbef98f6eccef1c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 21 Jul 2023 12:13:45 +0200 Subject: [PATCH 04/32] only send email if previous error message was not the same as current --- dbsync_daemon.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index c46bdea..1f31f1e 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -153,6 +153,8 @@ def main(): handle_error_and_exit(e) else: + error_msg = None + if not args.skip_init: try: dbsync.dbsync_init(mc) @@ -176,8 +178,9 @@ def main(): except dbsync.DbSyncError as e: logging.error(str(e)) - if can_send_email(config): + if can_send_email(config) and error_msg != str(e): send_email(str(e), datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + error_msg = str(e) logging.debug("Going to sleep") time.sleep(sleep_time) From cb6ecd08687ac6e9072ce96b61007f40050f7082 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 21 Jul 2023 12:18:42 +0200 Subject: [PATCH 05/32] add notification setting example --- config.yaml.default | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config.yaml.default b/config.yaml.default index 1d5f03b..1d7035f 100644 --- a/config.yaml.default +++ b/config.yaml.default @@ -29,3 +29,18 @@ connections: daemon: # How often to synchronize (in seconds) sleep_time: 10 + +# optional setting, if you do not want notifications on sync errors to be send via email just delete this section +notification: + # address of stmp server to send emails + smtp_server: "xxx.xxx" + # smtp server user + smtp_username: "user@email.xxx" + # smtp user's password + smtp_password: "password" + # email sender to be displayed in the message + email_sender: "dbsync@info.com" + # email subject + email_subject: "DB Sync Error" + # list of recipients of the email notification + email_recipients: ["recipient1@email.xxx", "recipient1@email.xxx"] \ No newline at end of file From 087390ec109046b66ea2200a3e7540490d03edcf Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 21 Jul 2023 12:22:38 +0200 Subject: [PATCH 06/32] fix typo --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 767d134..5527f6b 100644 --- a/config.py +++ b/config.py @@ -117,7 +117,7 @@ def validate_config(config): raise ConfigError(f"Config error: `{setting}` is missing from `notification`.") if not isinstance(config.notification.email_recipients, list): - raise ConfigError("Config error: `email_recipients` should be list of address.") + raise ConfigError("Config error: `email_recipients` should be list of addresses.") smtp_conn: smtplib.SMTP = None From ee6ca5c79dab2626926dd72b4de6e1ea38343bd8 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 09:15:15 +0200 Subject: [PATCH 07/32] fix error messages --- config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 5527f6b..37620cc 100644 --- a/config.py +++ b/config.py @@ -123,14 +123,15 @@ def validate_config(config): try: smtp_conn = smtplib.SMTP(config.notification.smtp_server) - except socket.gaierror: + except OSError: raise ConfigError(f"Config error: Cannot connect to SMTP server: `{config.notification.smtp_server}`.") try: smtp_conn.login(config.notification.smtp_username, config.notification.smtp_password) - except smtplib.SMTPAuthenticationError: + except smtplib.SMTPAuthenticationError as e: raise ConfigError( f"Config error: Cannot login to SMTP server with user: `{config.notification.smtp_username}` and a specified password." + f"SMTP Error: {str(e.smtp_error)}." ) From 8aded713e0e918837456b6a6a47f4b9403511a44 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:22:56 +0200 Subject: [PATCH 08/32] password and user are optional --- config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/config.py b/config.py index 37620cc..34299a6 100644 --- a/config.py +++ b/config.py @@ -105,8 +105,6 @@ def validate_config(config): if "notification" in config: settings = [ "smtp_server", - "smtp_username", - "smtp_password", "email_sender", "email_subject", "email_recipients", From be36fcf1f095b1ebc1ee2dd846938975448ad243 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:23:20 +0200 Subject: [PATCH 09/32] new config --- config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config.py b/config.py index 34299a6..5518fc1 100644 --- a/config.py +++ b/config.py @@ -117,6 +117,18 @@ def validate_config(config): if not isinstance(config.notification.email_recipients, list): raise ConfigError("Config error: `email_recipients` should be list of addresses.") + if "use_ssl" in config.notification: + if not isinstance(config.notification.use_ssl, bool): + raise ConfigError("`use_ssl` must be set to either `true` or `false`.") + + if "use_tls" in config.notification: + if not isinstance(config.notification.use_tls, bool): + raise ConfigError("`use_tls` must be set to either `true` or `false`.") + + if "smtp_port" in config.notification: + if not isinstance(config.notification.smtp_port, int): + raise ConfigError("`smtp_port` must be set an integer.") + smtp_conn: smtplib.SMTP = None try: From 664b83d00c17a4fb6bd41ca900ad6428fd6099b7 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:24:07 +0200 Subject: [PATCH 10/32] move smtp functions to separate file --- log_functions.py | 28 ---------------------- smtp_functions.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 smtp_functions.py diff --git a/log_functions.py b/log_functions.py index dbfec20..bbe657b 100644 --- a/log_functions.py +++ b/log_functions.py @@ -2,9 +2,6 @@ import pathlib import sys import typing -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText from dynaconf import Dynaconf @@ -54,28 +51,3 @@ def log_verbosity_to_logging(verbosity: str): def handle_error_and_exit(error: typing.Union[str, Exception]): logging.error(str(error)) sys.exit(1) - - -def send_email(error: str, date_time: str = None) -> None: - smtp_conn = smtplib.SMTP(config.notification.smtp_server) - smtp_conn.login(config.notification.smtp_username, config.notification.smtp_password) - - msg = MIMEMultipart() - msg["Subject"] = config.notification.email_subject - msg["From"] = config.notification.email_sender - msg["To"] = ", ".join(config.notification.email_recipients) - msg.preamble = error - - if date_time: - error = f"{date_time}: {error}" - - msg.attach(MIMEText(error, "plain")) - - smtp_conn.sendmail(config.notification.smtp_username, config.notification.email_recipients, msg.as_string()) - smtp_conn.quit() - - -def can_send_email(config: Dynaconf) -> bool: - if "notification" in config: - return True - return False diff --git a/smtp_functions.py b/smtp_functions.py new file mode 100644 index 0000000..7629956 --- /dev/null +++ b/smtp_functions.py @@ -0,0 +1,59 @@ +import datetime +import smtplib +import typing +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from dynaconf import Dynaconf + + +def create_connection(config: Dynaconf) -> typing.Union[smtplib.SMTP_SSL, smtplib.SMTP]: + if "smtp_port" in config.notification: + port = config.notification.smtp_port + else: + port = smtplib.SMTP_PORT + + if "use_ssl" in config.notification and config.notification.use_ssl: + host = smtplib.SMTP_SSL(config.notification.smtp_server, port) + else: + host = smtplib.SMTP(config.notification.smtp_server, port) + + return host + + +def log_smtp_user(host: typing.Union[smtplib.SMTP_SSL, smtplib.SMTP], config: Dynaconf) -> None: + if "use_tls" in config.notification and config.notification.use_tls: + host.starttls() + if config.notification.smtp_username and config.notification.smtp_password: + host.login(config.notification.smtp_username, config.notification.smtp_password) + + +def send_email(error: str, config: Dynaconf) -> None: + smtp_conn = create_connection(config) + + log_smtp_user(smtp_conn, config) + + msg = MIMEMultipart() + msg["Subject"] = config.notification.email_subject + msg["From"] = config.notification.email_sender + msg["To"] = ", ".join(config.notification.email_recipients) + msg.preamble = error + + date_time = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") + + error = f"{date_time}: {error}" + + msg.attach(MIMEText(error, "plain")) + + sender_email = config.notification.email_sender + if "smtp_username" in config.notification: + sender_email = config.notification.smtp_username + + smtp_conn.sendmail(sender_email, config.notification.email_recipients, msg.as_string()) + smtp_conn.quit() + + +def can_send_email(config: Dynaconf) -> bool: + if "notification" in config: + return True + return False From 08d77c60f9eaec0e552cae9fcdf01f42bb3dcfb8 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:24:33 +0200 Subject: [PATCH 11/32] use new functions --- config.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 5518fc1..c24c016 100644 --- a/config.py +++ b/config.py @@ -132,17 +132,16 @@ def validate_config(config): smtp_conn: smtplib.SMTP = None try: - smtp_conn = smtplib.SMTP(config.notification.smtp_server) + smtp_conn = create_connection(config) except OSError: raise ConfigError(f"Config error: Cannot connect to SMTP server: `{config.notification.smtp_server}`.") try: - smtp_conn.login(config.notification.smtp_username, config.notification.smtp_password) + log_smtp_user(smtp_conn, config) except smtplib.SMTPAuthenticationError as e: - raise ConfigError( - f"Config error: Cannot login to SMTP server with user: `{config.notification.smtp_username}` and a specified password." - f"SMTP Error: {str(e.smtp_error)}." - ) + raise ConfigError(f"Config SMTP Error: {str(e.smtp_error)}.") + + smtp_conn.quit() def get_ignored_tables( From 3f3522f1fbfac2bdcc5c8252f4d4182d30d68b82 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:30:56 +0200 Subject: [PATCH 12/32] update tests --- test/test_config.py | 60 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/test/test_config.py b/test/test_config.py index 2b91a4f..584579d 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -7,17 +7,10 @@ """ import pytest -from config import ( - config, - ConfigError, - validate_config, - get_ignored_tables, -) -from log_functions import can_send_email +from config import ConfigError, config, get_ignored_tables, validate_config +from smtp_functions import can_send_email -from .conftest import ( - _reset_config, -) +from .conftest import _reset_config def test_config(): @@ -276,7 +269,7 @@ def test_config_notification_setup(): } ) - with pytest.raises(ConfigError, match="Config error: `smtp_password`"): + with pytest.raises(ConfigError, match="Config error: `email_sender`"): validate_config(config) # another incomplete setting @@ -288,13 +281,53 @@ def test_config_notification_setup(): "smtp_server": "server", "smtp_username": "user", "smtp_password": "pass", + "email_sender": "dbsync@info.com", } } ) - with pytest.raises(ConfigError, match="Config error: `email_sender`"): + with pytest.raises(ConfigError, match="Config error: `email_subject`"): validate_config(config) + # bool variable test + _reset_config() + + config.update( + { + "NOTIFICATION": { + "smtp_server": "server", + "smtp_username": "user", + "smtp_password": "pass", + "email_sender": "dbsync@info.com", + "email_subject": "DB Sync Error", + "email_recipients": ["recipient1@test.com", "recipient2@test.com"], + "use_ssl": "some_string", + } + } + ) + + with pytest.raises(ConfigError, match="`use_ssl` must be set to either `true` or `false`"): + validate_config(config) + + # int variable test + _reset_config() + + config.update( + { + "NOTIFICATION": { + "smtp_server": "server", + "smtp_username": "user", + "smtp_password": "pass", + "email_sender": "dbsync@info.com", + "email_subject": "DB Sync Error", + "email_recipients": ["recipient1@test.com", "recipient2@test.com"], + "smtp_port": "some_string", + } + } + ) + + with pytest.raises(ConfigError, match="`smtp_port` must be set an integer"): + validate_config(config) # complete setting but does not work config.update( { @@ -305,6 +338,9 @@ def test_config_notification_setup(): "email_sender": "dbsync@sync.com", "email_subject": "DB Sync Error", "email_recipients": ["recipient1@test.com", "recipient2@test.com"], + "smtp_port": 25, + "use_ssl": False, + "use_tls": True, } } ) From acde9c7af74e33014f837355deee53efd74d359d Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:31:22 +0200 Subject: [PATCH 13/32] add imports --- config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.py b/config.py index c24c016..82e9a7a 100644 --- a/config.py +++ b/config.py @@ -16,6 +16,8 @@ import smtplib import socket +from smtp_functions import create_connection, log_smtp_user + config = Dynaconf( envvar_prefix=False, settings_files=[], From a41dbc9e672174b0a14b9df9bb177965d5f73d3e Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:33:28 +0200 Subject: [PATCH 14/32] update usage --- dbsync_daemon.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 1f31f1e..6024b1e 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -13,22 +13,10 @@ import pathlib import dbsync -from version import ( - __version__, -) -from config import ( - config, - validate_config, - ConfigError, - update_config_path, -) - -from log_functions import ( - setup_logger, - handle_error_and_exit, - send_email, - can_send_email, -) +from config import ConfigError, config, update_config_path, validate_config +from log_functions import handle_error_and_exit, setup_logger +from smtp_functions import can_send_email, send_email +from version import __version__ def is_pyinstaller() -> bool: @@ -179,7 +167,7 @@ def main(): except dbsync.DbSyncError as e: logging.error(str(e)) if can_send_email(config) and error_msg != str(e): - send_email(str(e), datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + send_email(str(e), config) error_msg = str(e) logging.debug("Going to sleep") From 20199b5c864494d82200c04c7b9960e261f7bbdc Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:39:56 +0200 Subject: [PATCH 15/32] use simple EmailMessage --- smtp_functions.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/smtp_functions.py b/smtp_functions.py index 7629956..967605d 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -1,8 +1,7 @@ import datetime import smtplib import typing -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText +from email.message import EmailMessage from dynaconf import Dynaconf @@ -33,17 +32,16 @@ def send_email(error: str, config: Dynaconf) -> None: log_smtp_user(smtp_conn, config) - msg = MIMEMultipart() + msg = EmailMessage() msg["Subject"] = config.notification.email_subject msg["From"] = config.notification.email_sender msg["To"] = ", ".join(config.notification.email_recipients) - msg.preamble = error date_time = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") error = f"{date_time}: {error}" - msg.attach(MIMEText(error, "plain")) + msg.set_content(error) sender_email = config.notification.email_sender if "smtp_username" in config.notification: From e9b8eaa87a69ef20a20a8c2e7a370fabcb83b962 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:47:02 +0200 Subject: [PATCH 16/32] add option to send tes email --- dbsync_daemon.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 6024b1e..aad699e 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -3,14 +3,14 @@ # - pull # - push -import datetime -import sys -import time import argparse -import platform +import datetime import logging import os import pathlib +import platform +import sys +import time import dbsync from config import ConfigError, config, update_config_path, validate_config @@ -91,6 +91,11 @@ def main(): default="DEBUG", help="Set level of logging into log-file.", ) + parser.add_argument( + "--test-notification-email", + action="store_true", + help="Send test notification email using the `notification` settings. Should be used to validate settings.", + ) args = parser.parse_args() @@ -113,6 +118,11 @@ def main(): except ConfigError as e: handle_error_and_exit(e) + if args.test_notification_email: + send_email("Mergin DB Sync test email.", config) + logging.debug("Email send!") + sys.exit(0) + if args.force_init and args.skip_init: handle_error_and_exit("Cannot use `--force-init` with `--skip-init` Initialization is required. ") From 9a2431bbf29272441dc86838f09562ecd86d5778 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:52:58 +0200 Subject: [PATCH 17/32] fix error messages --- config.py | 7 ++++--- test/test_config.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index 82e9a7a..5304d20 100644 --- a/config.py +++ b/config.py @@ -121,15 +121,16 @@ def validate_config(config): if "use_ssl" in config.notification: if not isinstance(config.notification.use_ssl, bool): - raise ConfigError("`use_ssl` must be set to either `true` or `false`.") + raise ConfigError("Config error: `use_ssl` must be set to either `true` or `false`.") if "use_tls" in config.notification: if not isinstance(config.notification.use_tls, bool): - raise ConfigError("`use_tls` must be set to either `true` or `false`.") + raise ConfigError("Config error: `use_tls` must be set to either `true` or `false`.") if "smtp_port" in config.notification: if not isinstance(config.notification.smtp_port, int): - raise ConfigError("`smtp_port` must be set an integer.") + raise ConfigError("Config error: `smtp_port` must be set an integer.") + smtp_conn: smtplib.SMTP = None diff --git a/test/test_config.py b/test/test_config.py index 584579d..36e8bd8 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -306,7 +306,7 @@ def test_config_notification_setup(): } ) - with pytest.raises(ConfigError, match="`use_ssl` must be set to either `true` or `false`"): + with pytest.raises(ConfigError, match="Config error: `use_ssl` must be set to either `true` or `false`"): validate_config(config) # int variable test @@ -326,7 +326,7 @@ def test_config_notification_setup(): } ) - with pytest.raises(ConfigError, match="`smtp_port` must be set an integer"): + with pytest.raises(ConfigError, match="Config error: `smtp_port` must be set an integer"): validate_config(config) # complete setting but does not work config.update( From 20ee1b8968b60d7bccd20956ab65b07cacddcaeb Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 10:54:06 +0200 Subject: [PATCH 18/32] add minimal email interval --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index 5304d20..0b2412b 100644 --- a/config.py +++ b/config.py @@ -131,6 +131,9 @@ def validate_config(config): if not isinstance(config.notification.smtp_port, int): raise ConfigError("Config error: `smtp_port` must be set an integer.") + if "minimal_email_interval" in config.notification: + if not isinstance(config.notification.minimal_email_interval, (int, float)): + raise ConfigError("Config error: `minimal_email_interval` must be set a number.") smtp_conn: smtplib.SMTP = None From 0b3c50539e928d1ff075130d35d33f441327a76c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 26 Jul 2023 11:15:54 +0200 Subject: [PATCH 19/32] add time difference between emails --- dbsync_daemon.py | 8 +++++--- smtp_functions.py | 43 ++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index aad699e..c7d51e4 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -159,6 +159,8 @@ def main(): except dbsync.DbSyncError as e: handle_error_and_exit(e) + last_email_send = None + while True: print(datetime.datetime.now()) @@ -176,9 +178,9 @@ def main(): except dbsync.DbSyncError as e: logging.error(str(e)) - if can_send_email(config) and error_msg != str(e): - send_email(str(e), config) - error_msg = str(e) + if can_send_email(config): + send_email(str(e), last_email_send, config) + last_email_send = datetime.datetime.now() logging.debug("Going to sleep") time.sleep(sleep_time) diff --git a/smtp_functions.py b/smtp_functions.py index 967605d..42971c8 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -27,28 +27,41 @@ def log_smtp_user(host: typing.Union[smtplib.SMTP_SSL, smtplib.SMTP], config: Dy host.login(config.notification.smtp_username, config.notification.smtp_password) -def send_email(error: str, config: Dynaconf) -> None: - smtp_conn = create_connection(config) +def should_send_another_email( + current_time: datetime.datetime, last_email_send: typing.Optional[datetime.datetime], config: Dynaconf +) -> bool: + if last_email_send is None: + return True + min_time_diff = config.notification.minimal_email_interval if "minimal_email_interval" in config.notification else 4 + time_diff = current_time - last_email_send + return time_diff.seconds > (min_time_diff * 3600) + + +def send_email(error: str, last_email_send: typing.Optional[datetime.datetime], config: Dynaconf) -> None: + current_time = datetime.datetime.now() + + if should_send_another_email(current_time, last_email_send, config): + smtp_conn = create_connection(config) - log_smtp_user(smtp_conn, config) + log_smtp_user(smtp_conn, config) - msg = EmailMessage() - msg["Subject"] = config.notification.email_subject - msg["From"] = config.notification.email_sender - msg["To"] = ", ".join(config.notification.email_recipients) + msg = EmailMessage() + msg["Subject"] = config.notification.email_subject + msg["From"] = config.notification.email_sender + msg["To"] = ", ".join(config.notification.email_recipients) - date_time = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") + date_time = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") - error = f"{date_time}: {error}" + error = f"{date_time}: {error}" - msg.set_content(error) + msg.set_content(error) - sender_email = config.notification.email_sender - if "smtp_username" in config.notification: - sender_email = config.notification.smtp_username + sender_email = config.notification.email_sender + if "smtp_username" in config.notification: + sender_email = config.notification.smtp_username - smtp_conn.sendmail(sender_email, config.notification.email_recipients, msg.as_string()) - smtp_conn.quit() + smtp_conn.sendmail(sender_email, config.notification.email_recipients, msg.as_string()) + smtp_conn.quit() def can_send_email(config: Dynaconf) -> bool: From 712860cfb0681a0163f8e0277faf1eb905f8c896 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:53:22 +0200 Subject: [PATCH 20/32] fix message Co-authored-by: Martin Dobias --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 0b2412b..922af73 100644 --- a/config.py +++ b/config.py @@ -129,7 +129,7 @@ def validate_config(config): if "smtp_port" in config.notification: if not isinstance(config.notification.smtp_port, int): - raise ConfigError("Config error: `smtp_port` must be set an integer.") + raise ConfigError("Config error: `smtp_port` must be set to an integer.") if "minimal_email_interval" in config.notification: if not isinstance(config.notification.minimal_email_interval, (int, float)): From 9a6f026a36cf217c46e53a9f05b04fdd9025bdfb Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:53:36 +0200 Subject: [PATCH 21/32] fix message Co-authored-by: Martin Dobias --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 922af73..4b0b8e0 100644 --- a/config.py +++ b/config.py @@ -117,7 +117,7 @@ def validate_config(config): raise ConfigError(f"Config error: `{setting}` is missing from `notification`.") if not isinstance(config.notification.email_recipients, list): - raise ConfigError("Config error: `email_recipients` should be list of addresses.") + raise ConfigError("Config error: `email_recipients` should be a list of addresses.") if "use_ssl" in config.notification: if not isinstance(config.notification.use_ssl, bool): From c8b1d3a768a3f207ac5a2e2faebca135ccb64d9c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:54:01 +0200 Subject: [PATCH 22/32] fix message Co-authored-by: Martin Dobias --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 4b0b8e0..0835557 100644 --- a/config.py +++ b/config.py @@ -133,7 +133,7 @@ def validate_config(config): if "minimal_email_interval" in config.notification: if not isinstance(config.notification.minimal_email_interval, (int, float)): - raise ConfigError("Config error: `minimal_email_interval` must be set a number.") + raise ConfigError("Config error: `minimal_email_interval` must be set to a number.") smtp_conn: smtplib.SMTP = None From a289efafa0b80ec3051a2491fa266a4530afc130 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:31:45 +0200 Subject: [PATCH 23/32] zero port --- smtp_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smtp_functions.py b/smtp_functions.py index 42971c8..73a6e86 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -10,7 +10,7 @@ def create_connection(config: Dynaconf) -> typing.Union[smtplib.SMTP_SSL, smtpli if "smtp_port" in config.notification: port = config.notification.smtp_port else: - port = smtplib.SMTP_PORT + port = 0 if "use_ssl" in config.notification and config.notification.use_ssl: host = smtplib.SMTP_SSL(config.notification.smtp_server, port) From 9192e9ac20d20c18ee18f41b4ca1ee9114cb7e6c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:31:57 +0200 Subject: [PATCH 24/32] fix typo in variable --- smtp_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smtp_functions.py b/smtp_functions.py index 73a6e86..fec84ee 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -28,12 +28,12 @@ def log_smtp_user(host: typing.Union[smtplib.SMTP_SSL, smtplib.SMTP], config: Dy def should_send_another_email( - current_time: datetime.datetime, last_email_send: typing.Optional[datetime.datetime], config: Dynaconf + current_time: datetime.datetime, last_email_sent: typing.Optional[datetime.datetime], config: Dynaconf ) -> bool: - if last_email_send is None: + if last_email_sent is None: return True min_time_diff = config.notification.minimal_email_interval if "minimal_email_interval" in config.notification else 4 - time_diff = current_time - last_email_send + time_diff = current_time - last_email_sent return time_diff.seconds > (min_time_diff * 3600) From 7bb5759afb8a70b4a86bd4650160f418c69baf83 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:52:52 +0200 Subject: [PATCH 25/32] catch errors and simplify into single function --- config.py | 21 ++++++++------------- smtp_functions.py | 12 ++++-------- test/test_config.py | 2 +- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/config.py b/config.py index 0835557..0c0c59d 100644 --- a/config.py +++ b/config.py @@ -16,7 +16,7 @@ import smtplib import socket -from smtp_functions import create_connection, log_smtp_user +from smtp_functions import create_connection_and_log_user config = Dynaconf( envvar_prefix=False, @@ -135,19 +135,14 @@ def validate_config(config): if not isinstance(config.notification.minimal_email_interval, (int, float)): raise ConfigError("Config error: `minimal_email_interval` must be set to a number.") - smtp_conn: smtplib.SMTP = None - - try: - smtp_conn = create_connection(config) - except OSError: - raise ConfigError(f"Config error: Cannot connect to SMTP server: `{config.notification.smtp_server}`.") - try: - log_smtp_user(smtp_conn, config) - except smtplib.SMTPAuthenticationError as e: - raise ConfigError(f"Config SMTP Error: {str(e.smtp_error)}.") - - smtp_conn.quit() + smtp_conn = create_connection_and_log_user(config) + smtp_conn.quit() + except (OSError, smtplib.SMTPConnectError, smtplib.SMTPAuthenticationError) as e: + err = str(e) + if isinstance(e, (smtplib.SMTPConnectError, smtplib.SMTPAuthenticationError)): + err = str(e.smtp_error) + raise ConfigError(f"Config SMTP Error: {err}.") def get_ignored_tables( diff --git a/smtp_functions.py b/smtp_functions.py index fec84ee..5f4e481 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -6,7 +6,7 @@ from dynaconf import Dynaconf -def create_connection(config: Dynaconf) -> typing.Union[smtplib.SMTP_SSL, smtplib.SMTP]: +def create_connection_and_log_user(config: Dynaconf) -> typing.Union[smtplib.SMTP_SSL, smtplib.SMTP]: if "smtp_port" in config.notification: port = config.notification.smtp_port else: @@ -17,15 +17,13 @@ def create_connection(config: Dynaconf) -> typing.Union[smtplib.SMTP_SSL, smtpli else: host = smtplib.SMTP(config.notification.smtp_server, port) - return host - - -def log_smtp_user(host: typing.Union[smtplib.SMTP_SSL, smtplib.SMTP], config: Dynaconf) -> None: if "use_tls" in config.notification and config.notification.use_tls: host.starttls() if config.notification.smtp_username and config.notification.smtp_password: host.login(config.notification.smtp_username, config.notification.smtp_password) + return host + def should_send_another_email( current_time: datetime.datetime, last_email_sent: typing.Optional[datetime.datetime], config: Dynaconf @@ -41,9 +39,7 @@ def send_email(error: str, last_email_send: typing.Optional[datetime.datetime], current_time = datetime.datetime.now() if should_send_another_email(current_time, last_email_send, config): - smtp_conn = create_connection(config) - - log_smtp_user(smtp_conn, config) + smtp_conn = create_connection_and_log_user(config) msg = EmailMessage() msg["Subject"] = config.notification.email_subject diff --git a/test/test_config.py b/test/test_config.py index 36e8bd8..cd6458c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -345,7 +345,7 @@ def test_config_notification_setup(): } ) - with pytest.raises(ConfigError, match="Config error: Cannot connect to SMTP server"): + with pytest.raises(ConfigError, match="Config SMTP Error"): validate_config(config) # notifications are set, emails can be send - but this config was not validated, as it would be in real run From 18d1a3afc7da42212b1a9136becdea9066460a1c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:53:04 +0200 Subject: [PATCH 26/32] fix typo --- dbsync_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index c7d51e4..81111f8 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -120,7 +120,7 @@ def main(): if args.test_notification_email: send_email("Mergin DB Sync test email.", config) - logging.debug("Email send!") + logging.debug("Email sent!") sys.exit(0) if args.force_init and args.skip_init: From 18c22a371d11f26b20fb2e165bbbbf8a904195eb Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:57:34 +0200 Subject: [PATCH 27/32] add docstrings --- smtp_functions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/smtp_functions.py b/smtp_functions.py index 5f4e481..e5ab97f 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -7,6 +7,7 @@ def create_connection_and_log_user(config: Dynaconf) -> typing.Union[smtplib.SMTP_SSL, smtplib.SMTP]: + """Create connection and log user to the SMTP server using the configuration in config.""" if "smtp_port" in config.notification: port = config.notification.smtp_port else: @@ -28,6 +29,8 @@ def create_connection_and_log_user(config: Dynaconf) -> typing.Union[smtplib.SMT def should_send_another_email( current_time: datetime.datetime, last_email_sent: typing.Optional[datetime.datetime], config: Dynaconf ) -> bool: + """Checks if another email should be sent. The time difference to last sent email needs to larger them + minimal interval specified in config or 4 hours if no value was specified.""" if last_email_sent is None: return True min_time_diff = config.notification.minimal_email_interval if "minimal_email_interval" in config.notification else 4 @@ -36,6 +39,7 @@ def should_send_another_email( def send_email(error: str, last_email_send: typing.Optional[datetime.datetime], config: Dynaconf) -> None: + """Sends email with provided error using the settings in config.""" current_time = datetime.datetime.now() if should_send_another_email(current_time, last_email_send, config): @@ -61,6 +65,7 @@ def send_email(error: str, last_email_send: typing.Optional[datetime.datetime], def can_send_email(config: Dynaconf) -> bool: + """Checks if notification settings is in the config and emails can be send.""" if "notification" in config: return True return False From 3d997ab09a3e553670339e5b7acc24f0b7f63e57 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 14:59:57 +0200 Subject: [PATCH 28/32] email sending should fail without crashing the deamon --- smtp_functions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/smtp_functions.py b/smtp_functions.py index e5ab97f..78df4be 100644 --- a/smtp_functions.py +++ b/smtp_functions.py @@ -43,8 +43,6 @@ def send_email(error: str, last_email_send: typing.Optional[datetime.datetime], current_time = datetime.datetime.now() if should_send_another_email(current_time, last_email_send, config): - smtp_conn = create_connection_and_log_user(config) - msg = EmailMessage() msg["Subject"] = config.notification.email_subject msg["From"] = config.notification.email_sender @@ -60,8 +58,12 @@ def send_email(error: str, last_email_send: typing.Optional[datetime.datetime], if "smtp_username" in config.notification: sender_email = config.notification.smtp_username - smtp_conn.sendmail(sender_email, config.notification.email_recipients, msg.as_string()) - smtp_conn.quit() + try: + smtp_conn = create_connection_and_log_user(config) + smtp_conn.sendmail(sender_email, config.notification.email_recipients, msg.as_string()) + smtp_conn.quit() + except: + pass def can_send_email(config: Dynaconf) -> bool: From 63716b387410b0e9bda4546fc083297d97e303ba Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 15:00:09 +0200 Subject: [PATCH 29/32] restyle imports --- config.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 0c0c59d..1a62cc0 100644 --- a/config.py +++ b/config.py @@ -6,15 +6,14 @@ License: MIT """ -from dynaconf import ( - Dynaconf, -) -import platform -import tempfile import pathlib -import subprocess +import platform import smtplib import socket +import subprocess +import tempfile + +from dynaconf import Dynaconf from smtp_functions import create_connection_and_log_user From e64ef030013cb63a0aee9fa9addaff6dd8fe4795 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 15:14:41 +0200 Subject: [PATCH 30/32] notification description in use.md --- config.yaml.default | 15 --------------- docs/using.md | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/config.yaml.default b/config.yaml.default index 1d7035f..1d5f03b 100644 --- a/config.yaml.default +++ b/config.yaml.default @@ -29,18 +29,3 @@ connections: daemon: # How often to synchronize (in seconds) sleep_time: 10 - -# optional setting, if you do not want notifications on sync errors to be send via email just delete this section -notification: - # address of stmp server to send emails - smtp_server: "xxx.xxx" - # smtp server user - smtp_username: "user@email.xxx" - # smtp user's password - smtp_password: "password" - # email sender to be displayed in the message - email_sender: "dbsync@info.com" - # email subject - email_subject: "DB Sync Error" - # list of recipients of the email notification - email_recipients: ["recipient1@email.xxx", "recipient1@email.xxx"] \ No newline at end of file diff --git a/docs/using.md b/docs/using.md index 2edda0a..616cd1e 100644 --- a/docs/using.md +++ b/docs/using.md @@ -76,6 +76,7 @@ daemon: - `--log-verbosity` use `errors` or `messages` to specify what should be logged. Default is `messages`. +- `--test-notification-email` used to test send notification email (see below for details about sending emails in case of sync fails) ## Excluding tables from sync @@ -93,3 +94,36 @@ connections: - table1 - table2 ``` + +## Sending emails in case of synchronization fail + +User can be notified about sync fails via email by including `notification` with proper setting in the config file + +The command line option `--test-notification-email` can be used to only send the notification email without running the daemon. This should be useful to test the `notification` settings setup. + +```yaml +# optional setting, if you do not want notifications on sync errors to be send via email just delete this section +notification: + # first four options are required to be present + # address of stmp server to send emails + smtp_server: "xxx.xxx" + # email sender to be displayed in the message + email_sender: "dbsync@info.com" + # email subject + email_subject: "DB Sync Error" + # list of recipients of the email notification + email_recipients: ["recipient1@email.xxx", "recipient1@email.xxx"] + # the following configuration is optional and does not need to be specified + # smtp server user + smtp_username: "user@email.xxx" + # smtp user's password + smtp_password: "password" + # smtp port + smtp_port: 25 + # use ssl true/false + use_ssl: false + # use tls true/false + use_tls: false + # minimal interval for sending emails in hours (default value is 4) to avoid sending too many emails + minimal_email_interval: 4 +``` From 873bfaff17ef510ed3208f0d31423ca570e3dd99 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 15:15:22 +0200 Subject: [PATCH 31/32] drop unused imports --- log_functions.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/log_functions.py b/log_functions.py index bbe657b..40125d7 100644 --- a/log_functions.py +++ b/log_functions.py @@ -3,10 +3,6 @@ import sys import typing -from dynaconf import Dynaconf - -from config import config - def filter_below_error(record): """Only lets through log messages with log level below ERROR .""" From c4ab2720e5e8d61f8c07f0a14d211afa061cab68 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 4 Aug 2023 15:16:09 +0200 Subject: [PATCH 32/32] drop unused import --- config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/config.py b/config.py index 1a62cc0..e2380cc 100644 --- a/config.py +++ b/config.py @@ -9,7 +9,6 @@ import pathlib import platform import smtplib -import socket import subprocess import tempfile