From a7d1fb569d2bb21d1f56ab6c596150badb760663 Mon Sep 17 00:00:00 2001 From: Thomas Bruckmann Date: Tue, 26 Apr 2022 15:01:25 +0200 Subject: [PATCH 1/4] adds two more options and makes it xtrabackup 8 compatible --- pyxbackup | 48 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/pyxbackup b/pyxbackup index 61a4e8e..082d485 100644 --- a/pyxbackup +++ b/pyxbackup @@ -4,6 +4,12 @@ # # @author Jervin Real +# TODO: Based on Server Version if starts with 8 remove following: +# --no-timeout switch +# --binlog-info=on switch +# additionally make --parallel default 4 and configurable +# add --rebuild-threads for --rebuild-threads + import sys, traceback, os, errno, signal, socket import time, calendar, shutil, re, pwd import smtplib, MySQLdb, base64 @@ -60,6 +66,8 @@ xb_opt_encrypt = False xb_opt_encrypt_key_file = None xb_opt_extra_ibx_options = None xb_opt_purge_bitmaps = None +xb_opt_parallel = 4 +xb_opt_rebuild_threads = 4 xb_hostname = None xb_user = None @@ -2009,9 +2017,13 @@ def run_xb(): xb_prepared_backup = "%s/P_%s" % (xb_opt_work_dir, xb_last_full) if xb_ibx_bin != 'innobackupex': - xb_ibx_opts = ' --backup' + xb_ibx_opts + xb_ibx_opts = ' --backup' + xb_ibx_opts + # --no-timestamp option is not available anymore in V8 of xtrabackup + if XB_VERSION_MAJOR != 8: + xb_ibx_opts = ' --no-timestamp' + xb_ibx_opts + else: + xb_ibx_opts = ' --no-timestamp' + xb_ibx_opts - xb_ibx_opts = ' --no-timestamp' + xb_ibx_opts if xb_opt_mysql_user: xb_ibx_opts = (' --user=%s ' % xb_opt_mysql_user) + xb_ibx_opts @@ -2035,11 +2047,11 @@ def run_xb(): if xb_opt_compress_with == 'qpress': xb_ibx_opts += ' --compress --compress-threads=4' - xb_ibx_opts += ' --stream=xbstream --parallel=4' + xb_ibx_opts += ' --stream=xbstream' xb_ibx_opts += ' --extra-lsndir=' + xb_this_backup os.mkdir(xb_this_backup) - else: - xb_ibx_opts += ' --parallel=4' + + xb_ibx_opts += " --parallel=%d" % xb_opt_parallel if xb_opt_encrypt and not xb_opt_apply_log: xb_ibx_opts += ' --encrypt=%s --encrypt-threads=4 --encrypt-key-file=%s' % ( @@ -2057,8 +2069,12 @@ def run_xb(): xb_ibx_opts += ' ' + xb_opt_extra_ibx_options if xb_ibx_bin != 'innobackupex': - # --binlog-info on lp152764811 - xb_ibx_opts += ' --binlog-info=on --target-dir ' + xb_this_backup + # --binlog-info option is not available anymore in V8 of xtrabackup + if XB_VERSION_MAJOR != 8: + # --binlog-info on lp152764811 + xb_ibx_opts += ' --binlog-info=on --target-dir ' + xb_this_backup + else: + xb_ibx_opts += ' --target-dir ' + xb_this_backup else: xb_ibx_opts += ' ' + xb_this_backup @@ -2919,6 +2935,8 @@ def init(): global xb_opt_encrypt_key_file global xb_opt_extra_ibx_options global xb_opt_purge_bitmaps + global xb_opt_parallel + global xb_opt_rebuild_threads global xb_server_version global xb_server_type @@ -3079,6 +3097,10 @@ Valid commands are: help=('If Changed Page Tracking is enabled, should we automatically ' 'purge bitmaps? Requires that a valid mysql-user and mysql-pass ' 'with SUPER privieleges is specified.')) + parser.add_option('', '--parallel', dest='parallel', type="int", + help='How much parallel to use with innobackupex --parallel, default 4') + parser.add_option('', '--rebuild-threads', dest='rebuild_threads', type="int", + help='How much rebuild-threads to use with innobackupex --rebuild-threads, default 4') (options, args) = parser.parse_args() @@ -3208,6 +3230,12 @@ Valid commands are: if xb_cfg.has_option(xb_opt_config_section, 'purge_bitmaps'): xb_opt_purge_bitmaps = xb_cfg.get(xb_opt_config_section, 'purge_bitmaps') + if xb_cfg.has_option(xb_opt_config_section, 'parallel'): + xb_opt_parallel = int(xb_cfg.get(xb_opt_config_section, 'parallel')) + + if xb_cfg.has_option(xb_opt_config_section, 'rebuild_threads'): + xb_opt_rebuild_threads = int(xb_cfg.get(xb_opt_config_section, 'rebuild_threads')) + if options.mysql_user: xb_opt_mysql_user = options.mysql_user if options.mysql_pass: xb_opt_mysql_pass = options.mysql_pass if options.mysql_host: xb_opt_mysql_host = options.mysql_host @@ -3264,6 +3292,8 @@ Valid commands are: if options.encrypt_key_file: xb_opt_encrypt_key_file = options.encrypt_key_file if options.extra_ibx_options: xb_opt_extra_ibx_options = options.extra_ibx_options if options.purge_bitmaps: xb_opt_purge_bitmaps = options.purge_bitmaps + if options.parallel: xb_opt_parallel = options.parallel + if options.rebuild_threads: xb_opt_rebuild_threads = options.rebuild_threads if xb_cfg: _debug('Found config file: ', xb_opt_config) @@ -3801,6 +3831,10 @@ class PyxOptions(object): help=('If Changed Page Tracking is enabled, should we automatically ' 'purge bitmaps? Requires that a valid mysql-user and mysql-pass ' 'with SUPER privieleges is specified.')) + parser.add_option('', '--parallel', dest='parallel', type="int", + help='How much parallel to use with innobackupex --parallel, default 4') + parser.add_option('', '--rebuild-threads', dest='rebuild_threads', type="int", + help='How much rebuild-threads to use with innobackupex --rebuild-threads, default 4') (options, args) = parser.parse_args() From 1ee0fb8579c3ebca561cf0bd32dbfc47405a7b26 Mon Sep 17 00:00:00 2001 From: Thomas Bruckmann Date: Tue, 26 Apr 2022 15:58:34 +0200 Subject: [PATCH 2/4] extends to make it python3 compatible and works smoother without innobackupex installed --- pyxbackup | 108 +++++++++++++++++++++++++++--------------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/pyxbackup b/pyxbackup index 082d485..8dbad0e 100644 --- a/pyxbackup +++ b/pyxbackup @@ -4,17 +4,11 @@ # # @author Jervin Real -# TODO: Based on Server Version if starts with 8 remove following: -# --no-timeout switch -# --binlog-info=on switch -# additionally make --parallel default 4 and configurable -# add --rebuild-threads for --rebuild-threads - import sys, traceback, os, errno, signal, socket import time, calendar, shutil, re, pwd -import smtplib, MySQLdb, base64 +import smtplib, pymysql, base64 from datetime import datetime, timedelta -from ConfigParser import ConfigParser, NoOptionError +from configparser import ConfigParser, NoOptionError from optparse import OptionParser from subprocess import Popen, PIPE, STDOUT, CalledProcessError from struct import unpack @@ -204,7 +198,7 @@ def _xb_version(verstr = None, tof = False): # weird, xtrabackup outputs version # string on STDERR instead of STDOUT out, err = p.communicate() - ver = re.search('[version|server] ([\d\.]+)', err) + ver = re.search('[version|server] ([\d\.]+)', err.decode('utf-8')) major, minor, rev = ver.group(1).split('.') XB_VERSION_MAJOR = int(major) if major else 0 @@ -249,9 +243,10 @@ def _out(tag, *msgs): out = "[%s] %s: %s" % (date(time.time()), tag, s) if xb_log_fd is not None: - os.write(xb_log_fd, "%s\n" % out) + b = str.encode("%s\n" % out) + os.write(xb_log_fd, b) - if not xb_opt_quiet: print out + if not xb_opt_quiet: print(out) def _say(*msgs): _out('INFO', *msgs) @@ -323,8 +318,13 @@ def _read_magic_chunk(bfile, size): def _check_binary(name): bin = _which(name) - if bin is None: - _die("%s script is not found in $PATH" % name) + # Since innobackupex is deprecated since years, we just create a symlink to xtrabackup instead of failing + if name == 'innobackupex': + if bin is None: + os.symlink(_which('xtrabackup'),'/usr/bin/innobackupex') + else: + if bin is None: + _die("%s script is not found in $PATH" % name) return bin @@ -407,7 +407,7 @@ def _create_lock_file(): def _xb_logfile_copy(bkp): log_file = None - # When backup is not compressed we need to preserve the + # When backup is not compressed we need to preserve the # xtrabackup_logfile since preparing directly from the # stor_dir will touch the logfile and we cannot use it # again @@ -465,7 +465,7 @@ def _check_in_progress(): if is_backup: try: os.kill(pid, 0) - except OSError, e: + except OSError as e: if e.errno == errno.ESRCH: _die("%s lock file exists but process is not running" % XB_LCK_FILE) elif e.errno == errno.EPERM: @@ -600,7 +600,7 @@ def _apply_log(bkp, incrdir=None, final=False): if cfp.get(XB_BIN_NAME,'backup_type') == 'incremental': _say('Preparing incremental backup: ', bkp) - if xb_ibx_bin != 'innobackupex': + if xb_ibx_bin != 'innobackupex': ibx_opts += " --incremental-dir %s --target-dir %s" % (bkp, incrdir) else: ibx_opts += " --incremental-dir %s %s" % (bkp, incrdir) else: @@ -640,7 +640,7 @@ def _apply_log(bkp, incrdir=None, final=False): return True - except Exception, e: + except Exception as e: _error("Command was: ", ibx_cmd.replace(xb_opt_mysql_pass,"*******")) _error("Error: process exited with status %s" % str(e)) _error("Please check innobackupex log file at %s" % ibx_log) @@ -664,10 +664,10 @@ def _prepare_backup(bkp, prep, final=False): if is_cmp: prep_tmp = os.path.join(os.path.dirname(prep), this_bkp) if is_of_type == XB_CMD_FULL: - if not os.path.isdir(prep): os.mkdir(prep, 0755) + if not os.path.isdir(prep): os.mkdir(prep, 755) cmp_to = prep else: - if not os.path.isdir(prep_tmp): os.mkdir(prep_tmp, 0755) + if not os.path.isdir(prep_tmp): os.mkdir(prep_tmp, 755) cmp_to = prep_tmp for fmt in ['xbs.gz', 'tar.gz', 'xbs.qp', 'xbs.qp.xbcrypt', 'qp', 'qp.xbcrypt']: @@ -883,7 +883,7 @@ def _extract_xgz(xgz, dest): _debug("Running gzip command: %s" % gz_cmd) _debug("Running xbstream command: %s" % xbs_cmd) - if not os.path.isdir(dest): os.mkdir(dest, 0755) + if not os.path.isdir(dest): os.mkdir(dest, 755) if not xb_opt_debug: FNULL = open(os.devnull, 'w') @@ -920,7 +920,7 @@ def _extract_xbs(xbs, dest, meta = None): _rotate_xtrabackup_info(os.path.dirname(xbs)) - if not os.path.isdir(dest): os.mkdir(dest, 0755) + if not os.path.isdir(dest): os.mkdir(dest, 755) _say("Extracting from xbstream format: %s" % xbs) FNULL = None @@ -1083,7 +1083,7 @@ def _extract_stream_qpress(xbs, dest, meta = None): _debug("Running xbcrypt command: %s" % xbc_cmd) _debug("Running xbstream command: %s" % xbs_cmd) - if not os.path.isdir(dest): os.mkdir(dest, 0755) + if not os.path.isdir(dest): os.mkdir(dest, 755) if not xb_opt_debug: FNULL = open(os.devnull, 'w') @@ -1136,7 +1136,7 @@ def _extract_nostream_qpress(qp, dest, meta = None): if xbc_cmd is not None: _debug("Running xbcrypt command: %s" % xbc_cmd) - if not os.path.isdir(dest): os.mkdir(dest, 0755) + if not os.path.isdir(dest): os.mkdir(dest, 755) if is_encrypted: if not xb_opt_debug: @@ -1423,7 +1423,7 @@ def _notify_by_email(subject, msg="", to=None): s.sendmail(fr, recpt.split(','), hdr + msg) s.quit() - except Exception, e: + except Exception as e: if xb_opt_debug: traceback.print_exc() _die("Could not send mail ({0}): {1}".format(e.errno, e.strerror)) @@ -1484,7 +1484,7 @@ def _ssh_execute(cmd, out=False, nowait=False): return True - except Exception, e: + except Exception as e: _error("Command was: ", ssh_cmd.replace(xb_opt_mysql_pass,"*******")) _error("Error: process exited with status %s" % str(e)) _exit_code(XB_EXIT_REMOTE_CMD_FAIL) @@ -1528,7 +1528,7 @@ def _binlog_from_backup(backup, full=None): if binlog == 'None': _warn("Invalid binlog record from backup, found '%s'" % binlog) binlog = False - except NoOptionError, e: + except NoOptionError as e: _warn("No binlog information from specified backup!") return binlog @@ -1653,7 +1653,7 @@ def _stream_binlog_from(): _die("Failed to connect to remote host, ", "unable to check list of binary logs.") - cur = xb_mysqldb.cursor(MySQLdb.cursors.DictCursor) + cur = xb_mysqldb.cursor(pymysql.cursors.DictCursor) cur.execute('SHOW BINARY LOGS') logs = [] low = None @@ -1726,9 +1726,9 @@ def _purge_bitmaps_to(lsn): return False try: - cur = xb_mysqldb.cursor(MySQLdb.cursors.DictCursor) + cur = xb_mysqldb.cursor(pymysql.cursors.DictCursor) cur.execute("PURGE CHANGED_PAGE_BITMAPS BEFORE %s" % lsn) - except MySQLdb.OperationalError, e: + except pymysql.OperationalError as e: _error("Got MySQL error %d, \"%s\" at execute" % (e.args[0], e.args[1])) _error("Failed to purge bitmaps!") _exit_code(XB_EXIT_BITMAP_PURGE_FAIL) @@ -1917,7 +1917,7 @@ def _server_version(): return False try: - cur = xb_mysqldb.cursor(MySQLdb.cursors.DictCursor) + cur = xb_mysqldb.cursor(pymysql.cursors.DictCursor) cur.execute("SELECT @@global.version AS version") ver = cur.fetchone()['version'].split('-') db_close() @@ -1925,7 +1925,7 @@ def _server_version(): _say("Detected source server as %s %s" % (ver[1], ver[0])) return (ver[0], ver[1].lower()) - except MySQLdb.OperationalError, e: + except pymysql.OperationalError as e: _error("Got MySQL error %d, \"%s\" at execute" % (e.args[0], e.args[1])) _exit_code(XB_EXIT_BY_DEATH) raise Exception("Failed to check server version!") @@ -1992,8 +1992,8 @@ def run_meta_query(): else: v.append('NULL') if len(v) > 0: - print ' '.join([str(i) for i in v]) - else: print 'NULL' + print(' '.join([str(i) for i in v])) + else: print('NULL') return True @@ -2182,7 +2182,7 @@ def run_xb(): xb_backup_is_success = True xb_info_bkp_end = date(time.time(), '%Y_%m_%d-%H_%M_%S') - except Exception, e: + except Exception as e: if xb_opt_mysql_pass is not None: _error("Command was: ", run_cmd.replace(xb_opt_mysql_pass,"*******")) else: @@ -2412,17 +2412,17 @@ def run_xb_list(): if f in xb_incr_list and xb_incr_list[f] and len(xb_incr_list[f]) > 0: s += ", incrementals: " + str(xb_incr_list[f]) - print s + print(s) if xb_weekly_list is not None and len(xb_weekly_list) > 0: - print "# Weekly list: %s" % str(xb_weekly_list) + print("# Weekly list: %s" % str(xb_weekly_list)) if xb_monthly_list is not None and len(xb_monthly_list) > 0: - print "# Monthly list: %s" % str(xb_monthly_list) + print("# Monthly list: %s" % str(xb_monthly_list)) if xb_binlogs_list is not None and len(xb_binlogs_list) > 0: - print "# Binary logs from %s to %s, %d total" % ( - xb_binlogs_list[0], xb_binlogs_list[-1], len(xb_binlogs_list)) + print("# Binary logs from %s to %s, %d total" % ( + xb_binlogs_list[0], xb_binlogs_list[-1], len(xb_binlogs_list))) def run_status(): """Display status of last backup - excludes any currently running backup""" @@ -2445,7 +2445,7 @@ def run_status(): try: os.kill(pid, 0) - except OSError, e: + except OSError as e: if e.errno == errno.ESRCH: ret = 2 txt = 'PID/lock file exists but process is not running' @@ -2482,8 +2482,8 @@ def run_status(): elif ret == 1: txt = "WARN - %s" % txt else: txt = "CRITICAL - %s" % txt - if xb_opt_status_format == 'nagios': print txt - elif xb_opt_status_format == 'zabbix': print ret + if xb_opt_status_format == 'nagios': print(txt) + elif xb_opt_status_format == 'zabbix': print(ret) sys.exit(ret) def run_xb_restore_set(prepare_path=None, finalize=True): @@ -2736,7 +2736,7 @@ def run_binlog_stream(): list_binlogs() os.chdir(xb_cwd) - except Exception, e: + except Exception as e: _error("Command was: ", run_cmd.replace(xb_opt_mysql_pass,"*******")) _error("Error: process exited with status %s" % str(e)) _exit_code(XB_EXIT_BINLOG_STREAM_FAIL) @@ -2763,7 +2763,7 @@ def prune_full_incr(): else: w = d w_dir = os.path.join(xb_stor_weekly, w) - os.mkdir(w_dir, 0755) + os.mkdir(w_dir, 755) shutil.copytree( os.path.join(xb_stor_full, d), os.path.join(w_dir, 'full')) @@ -2858,6 +2858,8 @@ def db_connect(): params = dict() + params['host'] = xb_opt_mysql_host + params['autocommit'] = True if xb_opt_mysql_user is not None: params['user'] = xb_opt_mysql_user @@ -2872,11 +2874,9 @@ def db_connect(): params['read_default_group'] = 'client' try: - xb_mysqldb = MySQLdb.connect(xb_opt_mysql_host, **params) + xb_mysqldb = pymysql.connect(**params) - # MySQLdb for some reason has autoccommit off by default - xb_mysqldb.autocommit(True) - except MySQLdb.Error, e: + except pymysql.Error as e: _error("Error ", e.args[0], ": ", e.args[1]) return False @@ -3368,11 +3368,11 @@ def check_dirs(): xb_stor_monthly = xb_opt_stor_dir + '/monthly' xb_stor_binlogs = xb_opt_stor_dir + '/binlogs' - if not os.path.isdir(xb_stor_full): os.mkdir(xb_stor_full, 0755) - if not os.path.isdir(xb_stor_incr): os.mkdir(xb_stor_incr, 0755) - if not os.path.isdir(xb_stor_weekly): os.mkdir(xb_stor_weekly, 0755) - if not os.path.isdir(xb_stor_monthly): os.mkdir(xb_stor_monthly, 0755) - if not os.path.isdir(xb_stor_binlogs): os.mkdir(xb_stor_binlogs, 0755) + if not os.path.isdir(xb_stor_full): os.mkdir(xb_stor_full, 755) + if not os.path.isdir(xb_stor_incr): os.mkdir(xb_stor_incr, 755) + if not os.path.isdir(xb_stor_weekly): os.mkdir(xb_stor_weekly, 755) + if not os.path.isdir(xb_stor_monthly): os.mkdir(xb_stor_monthly, 755) + if not os.path.isdir(xb_stor_binlogs): os.mkdir(xb_stor_binlogs, 755) def list_backups(): """List all valid backups inside the store directory""" @@ -3624,7 +3624,7 @@ if __name__ == "__main__": xb_backup_summary, xb_opt_notify_on_success) sys.exit(xb_exit_code) - except Exception, e: + except Exception as e: if xb_opt_notify_by_email: _notify_by_email( "MySQL backup script at %s exception!" % xb_hostname, From d6981753d462d5490c4c12eea8dc05df1bc6a4aa Mon Sep 17 00:00:00 2001 From: Thomas Bruckmann Date: Tue, 26 Apr 2022 17:05:56 +0200 Subject: [PATCH 3/4] updates readme --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3f70a6b..e7a1f02 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,12 @@ Features Dependencies ============ -The script is initially tested only with Python 2.6 on CentOS 6.5 and Python 2.7 on Ubuntu 14.04 - running it on newer versions i.e. 3.x may lead to incompatibility issues. Will appreciate pointers/pull requests on making it compatible with Python 3.x! +Python 3.x compatible, needs "configparser" and "pymysql" to be installed. Of course xtrabackup is also needed. Tested with Percona MySQL 8 and xtrabackup 8. No guarantee to be backwards compatible to Python 2.7. Also it requires that the xtrabackup binaries i.e. innobackupex, xtrabackup*, xbstream are found in your PATH environment. + + Configuration ============= @@ -141,7 +143,11 @@ Below are some valid options recognized from the configuration file: # file if they are not in default locations ($PATH and /etc/pyxbackup.cnf) remote_script=/usr/local/bin/pyxbackup --config=/path/to/custom/pyxbackup.cnf + # configures --parallel switch of xtrabackup + parallel = 4 + # configures --rebuild-threads switch of xtrabackup (only needed if prepare is used) + rebuild_threads = 4 Minimum Configuration ===================== @@ -156,7 +162,7 @@ First, create your local backup folders and install a single dependency: mkdir /backups/folder/stor mkdir /backups/folder/work - yum install MySQL-python # apt-get install python-mysqldb + pip3 install ConfigParser pymysql wget https://raw.githubusercontent.com/dotmanila/pyxbackup/master/pyxbackup chmod 0755 pyxbackup From 13f3ebf189a13d4e42ca5b2eda7892418ee9a52a Mon Sep 17 00:00:00 2001 From: Thomas Bruckmann Date: Fri, 29 Apr 2022 09:15:06 +0200 Subject: [PATCH 4/4] fix missing rebuild-threads option --- pyxbackup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyxbackup b/pyxbackup index 8dbad0e..550f084 100644 --- a/pyxbackup +++ b/pyxbackup @@ -588,9 +588,11 @@ def _apply_log(bkp, incrdir=None, final=False): ibx_opts = "" if xb_ibx_bin != 'innobackupex': ibx_opts = '--prepare ' + if XB_VERSION_MAJOR == 8: + ibx_opts += ' --rebuild-threads=%d' % xb_opt_rebuild_threads else: ibx_opts = '--apply-log ' - ibx_opts += "--use-memory=%dM" % xb_opt_prepare_memory + ibx_opts += " --use-memory=%dM" % xb_opt_prepare_memory log_fd = None p_tee = None