From 9dd83e97372f66d8c6bed29f223cda587c1d7c51 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 17 Mar 2015 19:11:30 +0100 Subject: [PATCH 1/7] Replaced keyserver. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6fef130..b092617 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM deviantony/python-dev -RUN apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A RUN echo 'deb http://repo.percona.com/apt trusty main\ndeb-src http://repo.percona.com/apt trusty main'\ > /etc/apt/sources.list.d/percona.list From 67836e1b629bca8a7fcccd90acbfa21d6a7d5c6c Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 17 Mar 2015 19:11:48 +0100 Subject: [PATCH 2/7] Added --no-compress option for full backup. --- xtrabackup/backup_tools.py | 27 ++++++++++++++++----------- xtrabackup/command_executor.py | 8 ++++++-- xtrabackup/filesystem_utils.py | 9 ++++++--- xtrabackup/full_backup.py | 6 ++++-- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/xtrabackup/backup_tools.py b/xtrabackup/backup_tools.py index 81b8e99..bd708e0 100644 --- a/xtrabackup/backup_tools.py +++ b/xtrabackup/backup_tools.py @@ -9,11 +9,12 @@ class BackupTool: - def __init__(self, log_file, output_file): + def __init__(self, log_file, output_file, no_compression): self.log_manager = log_manager.LogManager() self.stop_watch = timer.Timer() self.setup_logging(log_file) self.command_executor = CommandExecutor(output_file) + self.compress = not no_compression def setup_logging(self, log_file): self.logger = logging.getLogger(__name__) @@ -31,7 +32,10 @@ def prepare_workdir(self, path): filesystem_utils.mkdir_path(path, 0o755) self.workdir = path + '/xtrabackup_tmp' self.logger.debug("Temporary workdir: " + self.workdir) - self.archive_path = path + '/backup.tar.gz' + if self.compress: + self.archive_path = path + '/backup.tar.gz' + else: + self.archive_path = path + '/backup.tar' self.logger.debug("Temporary archive: " + self.archive_path) def prepare_repository(self, repository, incremental): @@ -51,7 +55,7 @@ def prepare_archive_name(self, incremental, incremental_cycle): else: backup_prefix = '' self.final_archive_path = filesystem_utils.prepare_archive_path( - self.backup_repository, backup_prefix) + self.backup_repository, backup_prefix, self.compress) def exec_incremental_backup(self, user, password, thread_count): self.stop_watch.start_timer() @@ -104,18 +108,18 @@ def prepare_backup(self, redo_logs): self.stop_watch.stop_timer(), self.stop_watch.duration_in_seconds()) - def compress_backup(self): + def archive_backup(self): self.stop_watch.start_timer() try: self.command_executor.create_archive( - self.workdir, self.archive_path) + self.workdir, self.archive_path, self.compress) except ProcessError: self.logger.error( - 'An error occured during the backup compression.', + 'An error occured during the archiving of the backup.', exc_info=True) self.clean() raise - self.logger.info("Backup compression time: %s - Duration: %s", + self.logger.info("Backup archiving time: %s - Duration: %s", self.stop_watch.stop_timer(), self.stop_watch.duration_in_seconds()) @@ -127,7 +131,7 @@ def transfer_backup(self, repository): self.final_archive_path) except Exception: self.logger.error( - 'An error occured during the backup compression.', + 'An error occured during the backup transfer.', exc_info=True) self.clean() raise @@ -178,14 +182,15 @@ def load_incremental_data(self): self.clean() raise - def start_full_backup(self, repository, workdir, user, password, threads): + def start_full_backup(self, repository, workdir, user, + password, threads): self.check_prerequisites(repository) self.prepare_workdir(workdir) self.prepare_repository(repository, False) self.prepare_archive_name(False, False) self.exec_full_backup(user, password, threads) self.prepare_backup(False) - self.compress_backup() + self.archive_backup() self.transfer_backup(repository) self.clean() @@ -202,6 +207,6 @@ def start_incremental_backup(self, repository, incremental, self.prepare_archive_name(incremental, True) self.exec_full_backup(user, password, threads) self.save_incremental_data(incremental) - self.compress_backup() + self.archive_backup() self.transfer_backup(repository) self.clean() diff --git a/xtrabackup/command_executor.py b/xtrabackup/command_executor.py index 279e3a9..efa052f 100644 --- a/xtrabackup/command_executor.py +++ b/xtrabackup/command_executor.py @@ -70,10 +70,14 @@ def exec_chown(self, user, group, directory_path): command = ['/bin/chown', '-R', user + ':' + group, directory_path] self.exec_command(command) - def create_archive(self, directory, archive_path): + def create_archive(self, directory, archive_path, compress): + if compress: + tar_options = 'cpvzf' + else: + tar_options = 'cpvf' command = [ 'tar', - 'cpvzf', + tar_options, archive_path, '-C', directory, '.'] diff --git a/xtrabackup/filesystem_utils.py b/xtrabackup/filesystem_utils.py index 6bcf016..93860b9 100644 --- a/xtrabackup/filesystem_utils.py +++ b/xtrabackup/filesystem_utils.py @@ -18,14 +18,17 @@ def create_sub_repository(repository_path, sub_directory): return sub_repository -def prepare_archive_path(archive_sub_repository, prefix): +def prepare_archive_path(archive_sub_repository, prefix, compress): archive_path = ''.join([ archive_sub_repository, '/', prefix, 'backup_', - datetime.datetime.now().strftime("%Y%m%d_%H%M"), - '.tar.gz']) + datetime.datetime.now().strftime("%Y%m%d_%H%M")]) + if compress: + archive_path = archive_path + '.tar.gz' + else: + archive_path = archive_path + '.tar' return archive_path diff --git a/xtrabackup/full_backup.py b/xtrabackup/full_backup.py index 6f3406c..1ec7b71 100755 --- a/xtrabackup/full_backup.py +++ b/xtrabackup/full_backup.py @@ -1,7 +1,7 @@ """Xtrabackup script Usage: - pyxtrabackup --user= [--password=] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] + pyxtrabackup --user= [--password=] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] [--no-compress] pyxtrabackup (-h | --help) pyxtrabackup --version @@ -15,6 +15,7 @@ --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. --backup-threads= Threads count [default: 1]. + --no-compress Do not create a compressed archive of the backup [default: false]. """ from docopt import docopt @@ -25,7 +26,8 @@ def main(): arguments = docopt(__doc__, version='3.0.1') - backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file']) + backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file'], + arguments['--no-compress']) try: backup_tool.start_full_backup(arguments[''], arguments['--tmp-dir'], From a3ef0f0cc9cea77f6dfd50030c6cfb41dda9e5a6 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 17 Mar 2015 19:15:57 +0100 Subject: [PATCH 3/7] Updated documentation. --- README.rst | 17 +++++++++++++---- xtrabackup/full_backup.py | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index a18a958..466cf58 100644 --- a/README.rst +++ b/README.rst @@ -46,18 +46,27 @@ You can also specify the following options: * --log-file: Log file for the script (default: */var/log/mysql/pyxtrabackup.log*). * --out-file: Log file for innobackupex output (default: */var/log/mysql/xtrabackup.out*). * --backup-threads: You can specify more threads in order to backup quicker (default: 1). - +* --no-compress: Do not compress the backup archive. Restoration ----------- -The archive is containing a binary backup of a MySQL server, all you need to do in order to restore the backup is to extract the content of the archive in your MySQL datadir, setup the permissions for the files and start your server: +The archive is containing a binary backup of a MySQL server, all you need to do in order to restore the backup is to extract the content of the archive in your MySQL datadir, setup the permissions for the files and start your server. -:: +Clean the MySQL datadir:: $ sudo rm -rf /path/to/mysql/datadir/* + +If you compressed the archive, uncompress and extract it:: + $ sudo tar xvpzf /path/to/backup_archive.tar.gz -C /path/to/mysql/datadir -$ sudo chown -R mysql:mysql /path/to/mysql/datadir + +Otherwise you just need to extract it:: + +$ sudo tar xvpf /path/to/backup_archive.tar -C /path/to/mysql/datadir + +Then restart your MySQL server:: + $ sudo service mysql start Setup an incremental backup cycle diff --git a/xtrabackup/full_backup.py b/xtrabackup/full_backup.py index 1ec7b71..1e19841 100755 --- a/xtrabackup/full_backup.py +++ b/xtrabackup/full_backup.py @@ -15,7 +15,7 @@ --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. --backup-threads= Threads count [default: 1]. - --no-compress Do not create a compressed archive of the backup [default: false]. + --no-compress Do not create a compressed archive of the backup. """ from docopt import docopt From 83355eab4efb4110ece52a50488e322815842d3a Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 17 Mar 2015 19:19:27 +0100 Subject: [PATCH 4/7] Added the no-compress option to incremental backup. --- xtrabackup/incremental_backup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xtrabackup/incremental_backup.py b/xtrabackup/incremental_backup.py index 1cc98de..b7ced65 100755 --- a/xtrabackup/incremental_backup.py +++ b/xtrabackup/incremental_backup.py @@ -1,7 +1,7 @@ """Xtrabackup script Usage: - pyxtrabackup-inc --user= [--password=] [--incremental] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] + pyxtrabackup-inc --user= [--password=] [--incremental] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] [--no-compress] pyxtrabackup-inc (-h | --help) pyxtrabackup --version @@ -16,6 +16,7 @@ --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. --backup-threads= Threads count [default: 1]. + --no-compress Do not create a compressed archive of the backup. """ from docopt import docopt @@ -26,7 +27,8 @@ def main(): arguments = docopt(__doc__, version='3.0.1') - backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file']) + backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file'], + arguments['--no-compress']) try: backup_tool.start_incremental_backup(arguments[''], arguments['--incremental'], From 61520b639fe0f97709232d0182a34b4bc0d8d65a Mon Sep 17 00:00:00 2001 From: tony Date: Wed, 18 Mar 2015 08:17:06 +0100 Subject: [PATCH 5/7] Added the --uncompressed-archives option to the restoration tool. --- xtrabackup/command_executor.py | 8 ++++++-- xtrabackup/restoration.py | 6 ++++-- xtrabackup/restoration_tools.py | 10 +++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/xtrabackup/command_executor.py b/xtrabackup/command_executor.py index efa052f..9eb7274 100644 --- a/xtrabackup/command_executor.py +++ b/xtrabackup/command_executor.py @@ -83,10 +83,14 @@ def create_archive(self, directory, archive_path, compress): directory, '.'] self.exec_command(command) - def extract_archive(self, archive_path, destination_path): + def extract_archive(self, archive_path, destination_path, compressed): + if compressed: + tar_options = 'xpvzf' + else: + tar_options = 'xpvf' command = [ 'tar', - 'xpvzf', + tar_options, archive_path, '-C', destination_path] diff --git a/xtrabackup/restoration.py b/xtrabackup/restoration.py index e392155..f842d05 100644 --- a/xtrabackup/restoration.py +++ b/xtrabackup/restoration.py @@ -1,7 +1,7 @@ """Xtrabackup script Usage: - pyxtrabackup-restore --base-archive= --incremental-archive= --user= [--password=] [--data-dir=] [--restart] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] + pyxtrabackup-restore --base-archive= --incremental-archive= --user= [--password=] [--data-dir=] [--restart] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] [--uncompressed-archives] pyxtrabackup-restore (-h | --help) pyxtrabackup --version @@ -19,6 +19,7 @@ --log-file= Log file [default: /var/log/mysql/pyxtrabackup-restore.log]. --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. --backup-threads= Threads count [default: 1]. + --uncompressed-archives Specify that the backup archives are not compressed. Use this option if you did backup with --no-compress. """ from docopt import docopt @@ -31,7 +32,8 @@ def main(): arguments = docopt(__doc__, version='3.0.0') restore_tool = RestorationTool(arguments['--log-file'], arguments['--out-file'], - arguments['--data-dir']) + arguments['--data-dir'], + arguments['--uncompressed-archives']) try: restore_tool.start_restoration(arguments['--base-archive'], arguments['--incremental-archive'], diff --git a/xtrabackup/restoration_tools.py b/xtrabackup/restoration_tools.py index 3f1b17c..df395b3 100644 --- a/xtrabackup/restoration_tools.py +++ b/xtrabackup/restoration_tools.py @@ -8,12 +8,13 @@ class RestorationTool: - def __init__(self, log_file, output_file, data_dir): + def __init__(self, log_file, output_file, data_dir, uncompressed_archives): self.log_manager = log_manager.LogManager() self.data_dir = data_dir self.stop_watch = timer.Timer() self.setup_logging(log_file) self.command_executor = CommandExecutor(output_file) + self.compressed_archives = not uncompressed_archives def setup_logging(self, log_file): self.logger = logging.getLogger(__name__) @@ -47,7 +48,9 @@ def clean_data_dir(self): def restore_base_backup(self, archive_path): self.stop_watch.start_timer() try: - self.command_executor.extract_archive(archive_path, self.data_dir) + self.command_executor.extract_archive(archive_path, + self.data_dir, + self.compressed_archives) self.command_executor.exec_backup_preparation(self.data_dir, True) except ProcessError: self.logger.error( @@ -84,7 +87,8 @@ def apply_incremental_backup(self, archive_repository, incremental_step): prefix, 'archive']) filesystem_utils.mkdir_path(extracted_archive_path, 0o755) self.command_executor.extract_archive(backup_archive, - extracted_archive_path) + extracted_archive_path, + self.compressed_archives) self.command_executor.exec_incremental_preparation( self.data_dir, extracted_archive_path) From d329445789523f5506e24b85a557f43fee814cf0 Mon Sep 17 00:00:00 2001 From: tony Date: Wed, 18 Mar 2015 08:39:37 +0100 Subject: [PATCH 6/7] Fixed lint issues. --- xtrabackup/full_backup.py | 35 +++++++++++++++------ xtrabackup/incremental_backup.py | 39 +++++++++++++++++------- xtrabackup/restoration.py | 52 +++++++++++++++++++++++--------- 3 files changed, 91 insertions(+), 35 deletions(-) diff --git a/xtrabackup/full_backup.py b/xtrabackup/full_backup.py index 1e19841..63c7d3e 100755 --- a/xtrabackup/full_backup.py +++ b/xtrabackup/full_backup.py @@ -1,21 +1,36 @@ """Xtrabackup script Usage: - pyxtrabackup --user= [--password=] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] [--no-compress] + pyxtrabackup --user= \ +[--password=] \ +[--tmp-dir=] \ +[--log-file=] \ +[--out-file=] \ +[--backup-threads=] \ +[--no-compress] pyxtrabackup (-h | --help) pyxtrabackup --version Options: - -h --help Show this screen. - --version Show version. - --user= MySQL user. - --password= MySQL password. - --tmp-dir= Temporary directory [default: /tmp]. - --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. - --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. - --backup-threads= Threads count [default: 1]. - --no-compress Do not create a compressed archive of the backup. + -h --help \ + Show this screen. + --version \ + Show version. + --user= \ + MySQL user. + --password= \ + MySQL password. + --tmp-dir= \ + Temporary directory [default: /tmp]. + --log-file= \ + Log file [default: /var/log/mysql/pyxtrabackup.log]. + --out-file= \ + Output file [default: /var/log/mysql/xtrabackup.out]. + --backup-threads= \ + Threads count [default: 1]. + --no-compress \ + Do not create a compressed archive of the backup. """ from docopt import docopt diff --git a/xtrabackup/incremental_backup.py b/xtrabackup/incremental_backup.py index b7ced65..d9a4df4 100755 --- a/xtrabackup/incremental_backup.py +++ b/xtrabackup/incremental_backup.py @@ -1,22 +1,39 @@ """Xtrabackup script Usage: - pyxtrabackup-inc --user= [--password=] [--incremental] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] [--no-compress] + pyxtrabackup-inc --user= \ +[--password=] \ +[--incremental] \ +[--tmp-dir=] \ +[--log-file=] \ +[--out-file=] \ +[--backup-threads=] \ +[--no-compress] pyxtrabackup-inc (-h | --help) pyxtrabackup --version Options: - -h --help Show this screen. - --version Show version. - --user= MySQL user. - --password= MySQL password. - --incremental Start an incremental cycle. - --tmp-dir= Temporary directory [default: /tmp]. - --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. - --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. - --backup-threads= Threads count [default: 1]. - --no-compress Do not create a compressed archive of the backup. + -h --help \ + Show this screen. + --version \ + Show version. + --user= \ + MySQL user. + --password= \ + MySQL password. + --incremental \ + Start an incremental cycle. + --tmp-dir= \ + Temporary directory [default: /tmp]. + --log-file= \ + Log file [default: /var/log/mysql/pyxtrabackup.log]. + --out-file= \ + Output file [default: /var/log/mysql/xtrabackup.out]. + --backup-threads= \ + Threads count [default: 1]. + --no-compress \ + Do not create a compressed archive of the backup. """ from docopt import docopt diff --git a/xtrabackup/restoration.py b/xtrabackup/restoration.py index f842d05..85d63ca 100644 --- a/xtrabackup/restoration.py +++ b/xtrabackup/restoration.py @@ -1,25 +1,49 @@ """Xtrabackup script Usage: - pyxtrabackup-restore --base-archive= --incremental-archive= --user= [--password=] [--data-dir=] [--restart] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] [--uncompressed-archives] + pyxtrabackup-restore --base-archive= \ +--incremental-archive= \ +--user= \ +[--password=] \ +[--data-dir=] \ +[--restart] \ +[--tmp-dir=] \ +[--log-file=] \ +[--out-file=] \ +[--backup-threads=] \ +[--uncompressed-archives] pyxtrabackup-restore (-h | --help) pyxtrabackup --version Options: - -h --help Show this screen. - --version Show version. - --user= MySQL user. - --password= MySQL password. - --base-archive= Base backup. - --incremental-archive= Incremental archive target. - --data-dir= MySQL server data directory [default: /var/lib/mysql] - --restart Restart the server after backup restoration. - --tmp-dir= Temporary directory [default: /tmp]. - --log-file= Log file [default: /var/log/mysql/pyxtrabackup-restore.log]. - --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. - --backup-threads= Threads count [default: 1]. - --uncompressed-archives Specify that the backup archives are not compressed. Use this option if you did backup with --no-compress. + -h --help \ + Show this screen. + --version \ + Show version. + --user= \ + MySQL user. + --password= \ + MySQL password. + --base-archive= \ + Base backup. + --incremental-archive= \ + Incremental archive target. + --data-dir= \ + MySQL server data directory [default: /var/lib/mysql] + --restart \ + Restart the server after backup restoration. + --tmp-dir= \ + Temporary directory [default: /tmp]. + --log-file= \ + Log file [default: /var/log/mysql/pyxtrabackup-restore.log]. + --out-file= \ + Output file [default: /var/log/mysql/xtrabackup.out]. + --backup-threads= \ + Threads count [default: 1]. + --uncompressed-archives \ + Specify that the backup archives are not compressed. \ +Use this option if you did backup with --no-compress. """ from docopt import docopt From 14a72ace098064f251b2c9ec0dad84a5c832fb39 Mon Sep 17 00:00:00 2001 From: tony Date: Wed, 18 Mar 2015 08:44:53 +0100 Subject: [PATCH 7/7] Updated README. --- README.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 466cf58..c5e0d24 100644 --- a/README.rst +++ b/README.rst @@ -100,6 +100,7 @@ You can also specify the following options: * --log-file: Log file for the script (default: */var/log/mysql/pyxtrabackup-inc.log*). * --out-file: Log file for innobackupex output (default: */var/log/mysql/xtrabackup.out*). * --backup-threads: You can specify more threads in order to backup quicker (default: 1). +* --no-compress: Do not compress the backup archives. Restoration @@ -107,14 +108,18 @@ Restoration *WARNING*: The folder structure and the file names created by the *pyxtrabackup-inc* binary needs to be respected in order to restore successfully: - * TIMESTAMP_FOLDER/INC/base_backup_DATETIME.tar.gz - * TIMESTAMP_FOLDER/INC/inc_1_backup_DATETIME.tar.gz - * TIMESTAMP_FOLDER/INC/inc_N_backup_DATETIME.tar.gzz + * TIMESTAMP_FOLDER/INC/base_backup_DATETIME.tar(.gz) + * TIMESTAMP_FOLDER/INC/inc_1_backup_DATETIME.tar(.gz) + * TIMESTAMP_FOLDER/INC/inc_N_backup_DATETIME.tar(.gz) To restore an incremental backup, you'll need to use the *pyxtrabackup-restore* binary the following way: :: $ pyxtrabackup-restore --base-archive= --incremental-archive= --user= +Also, if you did use the *--no-compress* option with the backup tools, you'll need to specify the *--uncompressed-archives* option: :: + +$ pyxtrabackup-restore --base-archive= --incremental-archive= --user= --uncompressed-archives + The binary will stop the MySQL service, remove all files present in MySQL datadir and import all the incremental backups up to the specified last incremental backup. For example, using the following parameters: :: @@ -134,6 +139,7 @@ You can also specify the following options: * --log-file: Log file for the script (default: */var/log/mysql/pyxtrabackup-restore.log*). * --out-file: Log file for innobackupex output (default: */var/log/mysql/xtrabackup.out*). * --backup-threads: You can specify more threads in order to backup quicker (default: 1). +* --uncompressed-archives: Do not try to uncompress backup archives. Use this option if you used the backup tool with --no-compress. Limitations ===========