diff --git a/Dockerfile b/Dockerfile index 6fef130..b092617 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM deviantony/python-dev -RUN apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A RUN echo 'deb http://repo.percona.com/apt trusty main\ndeb-src http://repo.percona.com/apt trusty main'\ > /etc/apt/sources.list.d/percona.list diff --git a/README.rst b/README.rst index a18a958..c5e0d24 100644 --- a/README.rst +++ b/README.rst @@ -46,18 +46,27 @@ You can also specify the following options: * --log-file: Log file for the script (default: */var/log/mysql/pyxtrabackup.log*). * --out-file: Log file for innobackupex output (default: */var/log/mysql/xtrabackup.out*). * --backup-threads: You can specify more threads in order to backup quicker (default: 1). - +* --no-compress: Do not compress the backup archive. Restoration ----------- -The archive is containing a binary backup of a MySQL server, all you need to do in order to restore the backup is to extract the content of the archive in your MySQL datadir, setup the permissions for the files and start your server: +The archive is containing a binary backup of a MySQL server, all you need to do in order to restore the backup is to extract the content of the archive in your MySQL datadir, setup the permissions for the files and start your server. -:: +Clean the MySQL datadir:: $ sudo rm -rf /path/to/mysql/datadir/* + +If you compressed the archive, uncompress and extract it:: + $ sudo tar xvpzf /path/to/backup_archive.tar.gz -C /path/to/mysql/datadir -$ sudo chown -R mysql:mysql /path/to/mysql/datadir + +Otherwise you just need to extract it:: + +$ sudo tar xvpf /path/to/backup_archive.tar -C /path/to/mysql/datadir + +Then restart your MySQL server:: + $ sudo service mysql start Setup an incremental backup cycle @@ -91,6 +100,7 @@ You can also specify the following options: * --log-file: Log file for the script (default: */var/log/mysql/pyxtrabackup-inc.log*). * --out-file: Log file for innobackupex output (default: */var/log/mysql/xtrabackup.out*). * --backup-threads: You can specify more threads in order to backup quicker (default: 1). +* --no-compress: Do not compress the backup archives. Restoration @@ -98,14 +108,18 @@ Restoration *WARNING*: The folder structure and the file names created by the *pyxtrabackup-inc* binary needs to be respected in order to restore successfully: - * TIMESTAMP_FOLDER/INC/base_backup_DATETIME.tar.gz - * TIMESTAMP_FOLDER/INC/inc_1_backup_DATETIME.tar.gz - * TIMESTAMP_FOLDER/INC/inc_N_backup_DATETIME.tar.gzz + * TIMESTAMP_FOLDER/INC/base_backup_DATETIME.tar(.gz) + * TIMESTAMP_FOLDER/INC/inc_1_backup_DATETIME.tar(.gz) + * TIMESTAMP_FOLDER/INC/inc_N_backup_DATETIME.tar(.gz) To restore an incremental backup, you'll need to use the *pyxtrabackup-restore* binary the following way: :: $ pyxtrabackup-restore --base-archive= --incremental-archive= --user= +Also, if you did use the *--no-compress* option with the backup tools, you'll need to specify the *--uncompressed-archives* option: :: + +$ pyxtrabackup-restore --base-archive= --incremental-archive= --user= --uncompressed-archives + The binary will stop the MySQL service, remove all files present in MySQL datadir and import all the incremental backups up to the specified last incremental backup. For example, using the following parameters: :: @@ -125,6 +139,7 @@ You can also specify the following options: * --log-file: Log file for the script (default: */var/log/mysql/pyxtrabackup-restore.log*). * --out-file: Log file for innobackupex output (default: */var/log/mysql/xtrabackup.out*). * --backup-threads: You can specify more threads in order to backup quicker (default: 1). +* --uncompressed-archives: Do not try to uncompress backup archives. Use this option if you used the backup tool with --no-compress. Limitations =========== diff --git a/xtrabackup/backup_tools.py b/xtrabackup/backup_tools.py index 81b8e99..bd708e0 100644 --- a/xtrabackup/backup_tools.py +++ b/xtrabackup/backup_tools.py @@ -9,11 +9,12 @@ class BackupTool: - def __init__(self, log_file, output_file): + def __init__(self, log_file, output_file, no_compression): self.log_manager = log_manager.LogManager() self.stop_watch = timer.Timer() self.setup_logging(log_file) self.command_executor = CommandExecutor(output_file) + self.compress = not no_compression def setup_logging(self, log_file): self.logger = logging.getLogger(__name__) @@ -31,7 +32,10 @@ def prepare_workdir(self, path): filesystem_utils.mkdir_path(path, 0o755) self.workdir = path + '/xtrabackup_tmp' self.logger.debug("Temporary workdir: " + self.workdir) - self.archive_path = path + '/backup.tar.gz' + if self.compress: + self.archive_path = path + '/backup.tar.gz' + else: + self.archive_path = path + '/backup.tar' self.logger.debug("Temporary archive: " + self.archive_path) def prepare_repository(self, repository, incremental): @@ -51,7 +55,7 @@ def prepare_archive_name(self, incremental, incremental_cycle): else: backup_prefix = '' self.final_archive_path = filesystem_utils.prepare_archive_path( - self.backup_repository, backup_prefix) + self.backup_repository, backup_prefix, self.compress) def exec_incremental_backup(self, user, password, thread_count): self.stop_watch.start_timer() @@ -104,18 +108,18 @@ def prepare_backup(self, redo_logs): self.stop_watch.stop_timer(), self.stop_watch.duration_in_seconds()) - def compress_backup(self): + def archive_backup(self): self.stop_watch.start_timer() try: self.command_executor.create_archive( - self.workdir, self.archive_path) + self.workdir, self.archive_path, self.compress) except ProcessError: self.logger.error( - 'An error occured during the backup compression.', + 'An error occured during the archiving of the backup.', exc_info=True) self.clean() raise - self.logger.info("Backup compression time: %s - Duration: %s", + self.logger.info("Backup archiving time: %s - Duration: %s", self.stop_watch.stop_timer(), self.stop_watch.duration_in_seconds()) @@ -127,7 +131,7 @@ def transfer_backup(self, repository): self.final_archive_path) except Exception: self.logger.error( - 'An error occured during the backup compression.', + 'An error occured during the backup transfer.', exc_info=True) self.clean() raise @@ -178,14 +182,15 @@ def load_incremental_data(self): self.clean() raise - def start_full_backup(self, repository, workdir, user, password, threads): + def start_full_backup(self, repository, workdir, user, + password, threads): self.check_prerequisites(repository) self.prepare_workdir(workdir) self.prepare_repository(repository, False) self.prepare_archive_name(False, False) self.exec_full_backup(user, password, threads) self.prepare_backup(False) - self.compress_backup() + self.archive_backup() self.transfer_backup(repository) self.clean() @@ -202,6 +207,6 @@ def start_incremental_backup(self, repository, incremental, self.prepare_archive_name(incremental, True) self.exec_full_backup(user, password, threads) self.save_incremental_data(incremental) - self.compress_backup() + self.archive_backup() self.transfer_backup(repository) self.clean() diff --git a/xtrabackup/command_executor.py b/xtrabackup/command_executor.py index 279e3a9..9eb7274 100644 --- a/xtrabackup/command_executor.py +++ b/xtrabackup/command_executor.py @@ -70,19 +70,27 @@ def exec_chown(self, user, group, directory_path): command = ['/bin/chown', '-R', user + ':' + group, directory_path] self.exec_command(command) - def create_archive(self, directory, archive_path): + def create_archive(self, directory, archive_path, compress): + if compress: + tar_options = 'cpvzf' + else: + tar_options = 'cpvf' command = [ 'tar', - 'cpvzf', + tar_options, archive_path, '-C', directory, '.'] self.exec_command(command) - def extract_archive(self, archive_path, destination_path): + def extract_archive(self, archive_path, destination_path, compressed): + if compressed: + tar_options = 'xpvzf' + else: + tar_options = 'xpvf' command = [ 'tar', - 'xpvzf', + tar_options, archive_path, '-C', destination_path] diff --git a/xtrabackup/filesystem_utils.py b/xtrabackup/filesystem_utils.py index 6bcf016..93860b9 100644 --- a/xtrabackup/filesystem_utils.py +++ b/xtrabackup/filesystem_utils.py @@ -18,14 +18,17 @@ def create_sub_repository(repository_path, sub_directory): return sub_repository -def prepare_archive_path(archive_sub_repository, prefix): +def prepare_archive_path(archive_sub_repository, prefix, compress): archive_path = ''.join([ archive_sub_repository, '/', prefix, 'backup_', - datetime.datetime.now().strftime("%Y%m%d_%H%M"), - '.tar.gz']) + datetime.datetime.now().strftime("%Y%m%d_%H%M")]) + if compress: + archive_path = archive_path + '.tar.gz' + else: + archive_path = archive_path + '.tar' return archive_path diff --git a/xtrabackup/full_backup.py b/xtrabackup/full_backup.py index 6f3406c..63c7d3e 100755 --- a/xtrabackup/full_backup.py +++ b/xtrabackup/full_backup.py @@ -1,20 +1,36 @@ """Xtrabackup script Usage: - pyxtrabackup --user= [--password=] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] + pyxtrabackup --user= \ +[--password=] \ +[--tmp-dir=] \ +[--log-file=] \ +[--out-file=] \ +[--backup-threads=] \ +[--no-compress] pyxtrabackup (-h | --help) pyxtrabackup --version Options: - -h --help Show this screen. - --version Show version. - --user= MySQL user. - --password= MySQL password. - --tmp-dir= Temporary directory [default: /tmp]. - --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. - --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. - --backup-threads= Threads count [default: 1]. + -h --help \ + Show this screen. + --version \ + Show version. + --user= \ + MySQL user. + --password= \ + MySQL password. + --tmp-dir= \ + Temporary directory [default: /tmp]. + --log-file= \ + Log file [default: /var/log/mysql/pyxtrabackup.log]. + --out-file= \ + Output file [default: /var/log/mysql/xtrabackup.out]. + --backup-threads= \ + Threads count [default: 1]. + --no-compress \ + Do not create a compressed archive of the backup. """ from docopt import docopt @@ -25,7 +41,8 @@ def main(): arguments = docopt(__doc__, version='3.0.1') - backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file']) + backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file'], + arguments['--no-compress']) try: backup_tool.start_full_backup(arguments[''], arguments['--tmp-dir'], diff --git a/xtrabackup/incremental_backup.py b/xtrabackup/incremental_backup.py index 1cc98de..d9a4df4 100755 --- a/xtrabackup/incremental_backup.py +++ b/xtrabackup/incremental_backup.py @@ -1,21 +1,39 @@ """Xtrabackup script Usage: - pyxtrabackup-inc --user= [--password=] [--incremental] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] + pyxtrabackup-inc --user= \ +[--password=] \ +[--incremental] \ +[--tmp-dir=] \ +[--log-file=] \ +[--out-file=] \ +[--backup-threads=] \ +[--no-compress] pyxtrabackup-inc (-h | --help) pyxtrabackup --version Options: - -h --help Show this screen. - --version Show version. - --user= MySQL user. - --password= MySQL password. - --incremental Start an incremental cycle. - --tmp-dir= Temporary directory [default: /tmp]. - --log-file= Log file [default: /var/log/mysql/pyxtrabackup.log]. - --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. - --backup-threads= Threads count [default: 1]. + -h --help \ + Show this screen. + --version \ + Show version. + --user= \ + MySQL user. + --password= \ + MySQL password. + --incremental \ + Start an incremental cycle. + --tmp-dir= \ + Temporary directory [default: /tmp]. + --log-file= \ + Log file [default: /var/log/mysql/pyxtrabackup.log]. + --out-file= \ + Output file [default: /var/log/mysql/xtrabackup.out]. + --backup-threads= \ + Threads count [default: 1]. + --no-compress \ + Do not create a compressed archive of the backup. """ from docopt import docopt @@ -26,7 +44,8 @@ def main(): arguments = docopt(__doc__, version='3.0.1') - backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file']) + backup_tool = BackupTool(arguments['--log-file'], arguments['--out-file'], + arguments['--no-compress']) try: backup_tool.start_incremental_backup(arguments[''], arguments['--incremental'], diff --git a/xtrabackup/restoration.py b/xtrabackup/restoration.py index e392155..85d63ca 100644 --- a/xtrabackup/restoration.py +++ b/xtrabackup/restoration.py @@ -1,24 +1,49 @@ """Xtrabackup script Usage: - pyxtrabackup-restore --base-archive= --incremental-archive= --user= [--password=] [--data-dir=] [--restart] [--tmp-dir=] [--log-file=] [--out-file=] [--backup-threads=] + pyxtrabackup-restore --base-archive= \ +--incremental-archive= \ +--user= \ +[--password=] \ +[--data-dir=] \ +[--restart] \ +[--tmp-dir=] \ +[--log-file=] \ +[--out-file=] \ +[--backup-threads=] \ +[--uncompressed-archives] pyxtrabackup-restore (-h | --help) pyxtrabackup --version Options: - -h --help Show this screen. - --version Show version. - --user= MySQL user. - --password= MySQL password. - --base-archive= Base backup. - --incremental-archive= Incremental archive target. - --data-dir= MySQL server data directory [default: /var/lib/mysql] - --restart Restart the server after backup restoration. - --tmp-dir= Temporary directory [default: /tmp]. - --log-file= Log file [default: /var/log/mysql/pyxtrabackup-restore.log]. - --out-file= Output file [default: /var/log/mysql/xtrabackup.out]. - --backup-threads= Threads count [default: 1]. + -h --help \ + Show this screen. + --version \ + Show version. + --user= \ + MySQL user. + --password= \ + MySQL password. + --base-archive= \ + Base backup. + --incremental-archive= \ + Incremental archive target. + --data-dir= \ + MySQL server data directory [default: /var/lib/mysql] + --restart \ + Restart the server after backup restoration. + --tmp-dir= \ + Temporary directory [default: /tmp]. + --log-file= \ + Log file [default: /var/log/mysql/pyxtrabackup-restore.log]. + --out-file= \ + Output file [default: /var/log/mysql/xtrabackup.out]. + --backup-threads= \ + Threads count [default: 1]. + --uncompressed-archives \ + Specify that the backup archives are not compressed. \ +Use this option if you did backup with --no-compress. """ from docopt import docopt @@ -31,7 +56,8 @@ def main(): arguments = docopt(__doc__, version='3.0.0') restore_tool = RestorationTool(arguments['--log-file'], arguments['--out-file'], - arguments['--data-dir']) + arguments['--data-dir'], + arguments['--uncompressed-archives']) try: restore_tool.start_restoration(arguments['--base-archive'], arguments['--incremental-archive'], diff --git a/xtrabackup/restoration_tools.py b/xtrabackup/restoration_tools.py index 3f1b17c..df395b3 100644 --- a/xtrabackup/restoration_tools.py +++ b/xtrabackup/restoration_tools.py @@ -8,12 +8,13 @@ class RestorationTool: - def __init__(self, log_file, output_file, data_dir): + def __init__(self, log_file, output_file, data_dir, uncompressed_archives): self.log_manager = log_manager.LogManager() self.data_dir = data_dir self.stop_watch = timer.Timer() self.setup_logging(log_file) self.command_executor = CommandExecutor(output_file) + self.compressed_archives = not uncompressed_archives def setup_logging(self, log_file): self.logger = logging.getLogger(__name__) @@ -47,7 +48,9 @@ def clean_data_dir(self): def restore_base_backup(self, archive_path): self.stop_watch.start_timer() try: - self.command_executor.extract_archive(archive_path, self.data_dir) + self.command_executor.extract_archive(archive_path, + self.data_dir, + self.compressed_archives) self.command_executor.exec_backup_preparation(self.data_dir, True) except ProcessError: self.logger.error( @@ -84,7 +87,8 @@ def apply_incremental_backup(self, archive_repository, incremental_step): prefix, 'archive']) filesystem_utils.mkdir_path(extracted_archive_path, 0o755) self.command_executor.extract_archive(backup_archive, - extracted_archive_path) + extracted_archive_path, + self.compressed_archives) self.command_executor.exec_incremental_preparation( self.data_dir, extracted_archive_path)