From 52e56bd1d63e55caf5bc301adf9fc7799a4bcbb9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Jan 2019 05:04:24 +0100 Subject: [PATCH 1/3] WIP: incremental repo check --- src/borg/archiver.py | 8 +++- src/borg/remote.py | 5 ++- src/borg/repository.py | 94 +++++++++++++++++++++++++++--------------- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f2861d2b54..f33b7fc59e 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -290,8 +290,11 @@ def do_check(self, args, repository): if args.repo_only and any((args.verify_data, args.first, args.last, args.prefix)): self.print_error("--repository-only contradicts --first, --last, --prefix and --verify-data arguments.") return EXIT_ERROR + if args.repair and args.max_duration: + self.print_error("--repair does not allow --max-duration argument.") + return EXIT_ERROR if not args.archives_only: - if not repository.check(repair=args.repair, save_space=args.save_space): + if not repository.check(repair=args.repair, save_space=args.save_space, max_duration=args.max_duration): return EXIT_WARNING if args.prefix: args.glob_archives = args.prefix + '*' @@ -2871,6 +2874,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='attempt to repair any inconsistencies found') subparser.add_argument('--save-space', dest='save_space', action='store_true', help='work slower, but using less space') + subparser.add_argument('--max-duration', metavar='SECONDS', dest='max_duration', + type=int, default=0, + help='do only a partial repo check for max. SECONDS seconds (Default: unlimited)') define_archive_filters_group(subparser) subparser = subparsers.add_parser('key', parents=[mid_common_parser], add_help=False, diff --git a/src/borg/remote.py b/src/borg/remote.py index 970c8ea920..d311cac8d0 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -893,8 +893,9 @@ def open(self, path, create=False, lock_wait=None, lock=True, exclusive=False, a make_parent_dirs=False): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) - def check(self, repair=False, save_space=False): + @api(since=parse_version('1.0.0'), + max_duration={'since': parse_version('1.2.0a4')}) + def check(self, repair=False, save_space=False, max_duration=0): """actual remoting is done via self.call in the @api decorator""" @api(since=parse_version('1.0.0'), diff --git a/src/borg/repository.py b/src/borg/repository.py index 727d4989e1..f7c818f448 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -890,7 +890,7 @@ def _rebuild_sparse(self, segment): # The outcome of the DELETE has been recorded in the PUT branch already self.compact[segment] += size - def check(self, repair=False, save_space=False): + def check(self, repair=False, save_space=False, max_duration=0): """Check repository consistency This method verifies all segment checksums and makes sure @@ -932,10 +932,26 @@ def report_error(msg): self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! segment_count = sum(1 for _ in self.io.segment_iterator()) logger.debug('Found %d segments', segment_count) + + partial = bool(max_duration) + assert not (repair and partial) + mode = 'partial' if partial else 'full' + if partial: + # continue a past partial check (if any) or start one from beginning + last_segment_checked = self.config.getint('repository', 'last_segment_checked', fallback=-1) + logger.info('skipping to segments >= %d', last_segment_checked + 1) + else: + # start from the beginning and also forget about any potential past partial checks + last_segment_checked = -1 + self.config.remove_option('repository', 'last_segment_checked') + self.save_config(self.path, self.config) + t_start = time.monotonic() pi = ProgressIndicatorPercent(total=segment_count, msg='Checking segments %3.1f%%', step=0.1, msgid='repository.check') for i, (segment, filename) in enumerate(self.io.segment_iterator()): pi.show(i) + if segment <= last_segment_checked: + continue if segment > transaction_id: continue try: @@ -946,7 +962,18 @@ def report_error(msg): if repair: self.io.recover_segment(segment, filename) objects = list(self.io.iter_objects(segment)) - self._update_index(segment, objects, report_error) + if not partial: + self._update_index(segment, objects, report_error) + if partial and time.monotonic() > t_start + max_duration: + logger.info('finished partial segment check, last segment checked is %d', segment) + self.config.set('repository', 'last_segment_checked', str(segment)) + self.save_config(self.path, self.config) + break + else: + logger.info('finished segment check at segment %d', segment) + self.config.remove_option('repository', 'last_segment_checked') + self.save_config(self.path, self.config) + pi.finish() # self.index, self.segments, self.compact now reflect the state of the segment files up to # We might need to add a commit tag if no committed segment is found @@ -954,42 +981,43 @@ def report_error(msg): report_error('Adding commit tag to segment {}'.format(transaction_id)) self.io.segment = transaction_id + 1 self.io.write_commit() - logger.info('Starting repository index check') - if current_index and not repair: - # current_index = "as found on disk" - # self.index = "as rebuilt in-memory from segments" - if len(current_index) != len(self.index): - report_error('Index object count mismatch.') - logger.error('committed index: %d objects', len(current_index)) - logger.error('rebuilt index: %d objects', len(self.index)) - - line_format = '%-64s %-16s %-16s' - not_found = '' - logger.warning(line_format, 'ID', 'rebuilt index', 'committed index') - for key, value in self.index.iteritems(): - current_value = current_index.get(key, not_found) - if current_value != value: - logger.warning(line_format, bin_to_hex(key), value, current_value) - for key, current_value in current_index.iteritems(): - if key in self.index: - continue - value = self.index.get(key, not_found) - if current_value != value: - logger.warning(line_format, bin_to_hex(key), value, current_value) - elif current_index: - for key, value in self.index.iteritems(): - if current_index.get(key, (-1, -1)) != value: - report_error('Index mismatch for key {}. {} != {}'.format(key, value, current_index.get(key, (-1, -1)))) - if repair: - self.write_index() + if not partial: + logger.info('Starting repository index check') + if current_index and not repair: + # current_index = "as found on disk" + # self.index = "as rebuilt in-memory from segments" + if len(current_index) != len(self.index): + report_error('Index object count mismatch.') + logger.error('committed index: %d objects', len(current_index)) + logger.error('rebuilt index: %d objects', len(self.index)) + + line_format = '%-64s %-16s %-16s' + not_found = '' + logger.warning(line_format, 'ID', 'rebuilt index', 'committed index') + for key, value in self.index.iteritems(): + current_value = current_index.get(key, not_found) + if current_value != value: + logger.warning(line_format, bin_to_hex(key), value, current_value) + for key, current_value in current_index.iteritems(): + if key in self.index: + continue + value = self.index.get(key, not_found) + if current_value != value: + logger.warning(line_format, bin_to_hex(key), value, current_value) + elif current_index: + for key, value in self.index.iteritems(): + if current_index.get(key, (-1, -1)) != value: + report_error('Index mismatch for key {}. {} != {}'.format(key, value, current_index.get(key, (-1, -1)))) + if repair: + self.write_index() self.rollback() if error_found: if repair: - logger.info('Completed repository check, errors found and repaired.') + logger.info('Finished %s repository check, errors found and repaired.', mode) else: - logger.error('Completed repository check, errors found.') + logger.error('Finished %s repository check, errors found.', mode) else: - logger.info('Completed repository check, no problems found.') + logger.info('Finished %s repository check, no problems found.', mode) return not error_found or repair def scan_low_level(self): From bf46b70346013ef73e4d3b60752bfa06148b579b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 7 Mar 2019 00:05:56 +0100 Subject: [PATCH 2/3] fixup: add "previously" value check api decorator --- src/borg/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index d311cac8d0..56ad6c7652 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -894,7 +894,7 @@ def open(self, path, create=False, lock_wait=None, lock=True, exclusive=False, a """actual remoting is done via self.call in the @api decorator""" @api(since=parse_version('1.0.0'), - max_duration={'since': parse_version('1.2.0a4')}) + max_duration={'since': parse_version('1.2.0a4'), 'previously': 0}) def check(self, repair=False, save_space=False, max_duration=0): """actual remoting is done via self.call in the @api decorator""" From 485803052b6d545ca18577e47b972ae89aeb7fa2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 7 Mar 2019 00:17:01 +0100 Subject: [PATCH 3/3] fixup: don't run an archives check based on unknown quality repo idx --- src/borg/archiver.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f33b7fc59e..19a7d0fdb0 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -293,6 +293,13 @@ def do_check(self, args, repository): if args.repair and args.max_duration: self.print_error("--repair does not allow --max-duration argument.") return EXIT_ERROR + if args.max_duration and not args.repo_only: + # when doing a partial repo check, we can only check crc32 checksums in segment files, + # we can't build a fresh repo index in memory to verify the on-disk index against it. + # thus, we should not do an archives check based on a unknown-quality on-disk repo index. + # also, there is no max_duration support in the archives check code anyway. + self.print_error("--repository-only is required for --max-duration support.") + return EXIT_ERROR if not args.archives_only: if not repository.check(repair=args.repair, save_space=args.save_space, max_duration=args.max_duration): return EXIT_WARNING